diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4a029fcb14ea24..d68706cad19697 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,12 @@ # Longitudinal safety gate and its protected test surface /.github/CODEOWNERS @daggerhashjack @SloPOS +/.github/workflows/ @daggerhashjack /.github/workflows/tests.yaml @daggerhashjack @SloPOS /.github/ci/tesla_preap_longitudinal_mutations.py @daggerhashjack @SloPOS /.github/ci/test_tesla_preap_longitudinal_workflow.py @daggerhashjack @SloPOS /selfdrive/controls/tests/test_tesla_preap_longcontrol.py @daggerhashjack @SloPOS /opendbc_repo @daggerhashjack @SloPOS +/selfdrive/test/process_replay/ @daggerhashjack +/third_party/acados/ @daggerhashjack +/tools/ci/Dockerfile.replay @daggerhashjack +/Dockerfile.replay @daggerhashjack diff --git a/.github/ci/process_replay_staged_inventory.json b/.github/ci/process_replay_staged_inventory.json new file mode 100644 index 00000000000000..3d5dd2ea85d391 --- /dev/null +++ b/.github/ci/process_replay_staged_inventory.json @@ -0,0 +1,438 @@ +{ + "allowed_processes": [ + "calibrationd", + "card", + "controlsd", + "dmonitoringd", + "lagd", + "locationd", + "paramsd", + "plannerd", + "radard", + "selfdrived", + "torqued", + "ubloxd" + ], + "cases": [ + { + "car_brand": "HYUNDAI", + "case_id": "hyundai", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd", + "selfdrived", + "radard", + "plannerd", + "calibrationd", + "dmonitoringd", + "locationd", + "paramsd", + "ubloxd", + "torqued" + ], + "source": "regenAA0FC4ED71E|2025-04-08--22-57-50--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "HYUNDAI2", + "case_id": "hyundai2", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenAFB9780D823|2025-04-08--23-00-34--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "TOYOTA", + "case_id": "toyota", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd", + "selfdrived", + "radard", + "plannerd", + "calibrationd", + "dmonitoringd", + "locationd", + "paramsd", + "ubloxd", + "torqued" + ], + "source": "regen218A4DCFAA1|2025-04-08--22-57-51--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "TOYOTA3", + "case_id": "toyota3", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen1455E3B4BDF|2025-04-09--03-26-06--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "HONDA", + "case_id": "honda", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenB328FF8BA0A|2025-04-08--22-57-45--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "HONDA2", + "case_id": "honda2", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen6170C8C9A35|2025-04-08--22-57-46--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "CHRYSLER", + "case_id": "chrysler", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen5B28FC2A437|2025-04-08--23-04-24--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "RAM", + "case_id": "ram", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenBF81EA96E08|2025-04-08--23-06-54--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "SUBARU", + "case_id": "subaru", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen7366F13F6A1|2025-04-08--23-07-07--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "GM", + "case_id": "gm", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen1271097D038|2025-04-09--03-26-00--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "NISSAN", + "case_id": "nissan", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen15D60604EAB|2025-04-08--23-06-59--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "VOLKSWAGEN", + "case_id": "volkswagen", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen0F2F06C9539|2025-04-08--23-06-56--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "MAZDA", + "case_id": "mazda", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenACF84CCF482|2024-08-30--03-21-55--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "FORD", + "case_id": "ford", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen755D8CB1E1F|2025-04-08--23-13-43--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "RIVIAN", + "case_id": "rivian", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen5FCAC896BBE|2025-04-08--23-13-35--0", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "TESLA", + "case_id": "tesla", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "TESLA", + "case_id": "nap-preap-pedal-v1", + "custom_params": { + "NAPAdaptiveAccel": true, + "NAPBrakeFactor": 1.0, + "NAPFollowDistance": 7, + "NAPForcePreAP": true, + "NAPPedalCalibDone": true, + "NAPPedalCalibFactor": 0.9478672985781991, + "NAPPedalCalibMax": 110.23299810159, + "NAPPedalCalibMin": 0.73299810159, + "NAPPedalCalibZero": 4.732998101589998, + "NAPPedalCanBus": 2, + "NAPPedalEnabled": true, + "NAPPedalProfile": 4, + "NAPRadarBehindNosecone": true, + "NAPRadarEnabled": true, + "NAPRadarOffset": -0.5, + "NAPiBoosterEnabled": false + }, + "executable": false, + "params_digest": "b5217b239c759d5a960872a99f2e9ff6cacdb37aff6e99e1ee1bd0c20dead809", + "processes": [ + "card", + "controlsd", + "selfdrived", + "radard", + "plannerd", + "lagd" + ], + "source": "", + "source_bytes": null, + "source_sha256": null + }, + { + "car_brand": "TESLA", + "case_id": "nap-preap-no-pedal-v1", + "custom_params": { + "NAPAdaptiveAccel": true, + "NAPBrakeFactor": 1.0, + "NAPFollowDistance": 5, + "NAPForcePreAP": true, + "NAPPedalCalibDone": true, + "NAPPedalCalibFactor": 0.946969696969697, + "NAPPedalCalibMax": 110.43821722611, + "NAPPedalCalibMin": 0.9382172261100007, + "NAPPedalCalibZero": 4.838217226109999, + "NAPPedalCanBus": 2, + "NAPPedalEnabled": false, + "NAPPedalProfile": 4, + "NAPRadarBehindNosecone": true, + "NAPRadarEnabled": true, + "NAPRadarOffset": -0.25, + "NAPiBoosterEnabled": false + }, + "executable": false, + "params_digest": "96b5f9f76bdf20f24bf2a29d408d87e60ebf377eccc3858b4f0c0de7c3c2e786", + "processes": [ + "card", + "controlsd", + "selfdrived", + "radard", + "plannerd", + "lagd" + ], + "source": "", + "source_bytes": null, + "source_sha256": null + } + ], + "expected_cases": 18, + "expected_params_digest": "200ac7871bca5f1026c3a271864d88cc7a28e36f10051e85ce73205409658522", + "expected_sources_digest": "d08466e4dfe7796111b612aed7bbc2532bb40567fc85b9d71ea4343440d7736f", + "expected_tasks": 78, + "inventory": "staged", + "partition": { + "card": 18, + "controlsd": 18, + "lagd": 18, + "other": 24 + }, + "schema_version": 1, + "task_ids": [ + "hyundai:card", + "hyundai:controlsd", + "hyundai:lagd", + "hyundai:selfdrived", + "hyundai:radard", + "hyundai:plannerd", + "hyundai:calibrationd", + "hyundai:dmonitoringd", + "hyundai:locationd", + "hyundai:paramsd", + "hyundai:ubloxd", + "hyundai:torqued", + "hyundai2:card", + "hyundai2:controlsd", + "hyundai2:lagd", + "toyota:card", + "toyota:controlsd", + "toyota:lagd", + "toyota:selfdrived", + "toyota:radard", + "toyota:plannerd", + "toyota:calibrationd", + "toyota:dmonitoringd", + "toyota:locationd", + "toyota:paramsd", + "toyota:ubloxd", + "toyota:torqued", + "toyota3:card", + "toyota3:controlsd", + "toyota3:lagd", + "honda:card", + "honda:controlsd", + "honda:lagd", + "honda2:card", + "honda2:controlsd", + "honda2:lagd", + "chrysler:card", + "chrysler:controlsd", + "chrysler:lagd", + "ram:card", + "ram:controlsd", + "ram:lagd", + "subaru:card", + "subaru:controlsd", + "subaru:lagd", + "gm:card", + "gm:controlsd", + "gm:lagd", + "nissan:card", + "nissan:controlsd", + "nissan:lagd", + "volkswagen:card", + "volkswagen:controlsd", + "volkswagen:lagd", + "mazda:card", + "mazda:controlsd", + "mazda:lagd", + "ford:card", + "ford:controlsd", + "ford:lagd", + "rivian:card", + "rivian:controlsd", + "rivian:lagd", + "tesla:card", + "tesla:controlsd", + "tesla:lagd", + "nap-preap-pedal-v1:card", + "nap-preap-pedal-v1:controlsd", + "nap-preap-pedal-v1:selfdrived", + "nap-preap-pedal-v1:radard", + "nap-preap-pedal-v1:plannerd", + "nap-preap-pedal-v1:lagd", + "nap-preap-no-pedal-v1:card", + "nap-preap-no-pedal-v1:controlsd", + "nap-preap-no-pedal-v1:selfdrived", + "nap-preap-no-pedal-v1:radard", + "nap-preap-no-pedal-v1:plannerd", + "nap-preap-no-pedal-v1:lagd" + ] +} diff --git a/.github/ci/test_tesla_preap_longitudinal_workflow.py b/.github/ci/test_tesla_preap_longitudinal_workflow.py index 12e60164ee1c64..20b3aa2318aeb9 100644 --- a/.github/ci/test_tesla_preap_longitudinal_workflow.py +++ b/.github/ci/test_tesla_preap_longitudinal_workflow.py @@ -23,6 +23,13 @@ def normalized_lines(block: str) -> set[str]: return {line.strip().removesuffix("\\").rstrip() for line in block.splitlines()} +def test_nap_ci_uses_fork_lfs_storage(): + workflow = WORKFLOW_PATH.read_text() + + assert "OPENPILOT_LFS_URL: https://github.com/${{ github.repository }}.git/info/lfs" in normalized_lines(workflow) + assert 'command: git config lfs.url "$OPENPILOT_LFS_URL" && git lfs pull' in normalized_lines(workflow) + + def test_nap_branches_run_on_push(): workflow = WORKFLOW_PATH.read_text() push_config = indented_block(workflow, " push:") @@ -81,6 +88,7 @@ def test_additive_follow_telemetry_is_ignored_by_process_replay(): def main(): test_nap_branches_run_on_push() + test_nap_ci_uses_fork_lfs_storage() test_named_focused_job_is_present() test_focused_tests_and_mutations_are_pinned() test_focused_job_builds_generated_mpc_dependencies() diff --git a/.github/ci/test_validate_process_replay_release.py b/.github/ci/test_validate_process_replay_release.py new file mode 100644 index 00000000000000..7eb03dd33b3196 --- /dev/null +++ b/.github/ci/test_validate_process_replay_release.py @@ -0,0 +1,322 @@ +"""Contract tests for process-replay release validator and publisher gates. + +These tests prove fail-closed publication behavior against pending inventory and +common release/manifest tampering. They do not run process replay itself. +""" + +from __future__ import annotations + +import hashlib +import json +import zipfile +from pathlib import Path + +import pytest + +import validate_process_replay_release as v + + +CI_DIR = Path(__file__).resolve().parent +WORKFLOW_PATH = CI_DIR.parent / "workflows" / "process_replay_refs.yaml" +STAGED_INVENTORY = CI_DIR / "process_replay_staged_inventory.json" + +SOURCE_SHA = "a" * 40 +OPENDBC_SHA = "b" * 40 +OTHER_OPENDBC = "c" * 40 + + +def _canonical(obj) -> bytes: + return v.canonical_json(obj) + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _build_cases(executable: bool = True, pin_sources: bool = True) -> list[dict]: + cases = [] + # 16 active-like full/partial + 2 pending-style ids => 18 cases / 78 tasks + # card/controlsd/lagd for all 18 = 54; remaining 24 "other" on first 2 full cases (12 each). + # Use exact allowed process set from staged inventory contract. + allowed = [ + "calibrationd", "card", "controlsd", "dmonitoringd", "lagd", "locationd", + "paramsd", "plannerd", "radard", "selfdrived", "torqued", "ubloxd", + ] + other = [p for p in allowed if p not in {"card", "controlsd", "lagd"}] + assert len(other) == 9 + # two full cases with core+all other (2*(3+9)=24 other tasks), sixteen core-only (16*3=48), total 72? + + # Need 78 = 18*3 core + 24 other => other tasks across cases must total 24. + # 2 full cases with all 9 other would be 18 other; need 24 => use 2 cases with 9 other + 2 cases with 3 other? + + # Simpler: first case has 12 procs? allowed only has 9 other. + # 9+9+3+3 = 24 other with four cases carrying extras. + for idx in range(18): + cid = f"case{idx:02d}" + procs = ["card", "controlsd", "lagd"] + if idx < 2: + procs = procs + other # 9 other each => 18 + elif idx < 4: + procs = procs + other[:3] # 3 other each => 6; total other 24 + source = f"https://example.test/rlogs/{cid}.zst" if executable else "" + case = { + "case_id": cid, + "car_brand": f"BRAND{idx}", + "source": source, + "source_sha256": ("1" * 64) if pin_sources and executable else None, + "source_bytes": 16 if pin_sources and executable else None, + "processes": procs, + "params_digest": "2" * 64, + "custom_params": {}, + "executable": bool(executable and source), + } + cases.append(case) + return cases + + +def _inventory_from_cases(cases: list[dict]) -> dict: + allowed = sorted({ + "calibrationd", "card", "controlsd", "dmonitoringd", "lagd", "locationd", + "paramsd", "plannerd", "radard", "selfdrived", "torqued", "ubloxd", + }) + task_ids = [] + for case in cases: + for proc in case["processes"]: + task_ids.append(f"{case['case_id']}:{proc}") + assert len(cases) == 18 + assert len(task_ids) == 78 + sorted_cases = sorted(cases, key=lambda c: c["case_id"]) + sources_payload = [ + { + "case_id": c["case_id"], + "source": c["source"], + "source_bytes": c["source_bytes"], + "source_sha256": c["source_sha256"], + } + for c in sorted_cases + ] + params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] + return { + "schema_version": 1, + "inventory": "staged", + "expected_cases": 18, + "expected_tasks": 78, + "partition": {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}, + "allowed_processes": allowed, + "cases": cases, + "task_ids": task_ids, + "expected_sources_digest": _sha256(_canonical(sources_payload)), + "expected_params_digest": _sha256(_canonical(params_payload)), + } + + +def _write_release(release: Path, cases: list[dict], *, opendbc_sha: str = OPENDBC_SHA) -> str: + release.mkdir(parents=True, exist_ok=True) + (release / "ref_commit").write_text(SOURCE_SHA + "\n", encoding="utf-8") + refs = {} + for case in cases: + for proc in case["processes"]: + task_id = f"{case['case_id']}:{proc}" + filename = f"{case['case_id']}__{proc}__{SOURCE_SHA}.zst" + payload = f"{task_id}\n".encode() + (release / filename).write_bytes(payload) + refs[task_id] = { + "filename": filename, + "size": len(payload), + "sha256": _sha256(payload), + } + sorted_cases = sorted((v.case_identity(c) for c in cases), key=lambda c: c["case_id"]) + sources_payload = [ + { + "case_id": c["case_id"], + "source": c["source"], + "source_bytes": c["source_bytes"], + "source_sha256": c["source_sha256"], + } + for c in sorted_cases + ] + params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] + manifest = { + "schema_version": 1, + "cases": sorted_cases, + "refs": {k: refs[k] for k in sorted(refs)}, + "openpilot_sha": SOURCE_SHA, + "opendbc_sha": opendbc_sha, + "sources_digest": _sha256(_canonical(sources_payload)), + "params_digest": _sha256(_canonical(params_payload)), + "previous_accepted_tag": "", + } + raw = _canonical(manifest) + (release / "manifest.json").write_bytes(raw) + return _sha256(raw) + + +def test_current_staged_inventory_is_rejected_at_publish_boundary(): + trusted = json.loads(STAGED_INVENTORY.read_text(encoding="utf-8")) + with pytest.raises(SystemExit, match="publication requires"): + v.require_publishable_inventory(trusted) + + +def test_validate_release_fails_closed_on_pending_repo_inventory(tmp_path: Path): + # Ignore synthetic cases; exercise the real checked-in pending inventory path. + inv_path = STAGED_INVENTORY + release = tmp_path / "release" + release.mkdir() + (release / "ref_commit").write_text(SOURCE_SHA + "\n", encoding="utf-8") + (release / "manifest.json").write_text("{}\n", encoding="utf-8") + trusted = v.load_trusted_inventory(inv_path) + with pytest.raises(SystemExit, match="publication requires"): + v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256="d" * 64, + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + + +def test_happy_path_accepts_fully_pinned_inventory(tmp_path: Path): + cases = _build_cases(executable=True, pin_sources=True) + inventory = _inventory_from_cases(cases) + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + release = tmp_path / "release" + manifest_sha = _write_release(release, cases, opendbc_sha=OPENDBC_SHA) + trusted = v.load_trusted_inventory(inv_path) + result = v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=manifest_sha, + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + assert result["release_source_sha"] == SOURCE_SHA + assert result["release_manifest_sha256"] == manifest_sha + assert result["release_tag"] == f"process-replay/v1/{SOURCE_SHA}-{manifest_sha[:12]}" + + +def test_opendbc_gitlink_mismatch_is_rejected(tmp_path: Path): + cases = _build_cases() + inventory = _inventory_from_cases(cases) + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + release = tmp_path / "release" + manifest_sha = _write_release(release, cases, opendbc_sha=OTHER_OPENDBC) + trusted = v.load_trusted_inventory(inv_path) + with pytest.raises(SystemExit, match="opendbc_sha .* != trusted gitlink"): + v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=manifest_sha, + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + + +def test_manifest_case_tamper_is_rejected(tmp_path: Path): + cases = _build_cases() + inventory = _inventory_from_cases(cases) + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + release = tmp_path / "release" + _write_release(release, cases) + # Tamper one pinned source hash after the fact and rewrite canonical manifest. + manifest = json.loads((release / "manifest.json").read_text(encoding="utf-8")) + manifest["cases"][0]["source_sha256"] = "e" * 64 + # keep refs/digests stale on purpose + tampered = _canonical(manifest) + (release / "manifest.json").write_bytes(tampered) + trusted = v.load_trusted_inventory(inv_path) + with pytest.raises(SystemExit): + v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=_sha256(tampered), + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + + +def test_ref_bytes_tamper_is_rejected(tmp_path: Path): + cases = _build_cases() + inventory = _inventory_from_cases(cases) + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + release = tmp_path / "release" + manifest_sha = _write_release(release, cases) + # Corrupt one ref payload without updating manifest metadata. + victim = next(release.glob(f"*__card__{SOURCE_SHA}.zst")) + victim.write_bytes(b"tampered-ref-bytes") + trusted = v.load_trusted_inventory(inv_path) + with pytest.raises(SystemExit, match="(size mismatch|hash mismatch)"): + v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=manifest_sha, + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + + +def test_non_executable_inventory_rejected(): + cases = _build_cases(executable=False, pin_sources=False) + inventory = _inventory_from_cases(cases) + with pytest.raises(SystemExit, match="executable=true"): + v.require_publishable_inventory(inventory) + + +def test_workflow_authenticates_controller_run_and_artifact_name(): + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + assert 'EXPECTED_ARTIFACT_NAME="nap-replay-${CONTROLLER_RUN_ID}"' in workflow + assert 'repos/${CONTROLLER_REPO}/actions/runs/${CONTROLLER_RUN_ID}' in workflow + assert '.github/workflows/replay.yaml' in workflow + assert 'nap-replay' in workflow + assert "RUN_STATUS" in workflow and "RUN_CONCLUSION" in workflow + assert "RUN_HEAD_SHA" in workflow + assert "RUN_EVENT" in workflow + assert "META_NAME" in workflow + + +def test_workflow_rejects_duplicate_and_unsafe_zip_members(): + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + assert "duplicate zip member" in workflow + assert "non-regular zip member rejected" in workflow + assert "external_attr" in workflow + assert "O_EXCL" in workflow + + +def test_workflow_binds_handoff_review_proof_and_opendbc_gitlink(): + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + assert "handoff-meta.json" in workflow + assert "compare" in workflow and "results.json" in workflow + assert "pass count must be 78" in workflow or "pass', -1)) != 78" in workflow + assert "HEAD:opendbc_repo" in workflow + assert "--expected-opendbc-sha" in workflow + + +def test_zip_extractor_snippet_rejects_symlink_member(tmp_path: Path): + """Exercise the same mode checks the workflow embeds before extraction.""" + zpath = tmp_path / "cand.zip" + with zipfile.ZipFile(zpath, "w") as zf: + info = zipfile.ZipInfo("release/evil.zst") + # symlink mode + info.external_attr = (0o120777 & 0xFFFF) << 16 + zf.writestr(info, b"target") + + S_IFMT = 0o170000 + S_IFREG = 0o100000 + with zipfile.ZipFile(zpath) as zf: + for info in zf.infolist(): + mode = (info.external_attr >> 16) & 0xFFFF + ftype = mode & S_IFMT + assert ftype not in (0, S_IFREG) diff --git a/.github/ci/validate_process_replay_release.py b/.github/ci/validate_process_replay_release.py new file mode 100755 index 00000000000000..8f3ca1fd8e1bc6 --- /dev/null +++ b/.github/ci/validate_process_replay_release.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +"""Fail-closed validation of a process-replay release tree. + +Trusted staged inventory is loaded from the protected nap-dev checkout +(GITHUB_SHA). Controller-supplied inventory is never authoritative. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any, Never + +REQUIRED_MANIFEST_KEYS = ( + "schema_version", + "cases", + "refs", + "openpilot_sha", + "opendbc_sha", + "sources_digest", + "params_digest", + "previous_accepted_tag", +) +REQUIRED_CASE_KEYS = ( + "case_id", + "car_brand", + "source", + "source_sha256", + "source_bytes", + "processes", + "params_digest", + "custom_params", + "executable", +) +REQUIRED_REF_KEYS = ("filename", "size", "sha256") +CASE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +PROC_RE = re.compile(r"^[a-z0-9_]+$") +SHA40_RE = re.compile(r"^[0-9a-f]{40}$") +SHA64_RE = re.compile(r"^[0-9a-f]{64}$") +TAG_RE = re.compile(r"^process-replay/v1/([0-9a-f]{40})-([0-9a-f]{12})$") +CORE_PROCS = frozenset({"card", "controlsd", "lagd"}) + + +def die(msg: str) -> Never: + raise SystemExit(f"validate_process_replay_release: {msg}") + + +def canonical_json(obj: Any) -> bytes: + return (json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n").encode("utf-8") + + +def load_trusted_inventory(path: Path) -> dict[str, Any]: + if not path.is_file() or path.is_symlink(): + die(f"trusted inventory missing or symlink: {path}") + raw = path.read_bytes() + try: + inventory = json.loads(raw.decode("utf-8")) + except json.JSONDecodeError as exc: + die(f"trusted inventory is not valid JSON: {exc}") + if not isinstance(inventory, dict): + die("trusted inventory must be an object") + if inventory.get("schema_version") != 1: + die("trusted inventory schema_version must be 1") + if inventory.get("inventory") != "staged": + die("trusted inventory must declare inventory=staged") + if inventory.get("expected_cases") != 18 or inventory.get("expected_tasks") != 78: + die("trusted inventory must declare 18 cases / 78 tasks") + partition = inventory.get("partition") + if partition != {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}: + die(f"trusted inventory partition mismatch: {partition}") + cases = inventory.get("cases") + task_ids = inventory.get("task_ids") + allowed = inventory.get("allowed_processes") + if not isinstance(cases, list) or len(cases) != 18: + die("trusted inventory cases must be a list of 18") + if not isinstance(task_ids, list) or len(task_ids) != 78: + die("trusted inventory task_ids must be a list of 78") + if not isinstance(allowed, list) or not all(isinstance(p, str) for p in allowed): + die("trusted inventory allowed_processes must be a string list") + if list(allowed) != sorted(allowed): + die("trusted inventory allowed_processes must be sorted") + derived: list[str] = [] + seen_cases: set[str] = set() + for case in cases: + if not isinstance(case, dict): + die("trusted case must be an object") + if set(case) != set(REQUIRED_CASE_KEYS): + die(f"trusted case keys must be exactly {REQUIRED_CASE_KEYS}") + cid = case["case_id"] + if not isinstance(cid, str) or not CASE_ID_RE.fullmatch(cid): + die(f"trusted case_id invalid: {cid!r}") + if cid in seen_cases: + die(f"duplicate trusted case_id: {cid}") + seen_cases.add(cid) + procs = case["processes"] + if not isinstance(procs, list) or not procs or len(set(procs)) != len(procs): + die(f"trusted processes invalid for {cid}") + for proc in procs: + if not isinstance(proc, str) or not PROC_RE.fullmatch(proc) or proc not in allowed: + die(f"trusted process rejected for {cid}: {proc!r}") + derived.append(f"{cid}:{proc}") + if derived != task_ids: + die("trusted task_ids do not match derived case/process inventory") + if len(set(task_ids)) != 78: + die("trusted task_ids must be unique") + sorted_cases = sorted(cases, key=lambda c: c["case_id"]) + sources_payload = [ + { + "case_id": c["case_id"], + "source": c["source"], + "source_bytes": c["source_bytes"], + "source_sha256": c["source_sha256"], + } + for c in sorted_cases + ] + params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] + sources_digest = hashlib.sha256(canonical_json(sources_payload)).hexdigest() + params_digest = hashlib.sha256(canonical_json(params_payload)).hexdigest() + if inventory.get("expected_sources_digest") != sources_digest: + die("trusted expected_sources_digest mismatch against recomputed full sorted case records") + if inventory.get("expected_params_digest") != params_digest: + die("trusted expected_params_digest mismatch against recomputed full sorted case records") + counts = {"card": 0, "controlsd": 0, "lagd": 0, "other": 0} + for task_id in task_ids: + process = task_id.split(":", 1)[1] + if process in CORE_PROCS: + counts[process] += 1 + else: + counts["other"] += 1 + if counts != partition: + die(f"trusted partition derived mismatch: {counts}") + return inventory + + +def list_release_files(release: Path) -> list[str]: + if release.is_symlink() or not release.is_dir(): + die("release/ must be a real directory") + rel_files: list[str] = [] + for path in sorted(release.rglob("*")): + rel = path.relative_to(release).as_posix() + if ".." in Path(rel).parts or rel.startswith("/"): + die(f"illegal path {rel}") + if path.is_symlink(): + die(f"symlink forbidden: {rel}") + if path.is_dir(): + continue + if not path.is_file(): + die(f"non-regular file forbidden: {rel}") + if "/" in rel: + die(f"nested release paths forbidden: {rel}") + rel_files.append(rel) + return rel_files + + +def case_identity(case: dict[str, Any]) -> dict[str, Any]: + return {key: case[key] for key in REQUIRED_CASE_KEYS} + + +def require_publishable_case(case: dict[str, Any], *, where: str) -> None: + """Publication boundary: every case must be executable with pinned source bytes.""" + cid = case.get("case_id", "") + if case.get("executable") is not True: + die(f"{where} case {cid}: publication requires executable=true (descriptors not finalized)") + source = case.get("source") + if not isinstance(source, str) or not source.strip(): + die(f"{where} case {cid}: publication requires non-empty source locator/URL") + source_bytes = case.get("source_bytes") + if not isinstance(source_bytes, int) or isinstance(source_bytes, bool) or source_bytes <= 0: + die(f"{where} case {cid}: publication requires positive source_bytes") + source_sha256 = case.get("source_sha256") + if not isinstance(source_sha256, str) or not SHA64_RE.fullmatch(source_sha256): + die(f"{where} case {cid}: publication requires 64 lowercase hex source_sha256") + + +def require_publishable_inventory(trusted: dict[str, Any]) -> None: + """Fail closed on pending/stale inventory until fixture descriptors are finalized.""" + cases = trusted.get("cases") + if not isinstance(cases, list): + die("trusted inventory cases missing") + for case in cases: + if not isinstance(case, dict): + die("trusted case must be an object") + require_publishable_case(case, where="trusted inventory") + + +def validate_release( + release: Path, + *, + trusted: dict[str, Any], + source_sha: str, + operation: str, + expected_manifest_sha256: str | None, + rollback_tag: str | None, + expected_opendbc_sha: str | None, +) -> dict[str, str]: + if not SHA40_RE.fullmatch(source_sha) or source_sha != source_sha.lower(): + die("source_sha must be lowercase 40-hex") + if operation not in {"publish", "rollback"}: + die(f"unknown operation {operation}") + + # Dormant publisher must not accept current pending inventory. + require_publishable_inventory(trusted) + + expected_cases = int(trusted["expected_cases"]) + expected_tasks = int(trusted["expected_tasks"]) + partition = dict(trusted["partition"]) + allowed_processes = set(trusted["allowed_processes"]) + trusted_cases = {c["case_id"]: case_identity(c) for c in trusted["cases"]} + trusted_task_ids = list(trusted["task_ids"]) + trusted_task_set = set(trusted_task_ids) + trusted_sources_digest = trusted["expected_sources_digest"] + trusted_params_digest = trusted["expected_params_digest"] + + rel_files = list_release_files(release) + if "ref_commit" not in rel_files or "manifest.json" not in rel_files: + die("release tree missing ref_commit or manifest.json") + zsts = [name for name in rel_files if name.endswith(".zst")] + extras = [name for name in rel_files if name not in {"ref_commit", "manifest.json"} and not name.endswith(".zst")] + if extras: + die(f"unexpected release files: {extras}") + if len(zsts) != expected_tasks: + die(f"expected {expected_tasks} zst refs, found {len(zsts)}") + + ref_commit = (release / "ref_commit").read_text(encoding="utf-8").strip() + if ref_commit != source_sha: + die(f"ref_commit {ref_commit} != source_sha {source_sha}") + + manifest_bytes = (release / "manifest.json").read_bytes() + if not manifest_bytes.endswith(b"\n"): + die("manifest.json must end with LF") + manifest_sha = hashlib.sha256(manifest_bytes).hexdigest() + if operation == "publish": + if not expected_manifest_sha256 or not SHA64_RE.fullmatch(expected_manifest_sha256): + die("publish requires expected manifest sha256") + if manifest_sha != expected_manifest_sha256: + die("manifest sha256 mismatch against dispatch input") + + try: + manifest = json.loads(manifest_bytes.decode("utf-8")) + except json.JSONDecodeError as exc: + die(f"manifest.json is not valid JSON: {exc}") + if not isinstance(manifest, dict): + die("manifest must be an object") + if canonical_json(manifest) != manifest_bytes: + die("manifest.json is not canonical sorted UTF-8 JSON with LF") + if set(manifest) != set(REQUIRED_MANIFEST_KEYS): + die(f"manifest keys must be exactly {REQUIRED_MANIFEST_KEYS}; got {sorted(manifest)}") + if manifest["schema_version"] != 1: + die("manifest schema_version must be integer 1") + if manifest["openpilot_sha"] != source_sha: + die("manifest openpilot_sha must equal source_sha") + if not isinstance(manifest["opendbc_sha"], str) or not SHA40_RE.fullmatch(manifest["opendbc_sha"]): + die("opendbc_sha must be 40 lowercase hex") + if not expected_opendbc_sha or not SHA40_RE.fullmatch(expected_opendbc_sha): + die("expected_opendbc_sha must be lowercase 40-hex from trusted gitlink") + if manifest["opendbc_sha"] != expected_opendbc_sha: + die(f"manifest opendbc_sha {manifest['opendbc_sha']} != trusted gitlink {expected_opendbc_sha}") + previous = manifest["previous_accepted_tag"] + if previous != "" and not (isinstance(previous, str) and TAG_RE.fullmatch(previous)): + die("previous_accepted_tag must be empty or process-replay/v1/<40-hex>-<12-hex>") + + cases = manifest["cases"] + refs = manifest["refs"] + if not isinstance(cases, list) or not isinstance(refs, dict): + die("manifest must contain cases list and refs object") + if len(cases) != expected_cases: + die(f"expected {expected_cases} cases, found {len(cases)}") + if len(refs) != expected_tasks: + die(f"expected {expected_tasks} refs, found {len(refs)}") + + case_ids: list[str] = [] + for case in cases: + if not isinstance(case, dict): + die("manifest case must be an object") + if set(case) != set(REQUIRED_CASE_KEYS): + die(f"manifest case keys must be exactly {REQUIRED_CASE_KEYS}") + cid = case["case_id"] + if not isinstance(cid, str) or not CASE_ID_RE.fullmatch(cid): + die(f"invalid case_id in manifest: {cid!r}") + require_publishable_case(case, where="manifest") + case_ids.append(cid) + trusted_case = trusted_cases.get(cid) + if trusted_case is None: + die(f"manifest case_id not in trusted inventory: {cid}") + if case_identity(case) != trusted_case: + die(f"manifest case identity diverges from trusted inventory: {cid}") + if len(set(case_ids)) != expected_cases: + die("case_id values must be unique") + if case_ids != sorted(case_ids): + die("cases must be sorted by case_id") + if set(case_ids) != set(trusted_cases): + die("manifest case set must equal trusted staged inventory") + + sorted_cases = sorted((case_identity(c) for c in cases), key=lambda c: c["case_id"]) + sources_payload = [ + { + "case_id": c["case_id"], + "source": c["source"], + "source_bytes": c["source_bytes"], + "source_sha256": c["source_sha256"], + } + for c in sorted_cases + ] + params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] + sources_digest = hashlib.sha256(canonical_json(sources_payload)).hexdigest() + params_digest = hashlib.sha256(canonical_json(params_payload)).hexdigest() + if manifest["sources_digest"] != sources_digest or sources_digest != trusted_sources_digest: + die("sources_digest mismatch against recomputed full sorted case records / trusted inventory") + if manifest["params_digest"] != params_digest or params_digest != trusted_params_digest: + die("params_digest mismatch against recomputed full sorted case records / trusted inventory") + + if list(refs.keys()) != sorted(refs.keys()): + die("refs must be sorted by task key") + if set(refs) != trusted_task_set: + die("manifest refs task set must equal trusted staged task inventory") + + counts = {"card": 0, "controlsd": 0, "lagd": 0, "other": 0} + name_re = re.compile(rf"^[a-z0-9][a-z0-9-]*__[a-z0-9_]+__{re.escape(source_sha)}\.zst$") + for task_id, meta in refs.items(): + if not isinstance(task_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*:[a-z0-9_]+", task_id): + die(f"bad task id {task_id}") + if not isinstance(meta, dict) or set(meta) != set(REQUIRED_REF_KEYS): + die(f"ref metadata keys must be exactly {REQUIRED_REF_KEYS} for {task_id}") + case_id, process = task_id.split(":", 1) + if case_id not in trusted_cases: + die(f"ref case_id not in trusted cases: {case_id}") + if process not in allowed_processes: + die(f"unknown process rejected: {process}") + if process not in trusted_cases[case_id]["processes"]: + die(f"process {process} not in trusted case {case_id}") + if process in CORE_PROCS: + counts[process] += 1 + else: + counts["other"] += 1 + filename = meta["filename"] + size = meta["size"] + digest = meta["sha256"] + if not isinstance(filename, str) or filename not in zsts: + die(f"ref filename missing from tree: {filename}") + if not name_re.fullmatch(filename): + die(f"ref filename must encode source_sha: {filename}") + if filename != f"{case_id}__{process}__{source_sha}.zst": + die(f"filename does not match task identity: {filename}") + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + die(f"invalid size for {filename}") + if not isinstance(digest, str) or not SHA64_RE.fullmatch(digest): + die(f"invalid sha256 for {filename}") + data = (release / filename).read_bytes() + if size != len(data): + die(f"size mismatch for {filename}") + actual = hashlib.sha256(data).hexdigest() + if digest != actual: + die(f"hash mismatch for {filename}") + + if counts != partition: + die(f"partition mismatch: {counts} != {partition}") + if set(zsts) != {refs[task_id]["filename"] for task_id in refs}: + die("zst set does not equal manifest filenames") + + release_tag = f"process-replay/v1/{source_sha}-{manifest_sha[:12]}" + if operation == "rollback": + if not rollback_tag: + die("rollback requires rollback_tag") + if not TAG_RE.fullmatch(rollback_tag): + die("rollback_tag must match process-replay/v1/<40-hex>-<12-hex>") + if rollback_tag != release_tag: + die(f"rollback_tag {rollback_tag} is not cryptographically bound to fetched manifest {release_tag}") + + return { + "release_source_sha": source_sha, + "release_manifest_sha256": manifest_sha, + "release_tag": release_tag, + } + + +def write_github_output(result: dict[str, str]) -> None: + out_path = os.environ.get("GITHUB_OUTPUT") + if not out_path: + return + with open(out_path, "a", encoding="utf-8") as handle: + for key, value in result.items(): + handle.write(f"{key}={value}\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--release-dir", required=True, type=Path) + parser.add_argument("--trusted-inventory", required=True, type=Path) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--operation", required=True, choices=("publish", "rollback")) + parser.add_argument("--expected-manifest-sha256", default="") + parser.add_argument("--rollback-tag", default="") + parser.add_argument( + "--expected-opendbc-sha", + required=True, + help="Trusted opendbc_repo gitlink SHA for source_sha (lowercase 40-hex)", + ) + args = parser.parse_args() + + trusted = load_trusted_inventory(args.trusted_inventory) + result = validate_release( + args.release_dir, + trusted=trusted, + source_sha=args.source_sha.strip().lower(), + operation=args.operation, + expected_manifest_sha256=(args.expected_manifest_sha256 or None), + rollback_tag=(args.rollback_tag or None), + expected_opendbc_sha=args.expected_opendbc_sha.strip().lower(), + ) + write_github_output(result) + print(f"validated release tree source_sha={result['release_source_sha']} tasks={trusted['expected_tasks']} tag={result['release_tag']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/process_replay_refs.yaml b/.github/workflows/process_replay_refs.yaml new file mode 100644 index 00000000000000..bc3ae3b481be05 --- /dev/null +++ b/.github/workflows/process_replay_refs.yaml @@ -0,0 +1,694 @@ +# GitHub-hosted reference publisher only. Never runs on self-hosted infrastructure. +# Candidate files are validated fail-closed from an immutable controller artifact +# against the trusted staged inventory checked out at GITHUB_SHA. +# CI_ARTIFACTS_TOKEN is exposed only in the final approved push step. +name: process replay refs + +on: + workflow_dispatch: + inputs: + operation: + description: "publish a verified controller candidate, or rollback to a protected tag" + type: choice + options: + - publish + - rollback + default: publish + controller_run_id: + description: "Private controller Actions run ID that uploaded the candidate" + required: false + type: string + artifact_id: + description: "Immutable controller artifact ID" + required: false + type: string + artifact_digest: + description: "Controller artifact digest (sha256:...)" + required: false + type: string + source_sha: + description: "Exact lowercase 40-hex nap-dev tip the candidate was generated from" + required: false + type: string + manifest_sha256: + description: "SHA-256 of release/manifest.json inside the candidate" + required: false + type: string + rollback_tag: + description: "Protected tag process-replay/v1/- for rollback" + required: false + type: string + +permissions: + contents: read + actions: write + +concurrency: + group: process-replay-refs + cancel-in-progress: false + +env: + SOURCE_REPO: NotAutopilot/openpilot + CONTROLLER_REPO: NotAutopilot/nap-ci-controller + ARTIFACTS_REPO: NotAutopilot/ci-artifacts + ARTIFACTS_BRANCH: process-replay + TRUSTED_INVENTORY_PATH: .github/ci/process_replay_staged_inventory.json + RELEASE_VALIDATOR: .github/ci/validate_process_replay_release.py + +jobs: + # workflow_dispatch is discoverable on the default branch (nap-release). + # Retarget to the protected nap-dev workflow/ref before any publication work. + retarget-from-default: + name: Retarget nap-release dispatch to nap-dev + if: github.repository == 'NotAutopilot/openpilot' && github.ref == 'refs/heads/nap-release' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Invoke protected nap-dev workflow + env: + GH_TOKEN: ${{ github.token }} + OPERATION: ${{ inputs.operation }} + IN_CONTROLLER_RUN_ID: ${{ inputs.controller_run_id }} + IN_ARTIFACT_ID: ${{ inputs.artifact_id }} + IN_ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + IN_SOURCE_SHA: ${{ inputs.source_sha }} + IN_MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + IN_ROLLBACK_TAG: ${{ inputs.rollback_tag }} + run: | + set -euo pipefail + case "${OPERATION}" in + publish|rollback) ;; + *) + echo "Invalid operation" >&2 + exit 1 + ;; + esac + ARGS=(--ref nap-dev -f "operation=${OPERATION}") + if [[ -n "${IN_CONTROLLER_RUN_ID}" ]]; then + [[ "${IN_CONTROLLER_RUN_ID}" =~ ^[0-9]+$ ]] || { echo "controller_run_id must be numeric" >&2; exit 1; } + ARGS+=(-f "controller_run_id=${IN_CONTROLLER_RUN_ID}") + fi + if [[ -n "${IN_ARTIFACT_ID}" ]]; then + [[ "${IN_ARTIFACT_ID}" =~ ^[0-9]+$ ]] || { echo "artifact_id must be numeric" >&2; exit 1; } + ARGS+=(-f "artifact_id=${IN_ARTIFACT_ID}") + fi + if [[ -n "${IN_ARTIFACT_DIGEST}" ]]; then + [[ "${IN_ARTIFACT_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "artifact_digest invalid" >&2; exit 1; } + ARGS+=(-f "artifact_digest=${IN_ARTIFACT_DIGEST}") + fi + if [[ -n "${IN_SOURCE_SHA}" ]]; then + [[ "${IN_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] || { echo "source_sha invalid" >&2; exit 1; } + ARGS+=(-f "source_sha=${IN_SOURCE_SHA}") + fi + if [[ -n "${IN_MANIFEST_SHA256}" ]]; then + [[ "${IN_MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]] || { echo "manifest_sha256 invalid" >&2; exit 1; } + ARGS+=(-f "manifest_sha256=${IN_MANIFEST_SHA256}") + fi + if [[ -n "${IN_ROLLBACK_TAG}" ]]; then + [[ "${IN_ROLLBACK_TAG}" =~ ^process-replay/v1/[0-9a-f]{40}-[0-9a-f]{12}$ ]] || { echo "rollback_tag invalid" >&2; exit 1; } + ARGS+=(-f "rollback_tag=${IN_ROLLBACK_TAG}") + fi + gh workflow run process_replay_refs.yaml "${ARGS[@]}" + echo "Retargeted publication workflow to refs/heads/nap-dev" + + gate-nap-dev: + name: Enforce protected nap-dev contract + if: github.repository == 'NotAutopilot/openpilot' && github.ref == 'refs/heads/nap-dev' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + operation: ${{ steps.validate.outputs.operation }} + source_sha: ${{ steps.validate.outputs.source_sha }} + controller_run_id: ${{ steps.validate.outputs.controller_run_id }} + artifact_id: ${{ steps.validate.outputs.artifact_id }} + artifact_digest: ${{ steps.validate.outputs.artifact_digest }} + manifest_sha256: ${{ steps.validate.outputs.manifest_sha256 }} + rollback_tag: ${{ steps.validate.outputs.rollback_tag }} + steps: + - name: Validate ref, sha, workflow_ref, and inputs + id: validate + env: + GH_TOKEN: ${{ github.token }} + OPERATION: ${{ inputs.operation }} + IN_SOURCE_SHA: ${{ inputs.source_sha }} + IN_CONTROLLER_RUN_ID: ${{ inputs.controller_run_id }} + IN_ARTIFACT_ID: ${{ inputs.artifact_id }} + IN_ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + IN_MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + IN_ROLLBACK_TAG: ${{ inputs.rollback_tag }} + run: | + set -euo pipefail + + if [[ "${GITHUB_REF}" != "refs/heads/nap-dev" ]]; then + echo "Publication requires github.ref=refs/heads/nap-dev" >&2 + exit 1 + fi + if [[ ! "${GITHUB_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "github.sha must be lowercase 40-hex" >&2 + exit 1 + fi + EXPECTED_WORKFLOW_REF="${SOURCE_REPO}/.github/workflows/process_replay_refs.yaml@refs/heads/nap-dev" + if [[ "${GITHUB_WORKFLOW_REF}" != "${EXPECTED_WORKFLOW_REF}" ]]; then + echo "github.workflow_ref must be ${EXPECTED_WORKFLOW_REF}, got ${GITHUB_WORKFLOW_REF}" >&2 + exit 1 + fi + + TIP="$(gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" + if [[ "${TIP}" != "${GITHUB_SHA}" ]]; then + echo "Refusing: workflow SHA ${GITHUB_SHA} is not current nap-dev tip ${TIP}" >&2 + exit 1 + fi + + case "${OPERATION}" in + publish) + for value_name in IN_SOURCE_SHA IN_CONTROLLER_RUN_ID IN_ARTIFACT_ID IN_ARTIFACT_DIGEST IN_MANIFEST_SHA256; do + if [[ -z "${!value_name}" ]]; then + echo "publish requires ${value_name#IN_}" >&2 + exit 1 + fi + done + if [[ ! "${IN_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "source_sha must be exact lowercase 40-hex" >&2 + exit 1 + fi + if [[ "${IN_SOURCE_SHA}" != "${TIP}" ]]; then + echo "source_sha must equal current nap-dev tip; regenerate candidate" >&2 + exit 1 + fi + if [[ ! "${IN_CONTROLLER_RUN_ID}" =~ ^[0-9]+$ ]]; then + echo "controller_run_id must be numeric" >&2 + exit 1 + fi + if [[ ! "${IN_ARTIFACT_ID}" =~ ^[0-9]+$ ]]; then + echo "artifact_id must be numeric" >&2 + exit 1 + fi + if [[ ! "${IN_ARTIFACT_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "artifact_digest must be sha256:<64 lowercase hex>" >&2 + exit 1 + fi + if [[ ! "${IN_MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then + echo "manifest_sha256 must be 64 lowercase hex" >&2 + exit 1 + fi + ;; + rollback) + if [[ -z "${IN_ROLLBACK_TAG}" ]]; then + echo "rollback requires rollback_tag" >&2 + exit 1 + fi + if [[ ! "${IN_ROLLBACK_TAG}" =~ ^process-replay/v1/[0-9a-f]{40}-[0-9a-f]{12}$ ]]; then + echo "rollback_tag must match process-replay/v1/<40-hex>-<12-hex>" >&2 + exit 1 + fi + IN_SOURCE_SHA="$(sed -E 's|^process-replay/v1/([0-9a-f]{40})-[0-9a-f]{12}$|\1|' <<<"${IN_ROLLBACK_TAG}")" + ;; + *) + echo "Unknown operation ${OPERATION}" >&2 + exit 1 + ;; + esac + + { + echo "operation=${OPERATION}" + echo "source_sha=${IN_SOURCE_SHA}" + echo "controller_run_id=${IN_CONTROLLER_RUN_ID}" + echo "artifact_id=${IN_ARTIFACT_ID}" + echo "artifact_digest=${IN_ARTIFACT_DIGEST}" + echo "manifest_sha256=${IN_MANIFEST_SHA256}" + echo "rollback_tag=${IN_ROLLBACK_TAG}" + } >> "${GITHUB_OUTPUT}" + + verify: + name: Verify candidate without execution + needs: gate-nap-dev + if: needs.gate-nap-dev.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + # No environment: publication token must not be available here. + outputs: + release_source_sha: ${{ steps.validate.outputs.release_source_sha }} + release_manifest_sha256: ${{ steps.validate.outputs.release_manifest_sha256 }} + release_tag: ${{ steps.validate.outputs.release_tag }} + steps: + - name: Checkout trusted nap-dev inventory + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Materialize candidate tree + id: fetch + env: + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} + CONTROLLER_RUN_ID: ${{ needs.gate-nap-dev.outputs.controller_run_id }} + ARTIFACT_ID: ${{ needs.gate-nap-dev.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.gate-nap-dev.outputs.artifact_digest }} + SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} + MANIFEST_SHA256: ${{ needs.gate-nap-dev.outputs.manifest_sha256 }} + ROLLBACK_TAG: ${{ needs.gate-nap-dev.outputs.rollback_tag }} + CONTROLLER_READ_TOKEN: ${{ secrets.NAP_CI_CONTROLLER_ARTIFACT_READ_TOKEN }} + GITHUB_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p candidate + if [[ "${OPERATION}" == "publish" ]]; then + if [[ -z "${CONTROLLER_READ_TOKEN}" ]]; then + echo "Missing NAP_CI_CONTROLLER_ARTIFACT_READ_TOKEN" >&2 + exit 1 + fi + export GH_TOKEN="${CONTROLLER_READ_TOKEN}" + + EXPECTED_ARTIFACT_NAME="nap-replay-${CONTROLLER_RUN_ID}" + RUN_JSON="$(gh api "repos/${CONTROLLER_REPO}/actions/runs/${CONTROLLER_RUN_ID}")" + RUN_PATH="$(jq -r '.path // empty' <<<"${RUN_JSON}")" + RUN_NAME="$(jq -r '.name // empty' <<<"${RUN_JSON}")" + RUN_EVENT="$(jq -r '.event // empty' <<<"${RUN_JSON}")" + RUN_STATUS="$(jq -r '.status // empty' <<<"${RUN_JSON}")" + RUN_CONCLUSION="$(jq -r '.conclusion // empty' <<<"${RUN_JSON}")" + RUN_HEAD_SHA="$(jq -r '.head_sha // empty' <<<"${RUN_JSON}")" + if [[ "${RUN_PATH}" != ".github/workflows/replay.yaml" ]]; then + echo "Controller run path mismatch: ${RUN_PATH}" >&2 + exit 1 + fi + if [[ "${RUN_NAME}" != "nap-replay" ]]; then + echo "Controller run name mismatch: ${RUN_NAME}" >&2 + exit 1 + fi + if [[ "${RUN_EVENT}" != "workflow_dispatch" && "${RUN_EVENT}" != "repository_dispatch" ]]; then + echo "Controller run event must be workflow_dispatch or repository_dispatch, got ${RUN_EVENT}" >&2 + exit 1 + fi + if [[ "${RUN_STATUS}" != "completed" || "${RUN_CONCLUSION}" != "success" ]]; then + echo "Controller run ${CONTROLLER_RUN_ID} is not a successful completion (${RUN_STATUS}/${RUN_CONCLUSION})" >&2 + exit 1 + fi + if [[ ! "${RUN_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Controller run head_sha must be lowercase 40-hex" >&2 + exit 1 + fi + + META="$(gh api "repos/${CONTROLLER_REPO}/actions/artifacts/${ARTIFACT_ID}")" + META_DIGEST="$(jq -r '.digest // empty' <<<"${META}")" + META_RUN="$(jq -r '.workflow_run.id' <<<"${META}")" + META_EXPIRED="$(jq -r '.expired' <<<"${META}")" + META_NAME="$(jq -r '.name // empty' <<<"${META}")" + if [[ "${META_EXPIRED}" == "true" ]]; then + echo "Controller artifact expired" >&2 + exit 1 + fi + if [[ "${META_RUN}" != "${CONTROLLER_RUN_ID}" ]]; then + echo "Artifact run ID mismatch: ${META_RUN} != ${CONTROLLER_RUN_ID}" >&2 + exit 1 + fi + if [[ "${META_NAME}" != "${EXPECTED_ARTIFACT_NAME}" ]]; then + echo "Artifact name mismatch: ${META_NAME} != ${EXPECTED_ARTIFACT_NAME}" >&2 + exit 1 + fi + if [[ "${META_DIGEST}" != "${ARTIFACT_DIGEST}" ]]; then + echo "REST artifact digest mismatch" >&2 + echo "expected ${ARTIFACT_DIGEST}" >&2 + echo "actual ${META_DIGEST}" >&2 + exit 1 + fi + gh api "repos/${CONTROLLER_REPO}/actions/artifacts/${ARTIFACT_ID}/zip" > /tmp/candidate.zip + python3 - <<'PY' + import os + import zipfile + from pathlib import Path + + S_IFMT = 0o170000 + S_IFREG = 0o100000 + S_IFDIR = 0o040000 + + zf = zipfile.ZipFile('/tmp/candidate.zip') + seen: set[str] = set() + for info in zf.infolist(): + name = info.filename + parts = Path(name).parts + if name.startswith('/') or '..' in parts: + raise SystemExit(f'path traversal in artifact: {name}') + norm = Path(*parts).as_posix() if parts else name + key = norm.rstrip('/') if norm.endswith('/') else norm + if key in seen: + raise SystemExit(f'duplicate zip member: {norm}') + seen.add(key) + + mode = (info.external_attr >> 16) & 0xFFFF + ftype = mode & S_IFMT + is_dir = name.endswith('/') or info.is_dir() + if is_dir: + if ftype not in (0, S_IFDIR): + raise SystemExit(f'unsafe directory zip member type: {name}') + continue + if ftype not in (0, S_IFREG): + raise SystemExit(f'non-regular zip member rejected: {name}') + + target = Path('candidate') / name + target.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + with zf.open(info) as src, open(fd, 'wb') as dst: + dst.write(src.read()) + PY + + # Handoff/review proof: bind source SHA, hosted controller run, manifest SHA, + # and require green 78-pass compare results before environment approval. + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + + source_sha = os.environ['SOURCE_SHA'] + manifest_sha = os.environ['MANIFEST_SHA256'] + controller_run_id = os.environ['CONTROLLER_RUN_ID'] + sha40 = re.compile(r'^[0-9a-f]{40}$') + sha64 = re.compile(r'^[0-9a-f]{64}$') + if not sha40.fullmatch(source_sha) or not sha64.fullmatch(manifest_sha): + raise SystemExit('invalid source_sha or manifest_sha256 for proof binding') + + root = Path('candidate') + handoff = root / 'handoff-meta.json' + if not handoff.is_file() or handoff.is_symlink(): + raise SystemExit('missing handoff-meta.json proof in controller artifact') + meta = json.loads(handoff.read_text(encoding='utf-8')) + if meta.get('source_sha') != source_sha: + raise SystemExit(f"handoff source_sha mismatch: {meta.get('source_sha')} != {source_sha}") + if meta.get('manifest_sha256') != manifest_sha: + raise SystemExit('handoff manifest_sha256 mismatch against dispatch input') + if str(meta.get('controller_run_id')) != str(controller_run_id): + raise SystemExit('handoff controller_run_id mismatch against authenticated hosted run') + operation = meta.get('operation') + if operation not in {'generate-staged', 'generate-active'}: + raise SystemExit(f'publish requires generate-* controller operation, got {operation!r}') + + release_manifest = root / 'release' / 'manifest.json' + if not release_manifest.is_file(): + raise SystemExit('candidate release/manifest.json missing after extraction') + actual = hashlib.sha256(release_manifest.read_bytes()).hexdigest() + if actual != manifest_sha: + raise SystemExit('release/manifest.json hash mismatch against handoff/dispatch proof') + + results_path = root / 'review' / operation / 'compare' / 'results.json' + if not results_path.is_file() or results_path.is_symlink(): + raise SystemExit(f'missing review proof {results_path}') + results = json.loads(results_path.read_text(encoding='utf-8')) + counts = results.get('status_counts') or {} + if int(counts.get('pass', -1)) != 78: + raise SystemExit(f"review proof pass count must be 78, got {counts.get('pass')}") + for key in ('diff', 'error', 'incomplete'): + if int(counts.get(key, -1)) != 0: + raise SystemExit(f'review proof requires zero {key}, got {counts.get(key)}') + tested = results.get('tested_commit') + if tested not in (None, '', source_sha): + raise SystemExit('review proof tested_commit must equal source_sha when present') + print(f'authenticated controller run {controller_run_id} with green 78-pass review proof') + PY + else + # Rollback reads an archived tag tree with the job token only. + export GH_TOKEN="${GITHUB_READ_TOKEN}" + REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.sha')" + OBJ_TYPE="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.type')" + if [[ "${OBJ_TYPE}" == "tag" ]]; then + REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/tags/${REF_SHA}" --jq '.object.sha')" + fi + mkdir -p candidate/release + TREE_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/commits/${REF_SHA}" --jq '.tree.sha')" + gh api "repos/${ARTIFACTS_REPO}/git/trees/${TREE_SHA}?recursive=1" > /tmp/tree.json + python3 - <<'PY' + import json, base64, urllib.request, os + from pathlib import Path + token = os.environ['GH_TOKEN'] + repo = os.environ['ARTIFACTS_REPO'] + tree = json.load(open('/tmp/tree.json')) + if tree.get('truncated'): + raise SystemExit('rollback tree truncated') + for entry in tree['tree']: + path = entry['path'] + if path.startswith('/') or '..' in Path(path).parts: + raise SystemExit(f'path traversal: {path}') + if entry['type'] != 'blob': + continue + url = f'https://api.github.com/repos/{repo}/git/blobs/{entry["sha"]}' + req = urllib.request.Request(url, headers={ + 'Authorization': f'Bearer {token}', + 'Accept': 'application/vnd.github+json', + }) + with urllib.request.urlopen(req) as resp: + blob = json.load(resp) + data = base64.b64decode(blob['content']) + target = Path('candidate/release') / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + PY + fi + test -d candidate/release + + - name: Fail-closed release-tree validation + id: validate + env: + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} + SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} + MANIFEST_SHA256: ${{ needs.gate-nap-dev.outputs.manifest_sha256 }} + ROLLBACK_TAG: ${{ needs.gate-nap-dev.outputs.rollback_tag }} + GITHUB_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + test -f "${TRUSTED_INVENTORY_PATH}" + test -f "${RELEASE_VALIDATOR}" + HEAD_SHA="$(git rev-parse HEAD)" + if [[ "${SOURCE_SHA}" == "${HEAD_SHA}" ]]; then + EXPECTED_OPENDBC_SHA="$(git rev-parse HEAD:opendbc_repo)" + else + # Rollback may target an older source tip than the publisher checkout. + export GH_TOKEN="${GITHUB_READ_TOKEN}" + EXPECTED_OPENDBC_SHA="$(gh api "repos/${SOURCE_REPO}/git/trees/${SOURCE_SHA}" \ + --jq '.tree[] | select(.path=="opendbc_repo" and .mode=="160000") | .sha')" + fi + if [[ ! "${EXPECTED_OPENDBC_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Failed to resolve trusted opendbc_repo gitlink for ${SOURCE_SHA}" >&2 + exit 1 + fi + python3 "${RELEASE_VALIDATOR}" \ + --release-dir candidate/release \ + --trusted-inventory "${TRUSTED_INVENTORY_PATH}" \ + --source-sha "${SOURCE_SHA}" \ + --operation "${OPERATION}" \ + --expected-manifest-sha256 "${MANIFEST_SHA256}" \ + --expected-opendbc-sha "${EXPECTED_OPENDBC_SHA}" \ + --rollback-tag "${ROLLBACK_TAG}" + + - name: Upload verified same-run artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: process-replay-release-verified-${{ needs.gate-nap-dev.outputs.source_sha }} + path: candidate/release + if-no-files-found: error + overwrite: false + retention-days: 14 + + publish: + name: Approve and publish to ci-artifacts + needs: [gate-nap-dev, verify] + if: needs.verify.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + environment: process-replay-publish + steps: + - name: Checkout trusted nap-dev inventory + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download verified same-run artifact + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: process-replay-release-verified-${{ needs.gate-nap-dev.outputs.source_sha }} + path: release + + - name: Revalidate release tree before push + env: + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} + SOURCE_SHA: ${{ needs.verify.outputs.release_source_sha }} + MANIFEST_SHA256: ${{ needs.verify.outputs.release_manifest_sha256 }} + ROLLBACK_TAG: ${{ needs.verify.outputs.release_tag }} + GITHUB_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + test -f "${TRUSTED_INVENTORY_PATH}" + test -f "${RELEASE_VALIDATOR}" + HEAD_SHA="$(git rev-parse HEAD)" + if [[ "${SOURCE_SHA}" == "${HEAD_SHA}" ]]; then + EXPECTED_OPENDBC_SHA="$(git rev-parse HEAD:opendbc_repo)" + else + export GH_TOKEN="${GITHUB_READ_TOKEN}" + EXPECTED_OPENDBC_SHA="$(gh api "repos/${SOURCE_REPO}/git/trees/${SOURCE_SHA}" \ + --jq '.tree[] | select(.path=="opendbc_repo" and .mode=="160000") | .sha')" + fi + if [[ ! "${EXPECTED_OPENDBC_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Failed to resolve trusted opendbc_repo gitlink for ${SOURCE_SHA}" >&2 + exit 1 + fi + # Same strict validator as the no-environment verifier. + python3 "${RELEASE_VALIDATOR}" \ + --release-dir release \ + --trusted-inventory "${TRUSTED_INVENTORY_PATH}" \ + --source-sha "${SOURCE_SHA}" \ + --operation "${OPERATION}" \ + --expected-manifest-sha256 "${MANIFEST_SHA256}" \ + --expected-opendbc-sha "${EXPECTED_OPENDBC_SHA}" \ + --rollback-tag "${ROLLBACK_TAG}" + + - name: Capture current process-replay head + id: head + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if gh api "repos/${ARTIFACTS_REPO}/git/ref/heads/${ARTIFACTS_BRANCH}" >/tmp/head.json 2>/dev/null; then + CAPTURED="$(jq -r '.object.sha' /tmp/head.json)" + echo "branch_exists=true" >> "${GITHUB_OUTPUT}" + else + CAPTURED="" + echo "branch_exists=false" >> "${GITHUB_OUTPUT}" + fi + if [[ -n "${CAPTURED}" && ! "${CAPTURED}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid captured head ${CAPTURED}" >&2 + exit 1 + fi + echo "captured_head=${CAPTURED}" >> "${GITHUB_OUTPUT}" + echo "Captured ${ARTIFACTS_BRANCH} head: ${CAPTURED:-}" + + - name: Push with force-with-lease, tag, and remote rehash + env: + # CI_ARTIFACTS_TOKEN is intentionally only available in this final push step. + CI_ARTIFACTS_TOKEN: ${{ secrets.CI_ARTIFACTS_TOKEN }} + JOB_TOKEN: ${{ github.token }} + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} + SOURCE_SHA: ${{ needs.verify.outputs.release_source_sha }} + MANIFEST_SHA256: ${{ needs.verify.outputs.release_manifest_sha256 }} + RELEASE_TAG: ${{ needs.verify.outputs.release_tag }} + CAPTURED_HEAD: ${{ steps.head.outputs.captured_head }} + BRANCH_EXISTS: ${{ steps.head.outputs.branch_exists }} + run: | + set -euo pipefail + if [[ -z "${CI_ARTIFACTS_TOKEN}" ]]; then + echo "Missing CI_ARTIFACTS_TOKEN in process-replay-publish environment" >&2 + exit 1 + fi + + TIP="$(GH_TOKEN="${JOB_TOKEN}" gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" + if [[ "${OPERATION}" == "publish" && "${TIP}" != "${SOURCE_SHA}" ]]; then + echo "nap-dev moved to ${TIP}; refusing stale candidate" >&2 + exit 1 + fi + + AUTH_URL="https://x-access-token:${CI_ARTIFACTS_TOKEN}@github.com/${ARTIFACTS_REPO}.git" + rm -rf /tmp/ci-artifacts + if [[ "${BRANCH_EXISTS}" == "true" ]]; then + git clone --depth 1 --branch "${ARTIFACTS_BRANCH}" "${AUTH_URL}" /tmp/ci-artifacts + else + git clone --depth 1 "${AUTH_URL}" /tmp/ci-artifacts + fi + + cd /tmp/ci-artifacts + git config user.name "process-replay-publisher" + git config user.email "process-replay-publisher@users.noreply.github.com" + + if [[ "${BRANCH_EXISTS}" == "true" ]]; then + git checkout -B "${ARTIFACTS_BRANCH}" "origin/${ARTIFACTS_BRANCH}" + CURRENT="$(git rev-parse HEAD)" + if [[ "${CURRENT}" != "${CAPTURED_HEAD}" ]]; then + echo "Captured head drifted before push: ${CURRENT} != ${CAPTURED_HEAD}" >&2 + exit 1 + fi + if ! git ls-remote --tags origin "refs/tags/process-replay/archive/${CAPTURED_HEAD}" | grep -q .; then + git tag "process-replay/archive/${CAPTURED_HEAD}" "${CAPTURED_HEAD}" + git push origin "refs/tags/process-replay/archive/${CAPTURED_HEAD}" + fi + else + git checkout --orphan "${ARTIFACTS_BRANCH}" + git rm -rf . >/dev/null 2>&1 || true + fi + + find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} + + cp -a "${GITHUB_WORKSPACE}/release/." . + + if [[ "${BRANCH_EXISTS}" == "true" ]] && git diff --quiet HEAD -- .; then + echo "Byte-identical release tree; no-op success" + NEW_COMMIT="${CAPTURED_HEAD}" + else + git add -A + git commit -m "process-replay refs for ${SOURCE_REPO}@${SOURCE_SHA}" + NEW_COMMIT="$(git rev-parse HEAD)" + + if git ls-remote --tags origin "refs/tags/${RELEASE_TAG}" | grep -q .; then + git fetch origin "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}" + TAG_COMMIT="$(git rev-list -n 1 "refs/tags/${RELEASE_TAG}")" + TREE_MATCH="$(git rev-parse "${TAG_COMMIT}^{tree}")" + NEW_TREE="$(git rev-parse "${NEW_COMMIT}^{tree}")" + if [[ "${TREE_MATCH}" != "${NEW_TREE}" ]]; then + echo "Release tag ${RELEASE_TAG} already exists with a different tree" >&2 + exit 1 + fi + else + git tag -a "${RELEASE_TAG}" -m "process-replay release ${SOURCE_SHA}" + git push origin "refs/tags/${RELEASE_TAG}" + fi + + if [[ "${BRANCH_EXISTS}" == "true" ]]; then + git push --force-with-lease="refs/heads/${ARTIFACTS_BRANCH}:${CAPTURED_HEAD}" origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" + else + git push -u origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" + fi + echo "Published ${ARTIFACTS_BRANCH} at ${NEW_COMMIT} tag ${RELEASE_TAG}" + fi + + # Remote rehash audit by pinned commit SHA (same final step; same token scope). + export GH_TOKEN="${CI_ARTIFACTS_TOKEN}" + export NEW_COMMIT + TREE="$(gh api "repos/${ARTIFACTS_REPO}/git/commits/${NEW_COMMIT}" --jq '.tree.sha')" + gh api "repos/${ARTIFACTS_REPO}/git/trees/${TREE}?recursive=1" > /tmp/remote-tree.json + python3 - <<'PY' + import hashlib, json, os, urllib.request, base64 + token = os.environ['GH_TOKEN'] + repo = os.environ['ARTIFACTS_REPO'] + source_sha = os.environ['SOURCE_SHA'] + expected = 78 + tree = json.load(open('/tmp/remote-tree.json')) + if tree.get('truncated'): + raise SystemExit('remote tree truncated') + blobs = {e['path']: e for e in tree['tree'] if e['type'] == 'blob'} + if 'ref_commit' not in blobs or 'manifest.json' not in blobs: + raise SystemExit('remote tree missing metadata') + zsts = [p for p in blobs if p.endswith('.zst')] + if len(zsts) != expected: + raise SystemExit(f'remote zst count {len(zsts)} != {expected}') + extras = [p for p in blobs if p not in {'ref_commit', 'manifest.json'} and not p.endswith('.zst')] + if extras: + raise SystemExit(f'remote extras: {extras}') + + def fetch(sha): + req = urllib.request.Request( + f'https://api.github.com/repos/{repo}/git/blobs/{sha}', + headers={'Authorization': f'Bearer {token}', 'Accept': 'application/vnd.github+json'}, + ) + with urllib.request.urlopen(req) as resp: + return base64.b64decode(json.load(resp)['content']) + + if fetch(blobs['ref_commit']['sha']).decode().strip() != source_sha: + raise SystemExit('remote ref_commit mismatch') + manifest_bytes = fetch(blobs['manifest.json']['sha']) + if hashlib.sha256(manifest_bytes).hexdigest() != os.environ['MANIFEST_SHA256']: + raise SystemExit('remote manifest hash mismatch') + manifest = json.loads(manifest_bytes) + for _, meta in manifest['refs'].items(): + data = fetch(blobs[meta['filename']]['sha']) + if hashlib.sha256(data).hexdigest() != meta['sha256'] or len(data) != meta['size']: + raise SystemExit(f'remote rehash failed for {meta["filename"]}') + if not meta['filename'].endswith(f'__{source_sha}.zst'): + raise SystemExit('remote filename source_sha mismatch') + print(f'remote rehash OK for {expected} refs at {os.environ["NEW_COMMIT"]}') + PY diff --git a/.github/workflows/process_replay_trusted.yaml b/.github/workflows/process_replay_trusted.yaml new file mode 100644 index 00000000000000..019f3fbfaea173 --- /dev/null +++ b/.github/workflows/process_replay_trusted.yaml @@ -0,0 +1,247 @@ +# Public post-merge dispatch only. Never schedules self-hosted runners. +# After a successful hosted `tests` run for an exact refs/heads/nap-dev push tip, +# require Actions API proof of exact workflow name/path/event/SHA and successful +# conclusions for process replay (card|controlsd|lagd|other), then dispatch the +# private controller with client_payload: +# source_sha / source_repo / operation=test / jobs=6 / public_workflow_run_id / nonce. +# Zero-runner proof for NotAutopilot/openpilot is a documented rollout prerequisite +# (install-runner / host attestation). Ordinary GITHUB_TOKEN cannot authorize the +# repository runners administration API, so this workflow does not call it. +name: process replay trusted dispatch + +on: + workflow_run: + workflows: + - tests + types: + - completed + +permissions: + contents: read + actions: read + +concurrency: + group: process-replay-trusted-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: true + +jobs: + dispatch-private-controller: + name: Dispatch private controller + if: > + github.repository == 'NotAutopilot/openpilot' && + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'nap-dev' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + # Environment-scoped fine-grained credential; never CI_ARTIFACTS_TOKEN. + # Restrict this environment to protected nap-dev in repo settings. + environment: process-replay-dispatch + env: + SOURCE_REPO: NotAutopilot/openpilot + CONTROLLER_REPO: NotAutopilot/nap-ci-controller + HOSTED_WORKFLOW_NAME: tests + EXPECTED_REF: refs/heads/nap-dev + DISPATCH_OPERATION: test + DISPATCH_JOBS: "6" + steps: + - name: Validate exact tip and successful hosted run + id: gate + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_SHA: ${{ github.event.workflow_run.head_sha }} + CANDIDATE_RUN_ID: ${{ github.event.workflow_run.id }} + CANDIDATE_EVENT: ${{ github.event.workflow_run.event }} + CANDIDATE_BRANCH: ${{ github.event.workflow_run.head_branch }} + CANDIDATE_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + CANDIDATE_WORKFLOW: ${{ github.event.workflow_run.name }} + CANDIDATE_PATH: ${{ github.event.workflow_run.path }} + run: | + set -euo pipefail + + echo "candidate_sha=${CANDIDATE_SHA}" + echo "candidate_run_id=${CANDIDATE_RUN_ID}" + echo "candidate_event=${CANDIDATE_EVENT}" + echo "candidate_branch=${CANDIDATE_BRANCH}" + echo "candidate_conclusion=${CANDIDATE_CONCLUSION}" + echo "candidate_workflow=${CANDIDATE_WORKFLOW}" + echo "candidate_path=${CANDIDATE_PATH}" + + if [[ "${CANDIDATE_CONCLUSION}" != "success" ]]; then + echo "Refusing dispatch: hosted workflow conclusion is not success" >&2 + exit 1 + fi + if [[ "${CANDIDATE_EVENT}" != "push" ]]; then + echo "Refusing dispatch: underlying event must be push" >&2 + exit 1 + fi + if [[ "${CANDIDATE_BRANCH}" != "nap-dev" ]]; then + echo "Refusing dispatch: head branch must be nap-dev" >&2 + exit 1 + fi + if [[ "${CANDIDATE_WORKFLOW}" != "${HOSTED_WORKFLOW_NAME}" ]]; then + echo "Refusing dispatch: unexpected workflow name ${CANDIDATE_WORKFLOW}" >&2 + exit 1 + fi + if [[ -n "${CANDIDATE_PATH}" && "${CANDIDATE_PATH}" != ".github/workflows/tests.yaml" ]]; then + echo "Refusing dispatch: unexpected workflow path ${CANDIDATE_PATH}" >&2 + exit 1 + fi + if [[ ! "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Refusing dispatch: source_sha must be exact lowercase 40-hex" >&2 + exit 1 + fi + if [[ ! "${CANDIDATE_RUN_ID}" =~ ^[0-9]+$ ]]; then + echo "Refusing dispatch: public workflow run ID must be numeric" >&2 + exit 1 + fi + + TIP="$(gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" + if [[ ! "${TIP}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Refusing dispatch: current nap-dev tip is not lowercase 40-hex: ${TIP}" >&2 + exit 1 + fi + if [[ "${TIP}" != "${CANDIDATE_SHA}" ]]; then + echo "Refusing dispatch: candidate ${CANDIDATE_SHA} is not the current ${EXPECTED_REF} tip ${TIP}" >&2 + exit 1 + fi + + # Re-check the exact hosted run via API (fail closed). + RUN_JSON="$(gh api "repos/${SOURCE_REPO}/actions/runs/${CANDIDATE_RUN_ID}")" + RUN_SHA="$(jq -r '.head_sha' <<<"${RUN_JSON}")" + RUN_BRANCH="$(jq -r '.head_branch' <<<"${RUN_JSON}")" + RUN_EVENT="$(jq -r '.event' <<<"${RUN_JSON}")" + RUN_CONCLUSION="$(jq -r '.conclusion' <<<"${RUN_JSON}")" + RUN_STATUS="$(jq -r '.status' <<<"${RUN_JSON}")" + RUN_PATH="$(jq -r '.path' <<<"${RUN_JSON}")" + RUN_NAME="$(jq -r '.name' <<<"${RUN_JSON}")" + + if [[ "${RUN_STATUS}" != "completed" || "${RUN_CONCLUSION}" != "success" ]]; then + echo "Refusing dispatch: hosted run ${CANDIDATE_RUN_ID} is not a successful completion" >&2 + exit 1 + fi + if [[ "${RUN_SHA}" != "${CANDIDATE_SHA}" || "${RUN_SHA}" != "${TIP}" ]]; then + echo "Refusing dispatch: hosted run SHA mismatch" >&2 + exit 1 + fi + if [[ "${RUN_BRANCH}" != "nap-dev" || "${RUN_EVENT}" != "push" ]]; then + echo "Refusing dispatch: hosted run is not an exact nap-dev push" >&2 + exit 1 + fi + if [[ "${RUN_NAME}" != "${HOSTED_WORKFLOW_NAME}" ]]; then + echo "Refusing dispatch: hosted run workflow name mismatch" >&2 + exit 1 + fi + if [[ "${RUN_PATH}" != ".github/workflows/tests.yaml" ]]; then + echo "Refusing dispatch: hosted run path mismatch: ${RUN_PATH}" >&2 + exit 1 + fi + + # Require successful conclusions for the four exact hosted process-replay jobs. + mapfile -t JOB_LINES < <( + gh api --paginate \ + "repos/${SOURCE_REPO}/actions/runs/${CANDIDATE_RUN_ID}/jobs" \ + --jq '.jobs[] | [.name, .status, .conclusion] | @tsv' + ) + declare -A JOB_STATUS=() + declare -A JOB_CONCLUSION=() + for line in "${JOB_LINES[@]}"; do + IFS=$' ' read -r job_name job_status job_conclusion <<<"${line}" + JOB_STATUS["${job_name}"]="${job_status}" + JOB_CONCLUSION["${job_name}"]="${job_conclusion}" + done + EXPECTED_JOBS=( + "process replay (card)" + "process replay (controlsd)" + "process replay (lagd)" + "process replay (other)" + ) + for job_name in "${EXPECTED_JOBS[@]}"; do + if [[ -z "${JOB_STATUS[${job_name}]+x}" ]]; then + echo "Refusing dispatch: missing required hosted job ${job_name}" >&2 + exit 1 + fi + if [[ "${JOB_STATUS[${job_name}]}" != "completed" || "${JOB_CONCLUSION[${job_name}]}" != "success" ]]; then + echo "Refusing dispatch: job ${job_name} status=${JOB_STATUS[${job_name}]} conclusion=${JOB_CONCLUSION[${job_name}]}" >&2 + exit 1 + fi + done + + # Zero public-repo self-hosted runners remain a rollout prerequisite attested + # during private runner install; ordinary GITHUB_TOKEN cannot query that API. + + NONCE="$(openssl rand -hex 16)" + if [[ ! "${NONCE}" =~ ^[0-9a-f]{32}$ ]]; then + echo "Failed to generate nonce" >&2 + exit 1 + fi + + { + echo "source_sha=${CANDIDATE_SHA}" + echo "public_workflow_run_id=${CANDIDATE_RUN_ID}" + echo "nonce=${NONCE}" + } >> "${GITHUB_OUTPUT}" + + - name: Dispatch private controller + env: + # Fine-grained token scoped to the private controller only. + GH_TOKEN: ${{ secrets.NAP_CI_CONTROLLER_DISPATCH_TOKEN }} + SOURCE_SHA: ${{ steps.gate.outputs.source_sha }} + PUBLIC_WORKFLOW_RUN_ID: ${{ steps.gate.outputs.public_workflow_run_id }} + NONCE: ${{ steps.gate.outputs.nonce }} + run: | + set -euo pipefail + + if [[ -z "${GH_TOKEN}" ]]; then + echo "Missing NAP_CI_CONTROLLER_DISPATCH_TOKEN in process-replay-dispatch environment" >&2 + exit 1 + fi + if [[ ! "${SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid source_sha" >&2 + exit 1 + fi + if [[ ! "${PUBLIC_WORKFLOW_RUN_ID}" =~ ^[0-9]+$ ]]; then + echo "Invalid public_workflow_run_id" >&2 + exit 1 + fi + if [[ ! "${NONCE}" =~ ^[0-9a-f]{32}$ ]]; then + echo "Invalid nonce" >&2 + exit 1 + fi + if [[ "${SOURCE_REPO}" != "NotAutopilot/openpilot" || "${DISPATCH_OPERATION}" != "test" || "${DISPATCH_JOBS}" != "6" ]]; then + echo "Invalid fixed dispatch contract" >&2 + exit 1 + fi + + # Tip must still match at dispatch time. + TIP="$(gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" + if [[ "${TIP}" != "${SOURCE_SHA}" ]]; then + echo "Refusing dispatch: nap-dev moved to ${TIP} before controller dispatch" >&2 + exit 1 + fi + + PAYLOAD="$(jq -cn \ + --arg source_sha "${SOURCE_SHA}" \ + --arg source_repo "${SOURCE_REPO}" \ + --arg operation "${DISPATCH_OPERATION}" \ + --arg jobs "${DISPATCH_JOBS}" \ + --arg public_workflow_run_id "${PUBLIC_WORKFLOW_RUN_ID}" \ + --arg nonce "${NONCE}" \ + '{ + event_type: "nap-replay", + client_payload: { + source_sha: $source_sha, + source_repo: $source_repo, + operation: $operation, + jobs: $jobs, + public_workflow_run_id: $public_workflow_run_id, + nonce: $nonce + } + }')" + + echo "Dispatching controller for ${SOURCE_REPO}@${SOURCE_SHA} operation=${DISPATCH_OPERATION} jobs=${DISPATCH_JOBS}" + gh api --method POST "repos/${CONTROLLER_REPO}/dispatches" \ + -H "Accept: application/vnd.github+json" \ + --input - <<<"${PAYLOAD}" + + echo "Dispatched private controller; public workflow does not schedule self-hosted runners." diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d69d9829581a73..5d4d00b28b0907 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -22,6 +22,7 @@ env: CI: 1 PYTHONPATH: ${{ github.workspace }} PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical + OPENPILOT_LFS_URL: https://github.com/${{ github.repository }}.git/info/lfs jobs: tesla_preap_longitudinal_regression: @@ -34,7 +35,12 @@ jobs: submodules: true - name: Check workflow contract run: python3 .github/ci/test_tesla_preap_longitudinal_workflow.py - - run: ./tools/op.sh setup + - name: Set up dependencies + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e + with: + timeout_minutes: 10 + max_attempts: 3 + command: ./tools/op.sh setup - name: Build focused test dependencies run: scons -j$(nproc) - name: Run focused longitudinal tests @@ -69,7 +75,7 @@ jobs: with: timeout_minutes: 2 max_attempts: 3 - command: git lfs pull + command: git config lfs.url "$OPENPILOT_LFS_URL" && git lfs pull - name: Build devel timeout-minutes: 1 run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh @@ -149,62 +155,117 @@ jobs: MAX_EXAMPLES=1 $PYTEST -m 'not slow' process_replay: - name: process replay - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16"]') - || fromJSON('["ubuntu-24.04"]') }} + name: process replay (${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - shard: card + filter_args: "--whitelist-procs card" + expected_tasks: 16 + - shard: controlsd + filter_args: "--whitelist-procs controlsd" + expected_tasks: 16 + - shard: lagd + filter_args: "--whitelist-procs lagd" + expected_tasks: 16 + - shard: other + filter_args: "--blacklist-procs card controlsd lagd" + expected_tasks: 18 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: submodules: true - - run: ./tools/op.sh setup + persist-credentials: false + - name: Set up dependencies + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e + with: + timeout_minutes: 10 + max_attempts: 3 + command: ./tools/op.sh setup - name: Install video encoding deps run: sudo apt-get install -y libva-dev libva-drm2 libdrm-dev - name: Build openpilot run: scons -j$(nproc) + - name: Verify shard inventory contracts + env: + SHARD: ${{ matrix.shard }} + FILTER_ARGS: ${{ matrix.filter_args }} + EXPECTED_TASKS: ${{ matrix.expected_tasks }} + run: | + set -euo pipefail + REPLAY=selfdrive/test/process_replay/test_processes.py + + python3 - <<'PY' + from openpilot.selfdrive.test.process_replay import test_processes as tp + + active_cases = tp.select_cases("active") + staged_cases = tp.select_cases("staged") + tp.assert_inventory_invariants(active_cases) + tp.assert_inventory_invariants(staged_cases) + + assert len(active_cases) == 16, len(active_cases) + assert len(staged_cases) == 18, len(staged_cases) + + active_tasks = tp.build_tasks(active_cases) + staged_tasks = tp.build_tasks(staged_cases) + assert len(active_tasks) == 66, len(active_tasks) + assert len(staged_tasks) == 78, len(staged_tasks) + + active_counts = {name: len(tp.shard_task_ids(active_tasks, name)) for name in ("card", "controlsd", "lagd", "other")} + assert active_counts == {"card": 16, "controlsd": 16, "lagd": 16, "other": 18}, active_counts + assert sum(active_counts.values()) == 66 + + staged_counts = {name: len(tp.shard_task_ids(staged_tasks, name)) for name in ("card", "controlsd", "lagd", "other")} + assert staged_counts == {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}, staged_counts + assert sum(staged_counts.values()) == 78 + + staged_task_ids = {task.task_id for task in staged_tasks} + staged_union = set(tp.shard_task_ids(staged_tasks, "card")) + staged_union |= set(tp.shard_task_ids(staged_tasks, "controlsd")) + staged_union |= set(tp.shard_task_ids(staged_tasks, "lagd")) + staged_union |= set(tp.shard_task_ids(staged_tasks, "other")) + assert staged_union == staged_task_ids, (len(staged_union), len(staged_task_ids)) + assert len(staged_union) == 78, len(staged_union) + print("inventory unit checks passed:", active_counts, staged_counts) + PY + + mapfile -t shard_tasks < <(${REPLAY} --inventory active --list-tasks ${FILTER_ARGS}) + echo "shard=${SHARD} list-tasks count=${#shard_tasks[@]}" + printf '%s\n' "${shard_tasks[@]}" + test "${#shard_tasks[@]}" -eq "${EXPECTED_TASKS}" + + active_total="$(${REPLAY} --inventory active --list-tasks | wc -l)" + staged_total="$(${REPLAY} --inventory staged --list-tasks | wc -l)" + echo "active_total=${active_total} staged_total=${staged_total}" + test "${active_total}" -eq 66 + test "${staged_total}" -eq 78 - name: Run replay - timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 45 }} - continue-on-error: ${{ github.ref == 'refs/heads/master' }} - run: selfdrive/test/process_replay/test_processes.py -j$(nproc) + run: | + set -euo pipefail + OUT_DIR="selfdrive/test/process_replay/out/${{ matrix.shard }}" + mkdir -p "${OUT_DIR}" + selfdrive/test/process_replay/test_processes.py \ + -j$(nproc) \ + --inventory active \ + --output-dir "${OUT_DIR}" \ + ${{ matrix.filter_args }} - name: Print diff - id: print-diff if: always() - run: cat selfdrive/test/process_replay/diff.txt - - uses: actions/upload-artifact@v6 + run: cat selfdrive/test/process_replay/out/${{ matrix.shard }}/diff.txt + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: always() - continue-on-error: true - with: - name: process_replay_diff.txt - path: selfdrive/test/process_replay/diff.txt - - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' - uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts - ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} - path: ${{ github.workspace }}/ci-artifacts - - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' - working-directory: ${{ github.workspace }}/ci-artifacts - run: | - git config user.name "GitHub Actions Bot" - git config user.email "<>" - git fetch origin process-replay || true - git checkout process-replay 2>/dev/null || git checkout --orphan process-replay - cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst . - echo "${{ github.sha }}" > ref_commit - git add . - git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit" - git push origin process-replay --force - - name: Run regen - if: false - timeout-minutes: 4 - env: - ONNXCPU: 1 - run: $PYTEST selfdrive/test/process_replay/test_regen.py + name: process_replay_${{ matrix.shard }} + path: | + selfdrive/test/process_replay/out/${{ matrix.shard }}/diff.txt + selfdrive/test/process_replay/out/${{ matrix.shard }}/results.json + selfdrive/test/process_replay/fakedata + if-no-files-found: error simulator_driving: name: simulator driving diff --git a/opendbc_repo b/opendbc_repo index cc1afdbeabffad..5c30f9491a872a 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit cc1afdbeabffad05712705f8bf8e85b964926eae +Subproject commit 5c30f9491a872a545a410365bbc0aafd78b22c4a diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 2d35ad8b307485..e043ac84e5773d 100755 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -80,7 +80,8 @@ def __init__(self, CP=None): ignore += ['driverCameraState', 'managerState'] if REPLAY: # no vipc in replay will make them ignored anyways - ignore += ['roadCameraState', 'wideRoadCameraState'] + # sanitized fixtures omit driverCameraState/managerState; ignore them in replay + ignore += ['roadCameraState', 'wideRoadCameraState', 'driverCameraState', 'managerState'] self.sm = messaging.SubMaster(['deviceState', 'pandaStates', 'peripheralState', 'modelV2', 'liveCalibration', 'carOutput', 'driverMonitoringState', 'longitudinalPlan', 'livePose', 'liveDelay', 'managerState', 'liveParameters', 'radarState', 'liveTorqueParameters', diff --git a/selfdrive/test/process_replay/README.md b/selfdrive/test/process_replay/README.md index 28f3b7cd2a5fc0..d4d47465ea56a1 100644 --- a/selfdrive/test/process_replay/README.md +++ b/selfdrive/test/process_replay/README.md @@ -1,53 +1,71 @@ # Process replay -Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment. +Process replay is a regression test that replays recorded source logs through individual openpilot processes and compares each process output against a known reference. -If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated. +## Cases and tasks -Use `test_processes.py` to run the test locally. -Log files are cached by default. Use `DISABLE_FILEREADER_CACHE='1' test_processes.py` to disable caching. +A **case** is one source input: car brand, source locator, selected processes, and optional typed custom params. +A **task** is one process on one case. Task IDs are canonical: `{case_id}:{process}`. -Currently the following processes are tested: +The active inventory is the legacy 16 sources / 66 tasks: +- `card`, `controlsd`, and `lagd` on every case +- the other nine non-model processes only on `HYUNDAI` and `TOYOTA` -* controlsd -* radard -* plannerd -* calibrationd -* dmonitoringd -* locationd -* paramsd -* ubloxd -* torqued +`--inventory staged` also lists two temporary Pre-AP descriptors (`PENDING_CASES`) for a total of 18 cases / 78 tasks. Those pending cases are listing/generation descriptors only until sanitized fixture URLs and digests are filled in; they are not executable in normal active runs. + +`source_segments` remains the regen compatibility list used by `regen_all.py`. + +## Running locally + +```bash +# full active suite +./test_processes.py -j$(nproc) + +# shard-style filters (diagnostics go to --output-dir) +./test_processes.py -j$(nproc) --whitelist-procs card --output-dir /tmp/replay/card +./test_processes.py -j$(nproc) --whitelist-procs controlsd --output-dir /tmp/replay/controlsd +./test_processes.py -j$(nproc) --whitelist-procs lagd --output-dir /tmp/replay/lagd +./test_processes.py -j$(nproc) --blacklist-procs card controlsd lagd --output-dir /tmp/replay/other + +# inventories +./test_processes.py --list-cases +./test_processes.py --list-tasks +./test_processes.py --inventory staged --list-tasks +``` + +Log downloads are cached by default. Disable with `DISABLE_FILEREADER_CACHE=1`. + +### CLI -### Usage ``` -Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS] - [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--update-refs] -Regression test to identify changes in a process's output -optional arguments: - -h, --help show this help message and exit - --whitelist-procs PROCS Whitelist given processes from the test (e.g. controlsd) - --whitelist-cars WHITELIST_CARS Whitelist given cars from the test (e.g. HONDA) - --blacklist-procs BLACKLIST_PROCS Blacklist given processes from the test (e.g. controlsd) - --blacklist-cars BLACKLIST_CARS Blacklist given cars from the test (e.g. HONDA) - --ignore-fields IGNORE_FIELDS Extra fields or msgs to ignore (e.g. driverMonitoringState.events) - --ignore-msgs IGNORE_MSGS Msgs to ignore (e.g. onroadEvents) - --update-refs Updates reference logs using current commit +Usage: test_processes.py [-h] + [--whitelist-procs PROCS] [--whitelist-cars CARS] + [--blacklist-procs PROCS] [--blacklist-cars CARS] + [--ignore-fields FIELDS] [--ignore-msgs MSGS] + [--update-refs] [--inventory {active,staged}] + [--list-cases] [--list-tasks] + [--output-dir DIR] [--reference-dir DIR] [-j JOBS] ``` -## Forks +- `--output-dir` only changes where `diff.txt` and `results.json` are written. +- `--reference-dir` selects an already-generated local candidate tree for comparison. +- `--update-refs` writes candidate refs for the selected tasks. It does not mark those candidates as accepted; acceptance is a second normal run against `--reference-dir`. -openpilot forks can use this test with their own reference logs, by default `test_proccesses.py` saves logs locally. +## References and diagnostics -To generate new logs: +Remote references are resolved from one pinned `NotAutopilot/ci-artifacts` commit on `process-replay`, not from a moving branch URL. Filenames use `{case_id}__{process}__{ref_commit}.zst`. Manifest-era trees require `manifest.json` and canonical filenames. Pre-bootstrap segment filenames are allowed only when the pinned artifacts commit is listed in `LEGACY_ARTIFACTS_COMMITS`. -`./test_processes.py` +Each selected case is downloaded once. When a case declares size/SHA-256, both are checked before replay. A digest failure becomes a task error for every task on that case; other cases continue. -Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit. +Workers return serializable `TaskResult` values. One task failure does not abort the pool. `diff.txt` and schema-versioned `results.json` are created before downloads start and rewritten after every settled task. The process exits nonzero on any diff, error, or incomplete inventory. After a normal compare run, only failed/different generated logs are kept; stale `fakedata` is cleared at start. + +## Params + +Runtime custom params stay native Python types (`bool`, `int`, `float`) and are validated against `common/params_keys.h` before scheduling. Manifest digests hash a separate canonical JSON array sorted by key with explicit `type` tags. ## API -Process replay test suite exposes programmatic APIs for simultaneously running processes or groups of processes on provided logs. +Process replay also exposes programmatic helpers for replaying processes on provided logs: ```py def replay_process_with_name(name: Union[str, Iterable[str]], lr: LogIterable, *args, **kwargs) -> List[capnp._DynamicStructReader]: @@ -58,68 +76,17 @@ def replay_process( ) -> List[capnp._DynamicStructReader]: ``` -Example usage: +Example: + ```py from openpilot.selfdrive.test.process_replay import replay_process_with_name from openpilot.tools.lib.logreader import LogReader lr = LogReader(...) - -# provide a name of the process to replay output_logs = replay_process_with_name('locationd', lr) - -# or list of names output_logs = replay_process_with_name(['ubloxd', 'locationd'], lr) ``` -Supported processes: -* controlsd -* radard -* plannerd -* calibrationd -* dmonitoringd -* locationd -* paramsd -* ubloxd -* torqued -* modeld -* dmonitoringmodeld - -Certain processes may require an initial state, which is usually supplied within `Params` and persisting from segment to segment (e.g CalibrationParams, LiveParameters). The `custom_params` is dictionary used to prepopulate `Params` with arbitrary values. The `get_custom_params_from_lr` helper is provided to fetch meaningful values from log files. - -```py -from openpilot.selfdrive.test.process_replay import get_custom_params_from_lr +Supported processes include `controlsd`, `radard`, `plannerd`, `calibrationd`, `dmonitoringd`, `locationd`, `paramsd`, `ubloxd`, `torqued`, `card`, `lagd`, `selfdrived`, `modeld`, and `dmonitoringmodeld`. -previous_segment_lr = LogReader(...) -current_segment_lr = LogReader(...) - -custom_params = get_custom_params_from_lr(previous_segment_lr, 'last') - -output_logs = replay_process_with_name('calibrationd', lr, custom_params=custom_params) -``` - -Replaying processes that use VisionIPC (e.g. modeld, dmonitoringmodeld) require additional `frs` dictionary with camera states as keys and `FrameReader` objects as values. - -```py -from openpilot.tools.lib.framereader import FrameReader - -frs = { - 'roadCameraState': FrameReader(...), - 'wideRoadCameraState': FrameReader(...), - 'driverCameraState': FrameReader(...), -} - -output_logs = replay_process_with_name(['modeld', 'dmonitoringmodeld'], lr, frs=frs) -``` - -To capture stdout/stderr of the replayed process, `captured_output_store` can be provided. - -```py -output_store = dict() -# pass dictionary by reference, it will be filled with standard outputs - even if process replay fails -output_logs = replay_process_with_name(['radard', 'plannerd'], lr, captured_output_store=output_store) - -# entries with captured output in format { 'out': '...', 'err': '...' } will be added to provided dictionary for each replayed process -print(output_store['radard']['out']) # radard stdout -print(output_store['radard']['err']) # radard stderr -``` +Use `custom_params` to seed `Params` with typed values. `get_custom_params_from_lr` can recover meaningful values from a previous segment. VisionIPC processes need an `frs` map of camera state names to `FrameReader` objects. Pass `captured_output_store` to collect stdout/stderr per process. diff --git a/selfdrive/test/process_replay/compare_logs.py b/selfdrive/test/process_replay/compare_logs.py index e2d912a83394e5..4c522c9150d98f 100755 --- a/selfdrive/test/process_replay/compare_logs.py +++ b/selfdrive/test/process_replay/compare_logs.py @@ -76,7 +76,7 @@ def _diff_capnp_values(v1, v2, path, tolerance): for i in range(n): yield from _diff_capnp_values(v1[i], v2[i], path + (str(i),), tolerance) if n2 > n: - yield 'add', dot, list(enumerate(v2[n:], n)) + yield 'add', dot, [(i, v2[i]) for i in range(n, n2)] if n1 > n: yield 'remove', dot, list(reversed([(i, v1[i]) for i in range(n, n1)])) diff --git a/selfdrive/test/process_replay/fixture_tool.py b/selfdrive/test/process_replay/fixture_tool.py new file mode 100755 index 00000000000000..52a54e8829292d --- /dev/null +++ b/selfdrive/test/process_replay/fixture_tool.py @@ -0,0 +1,1576 @@ +#!/usr/bin/env python3 +"""Build and audit small, deterministic public Pre-AP replay fixtures.""" +from __future__ import annotations + +import argparse +import base64 +import binascii +import json +import os +import re +import sys +import tempfile +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import capnp +import zstandard +from cereal import car, log as capnp_log +from openpilot.tools.lib.logreader import LogReader + +SCHEMA_VERSION = 1 +PLACEHOLDER_VIN = "00000000000000000" +VIN_LENGTH = 17 + +EXPECTED_SAFETY_MODEL = car.CarParams.SafetyModel.teslaPreap +# Fixed Pre-AP replay params enable pedal + radar emulation + nosecone => 7 / 6. +PREAP_FLAG_ENABLE_PEDAL = 1 +PREAP_FLAG_RADAR_EMULATION = 2 +PREAP_FLAG_RADAR_BEHIND_NOSECONE = 4 + +CASE_MODES = frozenset({"pedal", "no-pedal"}) +SAFE_LABEL_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +HEX_VALUE_RE = re.compile(r"^[0-9a-f]*$") +DENY_ENCODINGS = frozenset({"utf8", "hex", "base64"}) +VIN_FRAGMENT_MIN = 6 +VIN_FRAGMENT_MAX = 17 +CAN_OVERLAP = 16 +MAX_COMPRESSED_INPUT_BYTES = 64 * 1024 * 1024 +MAX_DECOMPRESSED_INPUT_BYTES = 512 * 1024 * 1024 +MAX_LOG_MESSAGES = 2_000_000 +MAX_CAN_CONCAT_BYTES = 4 * 1024 * 1024 +VALID_SHARE_THRESHOLD = 0.9 +ZSTD_CONTENTSIZE_UNKNOWN = (1 << 64) - 1 +ZSTD_DECOMPRESS_CHUNK = 1024 * 1024 +# Retained-as-is public Text constants. can[].dat is a scan sink only — never an +# inventory exemption for values that also originate from private/scrubbed leaves. +ATTESTED_PUBLIC_TEXT_DATA_LEAVES = frozenset({ + "carParams.brand", + "carParams.carFingerprint", +}) + +SIX_PROCESS_OUTPUTS: dict[str, frozenset[str]] = { + "card": frozenset({"sendcan", "carState", "carParams", "carOutput", "liveTracks"}), + "controlsd": frozenset({"carControl", "controlsState"}), + "selfdrived": frozenset({"selfdriveState", "onroadEvents"}), + "radard": frozenset({"radarState"}), + "plannerd": frozenset({"longitudinalPlan", "driverAssistance"}), + "lagd": frozenset({"liveDelay"}), +} + +POLICY_KEYS = frozenset({ + "schema_version", "allowed_services", "forbidden_services", "required_services", + "selected_gps_service", "freshness_services", "pedal", "sensitive_zero_fields", + "placeholder_vin", "cases", +}) +PEDAL_KEYS = frozenset({ + "address", "bus", "data_length", "counter_index", "counter_mask", + "checksum_index", "checksum", "minimum_frames", "minimum_counters", +}) +CASE_KEYS = frozenset({"mode", "fingerprint", "safety_param"}) +DENY_REQUIRED_KEYS = frozenset({"raw_input_sha256", "tokens"}) +DENY_OPTIONAL_KEYS = frozenset({"sanitized_output_sha256", "input_sha256"}) +# Attested release policy digest over POLICY_KEYS (canonical JSON). Weakened +# caller policies fail closed even when self-consistent. +RELEASE_POLICY_SHA256 = "8b37a7881af652246d5f7323dbd01f9e6c117440ee8d17fad77c88dfc3a1354f" +PEDAL_MIN_FRAMES = 10 +PEDAL_MIN_COUNTERS = 8 +ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" + + +def safety_model_ordinal(value: Any) -> int: + """SafetyModel ordinal. Static enums int(); DynamicEnum needs .raw (str() is "37").""" + return int(getattr(value, "raw", value)) + + +def safety_model_is_tesla_preap(value: Any) -> bool: + return safety_model_ordinal(value) == int(EXPECTED_SAFETY_MODEL) + + +def expected_safety_param(mode: str) -> int: + flags = PREAP_FLAG_RADAR_EMULATION | PREAP_FLAG_RADAR_BEHIND_NOSECONE + if mode == "pedal": + flags |= PREAP_FLAG_ENABLE_PEDAL + return int(flags) + + +@dataclass(frozen=True) +class DenyManifest: + tokens: tuple[tuple[str, bytes], ...] + raw_input_sha256: str + sanitized_output_sha256: str | None + vin: bytes + +MANDATORY_FORBIDDEN = frozenset({ + "clocks", "initData", "boot", "sentinel", + "roadCameraState", "driverCameraState", "wideRoadCameraState", + "logMessage", "errorLogMessage", "androidLog", "procLog", + "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "qRoadEncodeIdx", + "livestreamRoadEncodeIdx", "livestreamWideRoadEncodeIdx", "livestreamDriverEncodeIdx", + "roadEncodeData", "driverEncodeData", "wideRoadEncodeData", "qRoadEncodeData", + "livestreamRoadEncodeData", "livestreamWideRoadEncodeData", "livestreamDriverEncodeData", + "touch", + "qcomGnss", "ubloxRaw", "ubloxGnss", "gnssMeasurements", "gpsNMEA", + "uiDebug", "navInstruction", "navRoute", "navThumbnail", "mapRenderState", "thumbnail", + "managerState", "uploaderState", + "rawAudioData", "soundPressure", "audioFeedback", + "sendcan", +}) + +SENSITIVE_ZERO_BASELINE = frozenset({ + "gpsLocation.latitude", "gpsLocation.longitude", "gpsLocation.altitude", + "gpsLocation.bearingDeg", "gpsLocation.unixTimestampMillis", "gpsLocation.vNED[]", + "gpsLocationExternal.latitude", "gpsLocationExternal.longitude", "gpsLocationExternal.altitude", + "gpsLocationExternal.bearingDeg", "gpsLocationExternal.unixTimestampMillis", "gpsLocationExternal.vNED[]", +}) + + +class FixtureError(ValueError): + pass + + +def digest(data: bytes) -> str: + import hashlib + return hashlib.sha256(data).hexdigest() + + +def field_parts(path: str) -> list[str]: + parts: list[str] = [] + for part in path.split("."): + if not part: + continue + if part == "[]": + parts.append(part) + elif part.endswith("[]"): + parts.extend((part[:-2], "[]")) + else: + parts.append(part) + return parts + + +def path_join(prefix: str, name: str) -> str: + return name if not prefix else f"{prefix}.{name}" + + +def is_list_service(service: str) -> bool: + field = capnp_log.Event.schema.fields[service] + return field.proto.slot.type.which() == "list" + + +def event_service_names() -> set[str]: + return set(capnp_log.Event.schema.union_fields) + + +def schema_for_service(service: str) -> Any: + field = capnp_log.Event.schema.fields[service] + t = field.proto.slot.type + if t.which() == "list": + return field.schema.elementType + if t.which() == "struct": + return field.schema + raise FixtureError(f"unsupported service schema: {service}") + + +def resolve_policy_path(service: str, path: str) -> None: + """Fail closed if a policy path cannot be resolved in the current Cap'n Proto schema.""" + parts = field_parts(path) + if not parts: + raise FixtureError(f"empty policy path for {service}") + schema = schema_for_service(service) + index = 0 + # Event-level list services use a leading [] marker before element fields. + if is_list_service(service): + if parts[0] != "[]": + raise FixtureError(f"list service paths require leading []: {service}.{path}") + index = 1 + if index == len(parts): + raise FixtureError(f"list service path requires a field: {service}.{path}") + while index < len(parts): + part = parts[index] + if part == "[]": + raise FixtureError(f"invalid list marker in {service}.{path}") + if part not in schema.fieldnames: + raise FixtureError(f"policy field does not exist: {service}.{path}") + field = schema.fields[part] + index += 1 + if field.proto.which() == "group": + if index == len(parts): + raise FixtureError(f"path ends at group: {service}.{path}") + schema = field.schema + continue + if field.proto.which() != "slot": + raise FixtureError(f"unsupported field kind in {service}.{path}") + twhich = field.proto.slot.type.which() + if twhich == "list": + if index >= len(parts) or parts[index] != "[]": + raise FixtureError(f"list field requires []: {service}.{path}") + index += 1 + elem = field.proto.slot.type.list.elementType + if index == len(parts): + if elem.which() == "struct": + raise FixtureError(f"struct list requires subfield: {service}.{path}") + return + if elem.which() != "struct": + raise FixtureError(f"cannot descend into scalar list: {service}.{path}") + schema = field.schema.elementType + elif twhich == "struct": + if index == len(parts): + raise FixtureError(f"path ends at struct: {service}.{path}") + schema = field.schema + else: + if index != len(parts): + raise FixtureError(f"extra path after leaf: {service}.{path}") + return + raise FixtureError(f"path ends at non-leaf: {service}.{path}") + + +def expand_allowed_paths(service: str, paths: list[str]) -> set[str]: + """Canonical leaf path set for membership checks (with [] markers preserved).""" + resolved: set[str] = set() + for path in paths: + resolve_policy_path(service, path) + resolved.add(".".join(field_parts(path))) + return resolved + + +def copy_path(source: Any, destination: Any, path: str) -> None: + parts = field_parts(path) + + def walk(src: Any, dst: Any, remaining: list[str]) -> None: + if not remaining: + raise FixtureError("empty policy field path") + part = remaining[0] + if part == "[]": + if len(remaining) == 1: + for index, src_item in enumerate(src): + dst[index] = src_item + return + for src_item, dst_item in zip(src, dst, strict=True): + walk(src_item, dst_item, remaining[1:]) + return + + if hasattr(src, "schema") and part in src.schema.union_fields: + try: + active = src.which() + except capnp.KjException: + return + if active != part: + return + try: + dst_branch = dst.init(part) + except (AttributeError, TypeError, capnp.KjException) as error: + raise FixtureError(f"cannot initialize union branch: {path}") from error + walk(getattr(src, part), dst_branch, remaining[1:]) + return + + try: + src_value = getattr(src, part) + except (AttributeError, TypeError, capnp.KjException) as error: + raise FixtureError(f"policy field does not exist: {path}") from error + + if len(remaining) == 1: + try: + setattr(dst, part, src_value) + except (AttributeError, TypeError, capnp.KjException) as error: + raise FixtureError(f"cannot copy policy field: {path}") from error + return + + try: + if remaining[1] == "[]": + src_list = src_value + existing = getattr(dst, part) + if len(existing) == len(src_list) and len(existing) > 0: + dst_value = existing + else: + dst_value = dst.init(part, len(src_list)) + elif part in getattr(dst, "schema", type("x", (), {"union_fields": ()})).union_fields: + # only init inactive/missing union branch + try: + if dst.which() == part: + dst_value = getattr(dst, part) + else: + dst_value = dst.init(part) + except capnp.KjException: + dst_value = dst.init(part) + else: + dst_value = getattr(dst, part) + except (AttributeError, TypeError, capnp.KjException) as error: + raise FixtureError(f"policy field does not exist: {path}") from error + walk(src_value, dst_value, remaining[1:]) + + walk(source, destination, parts) + + +def copy_service(source: Any, destination: Any, service: str, paths: list[str]) -> None: + try: + source_value = getattr(source, service) + except AttributeError as error: + raise FixtureError(f"source service is unavailable: {service}") from error + try: + if is_list_service(service): + destination_value = destination.init(service, len(source_value)) + else: + destination_value = destination.init(service) + except (AttributeError, TypeError, capnp.KjException) as error: + raise FixtureError(f"cannot initialize service: {service}") from error + for path in paths: + copy_path(source_value, destination_value, path) + + +def zero_sensitive_location(location: Any) -> None: + for field in ("latitude", "longitude", "altitude", "bearingDeg", "unixTimestampMillis"): + if field in location.schema.fieldnames: + setattr(location, field, 0) + if "vNED" in location.schema.fieldnames: + location.vNED = [0.0 for _ in range(len(location.vNED) or 3)] + + +def build_event(source: Any, policy: dict[str, Any]) -> Any | None: + service = source.which() + allowed = policy["allowed_services"] + if service not in allowed: + return None + event = capnp_log.Event.new_message() + event.logMonoTime = source.logMonoTime + event.valid = source.valid + paths = allowed[service] + copy_service(source, event, service, paths) + if service == "carParams": + event.carParams.carVin = policy["placeholder_vin"] + if len(event.carParams.carFw): + event.carParams.carFw = [] + if service == policy["selected_gps_service"] or service in ("gpsLocation", "gpsLocationExternal"): + zero_sensitive_location(getattr(event, service)) + return event.as_reader() + + +def default_equal(value: Any, default: Any) -> bool: + if isinstance(value, (bytes, bytearray)) or isinstance(default, (bytes, bytearray)): + return bytes(value or b"") == bytes(default or b"") + if isinstance(value, str) or isinstance(default, str): + return str(value or "") == str(default or "") + try: + if hasattr(value, "to_dict") and hasattr(default, "to_dict"): + return value.to_dict() == default.to_dict() + except Exception: + pass + try: + return list(value) == list(default) + except TypeError: + return value == default + + +def populated_leaf_paths(struct: Any, prefix: str = "") -> dict[str, Any]: + """Return non-default leaf paths using schema reflection.""" + result: dict[str, Any] = {} + if struct is None: + return result + schema = struct.schema + active_union = None + if schema.union_fields: + try: + active_union = struct.which() + except capnp.KjException: + active_union = None + + for name in schema.fieldnames: + if name in schema.union_fields and name != active_union: + continue + field = schema.fields[name] + leaf = path_join(prefix, name) + try: + value = getattr(struct, name) + except capnp.KjException: + continue + if field.proto.which() == "group": + result.update(populated_leaf_paths(value, leaf)) + continue + if field.proto.which() != "slot": + continue + twhich = field.proto.slot.type.which() + if twhich == "struct": + result.update(populated_leaf_paths(value, leaf)) + elif twhich == "list": + elem = field.proto.slot.type.list.elementType + list_leaf = f"{leaf}[]" + if elem.which() == "struct": + for item in value: + result.update(populated_leaf_paths(item, list_leaf)) + else: + if len(value) and any(bool(item) if not isinstance(item, (int, float)) else float(item) != 0 for item in value): + result[list_leaf] = list(value) + elif twhich == "text": + if value: + result[leaf] = value + elif twhich == "data": + if bytes(value): + result[leaf] = bytes(value) + elif twhich == "bool": + if bool(value): + result[leaf] = bool(value) + elif twhich == "enum": + if int(getattr(value, "raw", value)) != 0: + result[leaf] = str(value) + else: + try: + if float(value) != 0: + result[leaf] = value + except Exception: + if value: + result[leaf] = value + return result + + +def path_matches_allowlist(leaf: str, allowed: set[str]) -> bool: + if leaf in allowed: + return True + # allowlist stores paths with [] markers; leaf paths from populated_leaf_paths also use [] + return leaf in allowed + + +def _normalize(value: Any) -> Any: + if isinstance(value, dict): + return {key: _normalize(item) for key, item in value.items() if item not in (None, "", [], {}, b"")} + if isinstance(value, list): + return [_normalize(item) for item in value] + return value + + +def _to_dict(value: Any) -> Any: + if hasattr(value, "to_dict"): + return _normalize(value.to_dict()) + try: + return _normalize([item.to_dict() if hasattr(item, "to_dict") else item for item in value]) + except TypeError: + return value + + +def validate_field_allowlist(messages: list[Any], policy: dict[str, Any]) -> None: + allowed_map: dict[str, set[str]] = policy["_allowed_leaf_sets"] + selected_gps = policy["selected_gps_service"] + for event in messages: + service = event.which() + if service not in allowed_map: + raise FixtureError(f"undeclared service in output: {service}") + rebuilt = build_event(event, policy) + if rebuilt is None: + raise FixtureError(f"undeclared service in output: {service}") + # Compare against a copy whose sensitive GPS zeros match sanitize semantics so + # allowlisting detects undeclared leaves, not sensitive-zero violations. + if service == selected_gps or service in ("gpsLocation", "gpsLocationExternal"): + original_builder = event.as_builder() + zero_sensitive_location(getattr(original_builder, service)) + original = _to_dict(getattr(original_builder, service)) + payload = getattr(original_builder, service) + else: + original = _to_dict(getattr(event, service)) + payload = getattr(event, service) + canonical = _to_dict(getattr(rebuilt, service)) + if original == canonical: + continue + rebuilt_payload = getattr(rebuilt, service) + if is_list_service(service): + leaves: dict[str, Any] = {} + canon_leaves: dict[str, Any] = {} + for item in payload: + leaves.update(populated_leaf_paths(item, "[]")) + for item in rebuilt_payload: + canon_leaves.update(populated_leaf_paths(item, "[]")) + else: + leaves = populated_leaf_paths(payload) + canon_leaves = populated_leaf_paths(rebuilt_payload) + allowed = allowed_map[service] + for leaf in sorted(set(leaves) - set(canon_leaves)): + if not path_matches_allowlist(leaf, allowed): + raise FixtureError(f"undeclared populated field: {service}.{leaf}") + raise FixtureError(f"undeclared populated field under service: {service}") + + +def iter_text_data_leaves(node: Any, prefix: str = "") -> list[tuple[str, bytes]]: + leaves: list[tuple[str, bytes]] = [] + if node is None: + return leaves + if isinstance(node, list) or (hasattr(node, "__iter__") and not hasattr(node, "schema") and not isinstance(node, (str, bytes, bytearray))): + try: + for item in node: + leaves.extend(iter_text_data_leaves(item, f"{prefix}[]" if prefix else "[]")) + return leaves + except TypeError: + pass + + schema = getattr(node, "schema", None) + if schema is None: + return leaves + + active_union = None + if schema.union_fields: + try: + active_union = node.which() + except capnp.KjException: + active_union = None + + for name in schema.fieldnames: + if name in schema.union_fields and name != active_union: + continue + field = schema.fields[name] + leaf = path_join(prefix, name) + try: + value = getattr(node, name) + except capnp.KjException: + continue + if field.proto.which() == "group": + leaves.extend(iter_text_data_leaves(value, leaf)) + continue + if field.proto.which() != "slot": + continue + twhich = field.proto.slot.type.which() + if twhich == "text": + leaves.append((leaf, str(value).encode())) + elif twhich == "data": + leaves.append((leaf, bytes(value))) + elif twhich == "struct": + leaves.extend(iter_text_data_leaves(value, leaf)) + elif twhich == "list": + elem = field.proto.slot.type.list.elementType + list_prefix = f"{leaf}[]" + if elem.which() == "struct": + for item in value: + leaves.extend(iter_text_data_leaves(item, list_prefix)) + elif elem.which() == "text": + for item in value: + leaves.append((list_prefix, str(item).encode())) + elif elem.which() == "data": + for item in value: + leaves.append((list_prefix, bytes(item))) + return leaves + + +def can_frames(message: Any) -> list[tuple[int, bytes, int]]: + return [(int(frame.address), bytes(frame.dat), int(frame.src)) for frame in message] + + +def vin_fragments(vin: bytes) -> list[bytes]: + fragments: list[bytes] = [] + upper = min(VIN_FRAGMENT_MAX, len(vin)) + for size in range(VIN_FRAGMENT_MIN, upper + 1): + for offset in range(len(vin) - size + 1): + fragments.append(vin[offset:offset + size]) + return fragments + + +def scan_buffer(matches: Counter[tuple[str, int, str]], source: str, buffer: bytes, + tokens: list[tuple[str, bytes]], vin: bytes | None) -> None: + for label, value in tokens: + if value and value in buffer: + matches[(label, len(value), source)] += buffer.count(value) + if vin: + for fragment in vin_fragments(vin): + if fragment and fragment in buffer: + matches[("vin", len(fragment), source)] += buffer.count(fragment) + + +def _append_capped(buffer: bytes, payload: bytes, limit: int = MAX_CAN_CONCAT_BYTES) -> bytes: + combined = buffer + payload + if len(combined) <= limit: + return combined + return combined[-limit:] + + +def token_scan(messages: list[Any], tokens: list[tuple[str, bytes]], vin: bytes | None) -> list[dict[str, Any]]: + matches: Counter[tuple[str, int, str]] = Counter() + scan_vin = vin if vin is not None else None + # Exact Text/Data leaves via schema reflection + for event in messages: + service = event.which() + payload = getattr(event, service) + for leaf, data in iter_text_data_leaves(payload, service): + scan_buffer(matches, leaf, data, tokens, scan_vin) + + # CAN per-frame, short rolling windows, full concatenations, and VIN-substring + # reassembly that skips unrelated interleaved payloads. + bus_suffix: dict[int, bytes] = {} + bus_concat: dict[int, bytes] = {} + addr_concat: dict[tuple[int, int], bytes] = {} + vin_bus_concat: dict[int, bytes] = {} + vin_global_concat = b"" + global_suffix = b"" + global_concat = b"" + for event in messages: + if event.which() != "can": + continue + for address, payload, bus in can_frames(event.can): + source = f"can[{bus}]@0x{address:x}" + scan_buffer(matches, source, payload, tokens, scan_vin) + prev = bus_suffix.get(bus, b"") + window = prev[-CAN_OVERLAP:] + payload + if prev: + scan_buffer(matches, f"can[{bus}].rolling", window, tokens, scan_vin) + bus_suffix[bus] = window[-CAN_OVERLAP:] + gwindow = global_suffix[-CAN_OVERLAP:] + payload + if global_suffix: + scan_buffer(matches, "can.global.rolling", gwindow, tokens, scan_vin) + global_suffix = gwindow[-CAN_OVERLAP:] + + bus_concat[bus] = _append_capped(bus_concat.get(bus, b""), payload) + addr_key = (bus, address) + addr_concat[addr_key] = _append_capped(addr_concat.get(addr_key, b""), payload) + global_concat = _append_capped(global_concat, payload) + + if scan_vin and payload and payload in scan_vin: + vin_bus_concat[bus] = _append_capped(vin_bus_concat.get(bus, b""), payload) + vin_global_concat = _append_capped(vin_global_concat, payload) + + for bus, buffer in bus_concat.items(): + scan_buffer(matches, f"can[{bus}].concat", buffer, tokens, scan_vin) + for (bus, address), buffer in addr_concat.items(): + scan_buffer(matches, f"can[{bus}]@0x{address:x}.concat", buffer, tokens, scan_vin) + if global_concat: + scan_buffer(matches, "can.global.concat", global_concat, tokens, scan_vin) + for bus, buffer in vin_bus_concat.items(): + scan_buffer(matches, f"can[{bus}].vin_reassembly", buffer, tokens, scan_vin) + if vin_global_concat: + scan_buffer(matches, "can.global.vin_reassembly", vin_global_concat, tokens, scan_vin) + + return [ + {"label": label, "length": length, "count": count, "source": source} + for (label, length, source), count in sorted(matches.items()) + ] + + +def zero_match_accounting(tokens: list[tuple[str, bytes]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + seen: set[tuple[str, int]] = set() + for label, value in tokens: + key = (label, len(value)) + if key in seen: + continue + seen.add(key) + rows.append({"label": label, "length": len(value), "count": 0, "source": ""}) + if label == "vin": + for size in range(VIN_FRAGMENT_MIN, min(VIN_FRAGMENT_MAX, len(value)) + 1): + frag_key = (label, size) + if frag_key in seen: + continue + seen.add(frag_key) + rows.append({"label": label, "length": size, "count": 0, "source": ""}) + return rows + + +def validate_can(messages: list[Any], policy: dict[str, Any], mode: str) -> dict[str, Any]: + rule = policy["pedal"] + frames: list[tuple[int, bytes, int]] = [] + for event in messages: + if event.which() == "can": + frames.extend(can_frames(event.can)) + pedal_frames = [(data, bus) for address, data, bus in frames if address == rule["address"]] + if mode == "no-pedal": + if pedal_frames: + raise FixtureError(f"no-pedal fixture contains pedal sensor frames: count={len(pedal_frames)}") + return {"address": rule["address"], "count": 0, "valid_count": 0, "counter_count": 0, "bus": rule["bus"]} + valid: list[bytes] = [] + counters: set[int] = set() + for data, bus in pedal_frames: + if bus != rule["bus"] or len(data) != rule["data_length"]: + continue + checksum = (sum(data[:rule["checksum_index"]]) + (rule["address"] & 0xff) + (rule["address"] >> 8)) & 0xff + if data[rule["checksum_index"]] == checksum: + valid.append(data) + counters.add(data[rule["counter_index"]] & rule["counter_mask"]) + if len(valid) < rule["minimum_frames"] or len(counters) < rule["minimum_counters"]: + raise FixtureError(f"pedal evidence insufficient: valid={len(valid)} counters={len(counters)}") + return { + "address": rule["address"], "count": len(pedal_frames), "valid_count": len(valid), + "counter_count": len(counters), "bus": rule["bus"], + } + + +def structural_checks(messages: list[Any], policy: dict[str, Any], case: str) -> dict[str, Any]: + if not messages: + raise FixtureError("fixture is empty") + counts = Counter(event.which() for event in messages) + valid_counts = Counter(event.which() for event in messages if event.valid) + valid_count = sum(valid_counts.values()) + valid_share = valid_count / len(messages) + if valid_share < VALID_SHARE_THRESHOLD: + raise FixtureError(f"valid-message share below 90 percent: {valid_share:.3f}") + + allowed = set(policy["allowed_services"]) + for service in counts: + if service not in allowed: + raise FixtureError(f"undeclared service in output: {service}") + + for service, minimum in policy["required_services"].items(): + if valid_counts.get(service, 0) < minimum: + raise FixtureError(f"required valid service count unmet: {service}") + + params = [event.carParams for event in messages if event.which() == "carParams"] + if not params: + raise FixtureError("carParams is required") + expected = policy["cases"][case] + pedal = expected["mode"] == "pedal" + expected_param = int(expected["safety_param"]) + if expected_param != expected_safety_param(expected["mode"]): + raise FixtureError("policy safety_param does not match fixed Pre-AP flags") + for cp in params: + if cp.carVin != policy["placeholder_vin"]: + raise FixtureError("carParams.carVin is not the public placeholder") + if len(cp.carFw) != 0: + raise FixtureError("carParams.carFw must be empty") + if cp.brand != "tesla": + raise FixtureError("carParams.brand must be tesla") + if cp.carFingerprint != expected["fingerprint"]: + raise FixtureError("unexpected car fingerprint") + if len(cp.safetyConfigs) != 1: + raise FixtureError("carParams must contain exactly one safetyConfigs entry") + safety = cp.safetyConfigs[0] + if not safety_model_is_tesla_preap(safety.safetyModel): + raise FixtureError("carParams safetyModel must be teslaPreap") + if int(safety.safetyParam) != expected_param: + raise FixtureError(f"carParams safetyParam must be {expected_param}") + if bool(cp.openpilotLongitudinalControl) != pedal or bool(cp.pcmCruise) == pedal: + raise FixtureError("Pre-AP pedal/longitudinal assertions failed") + + # Sensitive zeros must fail closed before allowlist rebuild compare, which also + # zeroes GPS and would otherwise mislabel vNED/location leaks as undeclared fields. + for field in policy["sensitive_zero_fields"]: + parts = field_parts(field) + service = parts[0] + for event in messages: + if event.which() != service: + continue + node: Any = getattr(event, service) + index = 1 + while index < len(parts): + part = parts[index] + if part == "[]": + if index == len(parts) - 1: + if any(float(item) != 0 for item in node): + raise FixtureError(f"sensitive field is not zero: {field}") + break + raise FixtureError(f"invalid sensitive path: {field}") + node = getattr(node, part) + index += 1 + else: + if hasattr(node, "__iter__") and not isinstance(node, (str, bytes, bytearray)): + if any(float(item) != 0 for item in node): + raise FixtureError(f"sensitive field is not zero: {field}") + elif float(node) != 0: + raise FixtureError(f"sensitive field is not zero: {field}") + + validate_field_allowlist(messages, policy) + + return { + "message_counts": dict(sorted(counts.items())), + "valid_message_counts": dict(sorted(valid_counts.items())), + "valid_message_share": valid_share, + } + + +def canonical_policy_digest(raw: dict[str, Any]) -> str: + material = {key: raw[key] for key in sorted(POLICY_KEYS)} + return digest(json.dumps(material, sort_keys=True, separators=(",", ":")).encode()) + + +def parse_policy(path: str) -> dict[str, Any]: + try: + with open(path, encoding="utf-8") as stream: + raw = json.load(stream) + except json.JSONDecodeError as error: + raise FixtureError("policy is malformed JSON") from error + except OSError as error: + raise FixtureError("policy cannot be read") from error + if not isinstance(raw, dict): + raise FixtureError("policy root must be an object") + if set(raw) != POLICY_KEYS: + raise FixtureError("policy keys are invalid") + if raw["schema_version"] != SCHEMA_VERSION: + raise FixtureError("unsupported policy schema_version") + if raw["placeholder_vin"] != PLACEHOLDER_VIN: + raise FixtureError("placeholder_vin must be the fixed public value") + if canonical_policy_digest(raw) != RELEASE_POLICY_SHA256: + raise FixtureError("policy is not the attested release contract") + + allowed = raw["allowed_services"] + forbidden = raw["forbidden_services"] + required = raw["required_services"] + freshness = raw["freshness_services"] + if not isinstance(allowed, dict) or not allowed: + raise FixtureError("allowed_services must be a nonempty object") + if not isinstance(forbidden, list) or not all(isinstance(item, str) for item in forbidden): + raise FixtureError("forbidden_services must be a string list") + if not isinstance(required, dict) or not required: + raise FixtureError("required_services must be a nonempty object") + if not isinstance(freshness, list) or not all(isinstance(item, str) for item in freshness): + raise FixtureError("freshness_services must be a string list") + if not isinstance(raw["selected_gps_service"], str): + raise FixtureError("selected_gps_service must be a string") + + known = event_service_names() + for service in list(allowed) + forbidden + list(required) + freshness + [raw["selected_gps_service"]]: + if service not in known: + raise FixtureError(f"unknown Event service in policy: {service}") + + if set(allowed) & set(forbidden): + raise FixtureError("allowed_services and forbidden_services must be disjoint") + missing_forbidden = MANDATORY_FORBIDDEN - set(forbidden) + if missing_forbidden: + raise FixtureError("forbidden_services missing mandatory baseline") + if raw["selected_gps_service"] not in allowed: + raise FixtureError("selected_gps_service must be allowed") + other_gps = {"gpsLocation", "gpsLocationExternal"} - {raw["selected_gps_service"]} + if other_gps & set(allowed): + raise FixtureError("only the selected GPS service may be allowed") + for service in freshness: + if service not in allowed: + raise FixtureError(f"freshness service must be allowed: {service}") + + leaf_sets: dict[str, set[str]] = {} + for service, paths in allowed.items(): + if not isinstance(service, str) or not isinstance(paths, list) or not all(isinstance(item, str) for item in paths): + raise FixtureError("allowed_services entries must map to string lists") + if service in freshness and paths: + # freshness-only services must not copy identifying leaves + raise FixtureError(f"freshness service must not declare identifying leaves: {service}") + leaf_sets[service] = expand_allowed_paths(service, paths) if paths else set() + + for service, minimum in required.items(): + if service not in allowed: + raise FixtureError(f"required service is not allowed: {service}") + if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 1: + raise FixtureError(f"required service minimum must be a positive int: {service}") + + pedal = raw["pedal"] + if not isinstance(pedal, dict) or set(pedal) != PEDAL_KEYS: + raise FixtureError("pedal policy keys are invalid") + for key in ("address", "bus", "data_length", "counter_index", "counter_mask", "checksum_index", + "minimum_frames", "minimum_counters"): + value = pedal[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise FixtureError(f"pedal.{key} must be a non-negative int") + if pedal["checksum"] != "tesla_sum8_address": + raise FixtureError("unsupported pedal checksum") + if pedal["checksum_index"] >= pedal["data_length"] or pedal["counter_index"] >= pedal["data_length"]: + raise FixtureError("pedal indices out of range") + if pedal["data_length"] < 1 or pedal["bus"] > 255: + raise FixtureError("pedal ranges are invalid") + if pedal["minimum_frames"] < PEDAL_MIN_FRAMES or pedal["minimum_counters"] < PEDAL_MIN_COUNTERS: + raise FixtureError("pedal evidence minima below release floor") + + zeros = raw["sensitive_zero_fields"] + if not isinstance(zeros, list) or not all(isinstance(item, str) for item in zeros): + raise FixtureError("sensitive_zero_fields must be a string list") + selected = raw["selected_gps_service"] + required_zeros = {item for item in SENSITIVE_ZERO_BASELINE if item.startswith(selected + ".")} + if not required_zeros.issubset(set(zeros)): + raise FixtureError("sensitive_zero_fields missing mandatory GPS zeros including vNED") + for item in zeros: + service = item.split(".", 1)[0] + if service not in allowed and service not in ("gpsLocation", "gpsLocationExternal"): + raise FixtureError(f"sensitive zero service not allowed: {service}") + # validate path shape against schema when service allowed + if service in allowed: + rest = item.split(".", 1)[1] + resolve_policy_path(service, rest) + + cases = raw["cases"] + if not isinstance(cases, dict) or not cases: + raise FixtureError("cases must be a nonempty object") + for name, case in cases.items(): + if not isinstance(name, str) or not isinstance(case, dict) or set(case) != CASE_KEYS: + raise FixtureError("case entries are invalid") + if case["mode"] not in CASE_MODES or not isinstance(case["fingerprint"], str) or not case["fingerprint"]: + raise FixtureError(f"case mode/fingerprint invalid: {name}") + if case["fingerprint"] != "TESLA_MODEL_S_PREAP": + raise FixtureError("only TESLA_MODEL_S_PREAP cases are supported") + safety_param = case["safety_param"] + if not isinstance(safety_param, int) or isinstance(safety_param, bool): + raise FixtureError(f"case safety_param must be an int: {name}") + if safety_param != expected_safety_param(case["mode"]): + raise FixtureError(f"case safety_param must be {expected_safety_param(case['mode'])}: {name}") + + raw["_allowed_leaf_sets"] = leaf_sets + return raw + + +def decode_deny_token_value(encoding: str, value: str) -> bytes: + """Decode a deny token payload with strict, canonical binary-safe encodings.""" + if encoding not in DENY_ENCODINGS: + raise FixtureError("deny token encoding must be utf8, hex, or base64") + if not isinstance(value, str): + raise FixtureError("deny token value must be a string") + if encoding == "utf8": + try: + return value.encode("utf-8") + except UnicodeEncodeError as error: + raise FixtureError("deny token utf8 value is invalid") from error + if encoding == "hex": + if len(value) % 2 != 0 or not HEX_VALUE_RE.fullmatch(value): + raise FixtureError("deny token hex value must be lowercase hexadecimal") + return bytes.fromhex(value) + try: + decoded = base64.b64decode(value, validate=True) + except binascii.Error as error: + raise FixtureError("deny token base64 value is invalid") from error + if base64.b64encode(decoded).decode("ascii") != value: + raise FixtureError("deny token base64 value must be canonical") + return decoded + + +def load_deny(path: str) -> DenyManifest: + try: + with open(path, encoding="utf-8") as stream: + raw = json.load(stream) + except json.JSONDecodeError as error: + raise FixtureError("deny file is malformed JSON") from error + if not isinstance(raw, dict): + raise FixtureError("deny file root must be an object") + unknown = set(raw) - DENY_REQUIRED_KEYS - DENY_OPTIONAL_KEYS + if unknown: + raise FixtureError("deny file contains unknown keys") + if "raw_input_sha256" not in raw and "input_sha256" not in raw: + raise FixtureError("deny raw_input_sha256 must be lowercase hexadecimal SHA-256") + tokens_raw = raw.get("tokens") + if not isinstance(tokens_raw, list) or not tokens_raw: + raise FixtureError("deny file must contain a nonempty tokens list") + expected = raw.get("raw_input_sha256", raw.get("input_sha256")) + if not isinstance(expected, str) or not SHA256_RE.fullmatch(expected): + raise FixtureError("deny raw_input_sha256 must be lowercase hexadecimal SHA-256") + sanitized = raw.get("sanitized_output_sha256") + if sanitized is not None and (not isinstance(sanitized, str) or not SHA256_RE.fullmatch(sanitized)): + raise FixtureError("deny sanitized_output_sha256 must be lowercase hexadecimal SHA-256") + + tokens: list[tuple[str, bytes]] = [] + labels: set[str] = set() + values: set[bytes] = set() + for entry in tokens_raw: + if not isinstance(entry, dict) or set(entry) != {"label", "encoding", "value"}: + raise FixtureError("deny token entries require exactly label, encoding, and value") + label = entry["label"] + encoding = entry["encoding"] + value = entry["value"] + if not isinstance(label, str) or not SAFE_LABEL_RE.fullmatch(label): + raise FixtureError("deny token label is not a safe identifier") + if not isinstance(encoding, str): + raise FixtureError("deny token encoding must be utf8, hex, or base64") + encoded = decode_deny_token_value(encoding, value) + if len(encoded) < VIN_FRAGMENT_MIN: + raise FixtureError("deny token is shorter than six bytes") + if label in labels: + raise FixtureError("deny token labels must be unique") + if encoded in values: + raise FixtureError("deny token values must be unique") + labels.add(label) + values.add(encoded) + tokens.append((label, encoded)) + + vin_tokens = [(label, value) for label, value in tokens if label == "vin"] + if len(vin_tokens) != 1: + raise FixtureError("deny file must contain exactly one vin token") + vin = vin_tokens[0][1] + if len(vin) != VIN_LENGTH: + raise FixtureError("vin token must be exactly 17 bytes") + return DenyManifest( + tokens=tuple(tokens), + raw_input_sha256=expected, + sanitized_output_sha256=sanitized, + vin=vin, + ) + + +def extract_source_vins(messages: list[Any]) -> set[str]: + vins = {str(event.carParams.carVin) for event in messages if event.which() == "carParams"} + if not vins: + raise FixtureError("source carParams.carVin is required") + return vins + + +def verify_deny_vin_against_source(messages: list[Any], vin: bytes) -> None: + vins = extract_source_vins(messages) + if len(vins) != 1: + raise FixtureError("source carParams.carVin must be unique") + source_vin = next(iter(vins)) + if len(source_vin.encode()) != VIN_LENGTH: + raise FixtureError("source carParams.carVin must be exactly 17 bytes") + if source_vin.encode() != vin: + raise FixtureError("deny vin token does not match source carParams.carVin") + + +def relative_text_data_leaf(service: str, leaf: str) -> str: + if leaf == service: + return "" + if leaf.startswith(service) and len(leaf) > len(service) and leaf[len(service)] in ".[": + rest = leaf[len(service):] + return rest[1:] if rest.startswith(".") else rest + raise FixtureError(f"text/data leaf provenance is malformed: {leaf}") + + +def is_scrubbed_private_leaf(leaf: str) -> bool: + # carVin is allowlisted but replaced; carFw is stripped even if nested Data appears. + return leaf == "carParams.carVin" or leaf.startswith("carParams.carFw[]") + + +def is_private_text_data_leaf(service: str, leaf: str, policy: dict[str, Any]) -> bool: + """True when a raw Text/Data leaf must be covered by deny tokens (provenance-based).""" + if is_scrubbed_private_leaf(leaf): + return True + if leaf in ATTESTED_PUBLIC_TEXT_DATA_LEAVES: + return False + allowed = policy["_allowed_leaf_sets"].get(service) + if allowed is None: + return True + rel = relative_text_data_leaf(service, leaf) + # Allowlisted retained channels (including can[].dat) are not inventory sources. + # They remain token-scan sinks and never exempt private-provenance duplicates. + if path_matches_allowlist(rel, allowed): + return False + return True + + +def derive_private_text_data_inventory(messages: list[Any], policy: dict[str, Any]) -> set[bytes]: + """Inventory private Text/Data by raw leaf provenance, not sanitized set-subtraction.""" + inventory: set[bytes] = set() + for event in messages: + service = event.which() + payload = getattr(event, service) + for leaf, data in iter_text_data_leaves(payload, service): + if len(data) < VIN_FRAGMENT_MIN: + continue + if is_private_text_data_leaf(service, leaf, policy): + inventory.add(data) + return inventory + + +def verify_deny_inventory_completeness( + source_messages: list[Any], + tokens: list[tuple[str, bytes]], + vin: bytes, + policy: dict[str, Any], +) -> None: + verify_deny_vin_against_source(source_messages, vin) + private_inventory = derive_private_text_data_inventory(source_messages, policy) + covered = {value for _label, value in tokens} + missing = private_inventory - covered + if missing: + raise FixtureError("deny tokens incomplete for decoded private Text/Data inventory") + + +def lstat_path(path: Path) -> os.stat_result: + try: + return os.lstat(path) + except OSError as error: + raise FixtureError("path is inaccessible") from error + + +def assert_not_symlink(path: Path, label: str) -> None: + if path.exists() and path.is_symlink(): + raise FixtureError(f"{label} must not be a symlink") + + +def same_file(stat_a: os.stat_result, stat_b: os.stat_result) -> bool: + return stat_a.st_ino == stat_b.st_ino and stat_a.st_dev == stat_b.st_dev + + +def reject_symlink_parents(path: Path, label: str) -> None: + current = path if path.is_absolute() else Path.cwd() / path + for parent in [current, *current.parents]: + try: + if parent.is_symlink(): + raise FixtureError(f"{label} parent must not be a symlink") + except OSError as error: + raise FixtureError(f"{label} path is inaccessible") from error + + +def canonicalize_endpoint(path: Path, label: str) -> Path: + reject_symlink_parents(path, label) + absolute = path if path.is_absolute() else Path.cwd() / path + parent = absolute.parent + try: + resolved_parent = parent.resolve(strict=False) + except OSError as error: + raise FixtureError(f"{label} path is inaccessible") from error + reject_symlink_parents(resolved_parent / absolute.name, label) + canonical = resolved_parent / absolute.name + if canonical.exists() or canonical.is_symlink(): + assert_not_symlink(canonical, label) + return canonical + + +def validate_path_separation(paths: dict[str, Path]) -> dict[str, Path]: + resolved: dict[str, Path] = {} + for label, path in paths.items(): + resolved[label] = canonicalize_endpoint(path, label) + items = list(resolved.items()) + for index, (label_a, path_a) in enumerate(items): + for label_b, path_b in items[index + 1:]: + if path_a == path_b: + raise FixtureError(f"path collision between {label_a} and {label_b}") + if path_a.exists() and path_b.exists(): + if same_file(lstat_path(path_a), lstat_path(path_b)): + raise FixtureError(f"inode alias between {label_a} and {label_b}") + return resolved + + +def open_private_temp(destination_dir: Path, suffix: str) -> tuple[int, Path]: + destination_dir.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=".fixture-", suffix=suffix, dir=str(destination_dir)) + try: + os.fchmod(fd, 0o600) + except OSError: + os.close(fd) + Path(name).unlink(missing_ok=True) + raise + path = Path(name) + if path.is_symlink(): + os.close(fd) + path.unlink(missing_ok=True) + raise FixtureError("temporary path must not be a symlink") + return fd, path + + +def write_bytes_atomic(destination: Path, data: bytes) -> None: + fd, tmp = open_private_temp(destination.parent, suffix=".tmp") + try: + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, destination) + except Exception: + try: + os.close(fd) + except Exception: + pass + tmp.unlink(missing_ok=True) + raise + + +def write_json_atomic(destination: Path, payload: dict[str, Any]) -> None: + data = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + write_bytes_atomic(destination, data) + + +def serialize_log_bytes(messages: list[Any], *, compress: bool = True) -> bytes: + data = b"".join(message.as_builder().to_bytes() for message in messages) + if compress: + return zstandard.ZstdCompressor(level=10).compress(data) + return data + + +def write_fd_bytes(fd: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(fd, view) + if written <= 0: + raise FixtureError("failed to write temporary fixture bytes") + view = view[written:] + os.fsync(fd) + + +def public_status(*, case: str, scope: str, output_sha256: str | None, output_bytes: int | None, + performed: bool | None = None) -> dict[str, Any]: + report: dict[str, Any] = {"status": "ok", "case": case, "scope": scope} + if output_sha256 is not None: + report["sanitized_output_sha256" if scope != "public" else "fixture_sha256"] = output_sha256 + if output_bytes is not None: + report["output_bytes"] = output_bytes + if performed is not None: + report["private_token_scan"] = {"performed": performed} + return report + + +def build_validation_report( + *, + policy: dict[str, Any], + case: str, + messages: list[Any], + fixture_bytes: bytes, + scope: str, + tokens: list[tuple[str, bytes]] | None, + vin: bytes | None, + source_message_counts: dict[str, int] | None, +) -> dict[str, Any]: + report: dict[str, Any] = { + "schema_version": policy["schema_version"], + "case": case, + "scope": scope, + "fixture_sha256": digest(fixture_bytes), + "output_bytes": len(fixture_bytes), + } + if source_message_counts is not None: + report["source_message_counts"] = source_message_counts + report.update(structural_checks(messages, policy, case)) + report["can_evidence"] = validate_can(messages, policy, policy["cases"][case]["mode"]) + + if scope == "public": + report["private_token_scan"] = {"performed": False} + else: + assert tokens is not None and vin is not None + matches = token_scan(messages, tokens, vin) + accounting = zero_match_accounting(list(tokens)) + if matches: + by_key = {(row["label"], row["length"]): row for row in accounting} + for match in matches: + key = (match["label"], match["length"]) + row = by_key.get(key) + if row is None: + accounting.append(dict(match)) + else: + row["count"] = match["count"] + row["source"] = match["source"] + report["private_token_scan"] = {"performed": True, "token_matches": matches} + report["token_match_accounting"] = accounting + report["_rejected_private_token"] = True + else: + report["private_token_scan"] = {"performed": True, "token_matches": accounting} + return report + + +def accept_six_process_outputs( + *, + case: str, + policy: dict[str, Any], + process_outputs: Mapping[str, list[Any]], + case_params: Mapping[str, Any], + valid_share_threshold: float = VALID_SHARE_THRESHOLD, +) -> dict[str, Any]: + """Publication gate over fixed typed case params and six-process replay outputs. + + Does not itself execute process replay. Callers supply the per-process output + streams produced under ``case_params`` (for example NAP_PREAP_*_PARAMS). + """ + if case not in policy["cases"]: + raise FixtureError("unknown fixture case") + expected = policy["cases"][case] + mode = expected["mode"] + if not isinstance(case_params.get("NAPPedalEnabled"), bool): + raise FixtureError("case_params.NAPPedalEnabled must be a bool") + if bool(case_params["NAPPedalEnabled"]) != (mode == "pedal"): + raise FixtureError("case_params.NAPPedalEnabled does not match case mode") + if case_params.get("NAPRadarEnabled") is not True or case_params.get("NAPRadarBehindNosecone") is not True: + raise FixtureError("case_params must enable radar emulation and nosecone flags") + + missing = [name for name in SIX_PROCESS_OUTPUTS if name not in process_outputs] + if missing: + raise FixtureError(f"missing process outputs: {','.join(missing)}") + + per_process: dict[str, Any] = {} + for proc_name, required_services in SIX_PROCESS_OUTPUTS.items(): + messages = list(process_outputs[proc_name]) + if not messages: + raise FixtureError(f"process produced no outputs: {proc_name}") + counts = Counter(message.which() for message in messages) + valid_counts = Counter(message.which() for message in messages if message.valid) + present = set(counts) + if not required_services.issubset(present): + raise FixtureError(f"process missing declared outputs: {proc_name}") + shares: dict[str, float] = {} + for service in required_services: + total = counts[service] + share = valid_counts.get(service, 0) / total + shares[service] = share + if share < valid_share_threshold: + raise FixtureError(f"process valid-share below threshold: {proc_name}.{service}") + per_process[proc_name] = { + "message_counts": dict(sorted(counts.items())), + "valid_shares": dict(sorted(shares.items())), + } + + card_params = [message.carParams for message in process_outputs["card"] if message.which() == "carParams"] + if not card_params: + raise FixtureError("card did not emit carParams") + expected_param = int(expected["safety_param"]) + for cp in card_params: + if cp.carFingerprint != expected["fingerprint"]: + raise FixtureError("card carParams fingerprint mismatch") + if len(cp.safetyConfigs) != 1 or not safety_model_is_tesla_preap(cp.safetyConfigs[0].safetyModel): + raise FixtureError("card carParams safety config mismatch") + if int(cp.safetyConfigs[0].safetyParam) != expected_param: + raise FixtureError("card carParams safetyParam mismatch") + pedal = mode == "pedal" + if bool(cp.openpilotLongitudinalControl) != pedal or bool(cp.pcmCruise) == pedal: + raise FixtureError("card carParams mode mismatch") + + return { + "status": "ok", + "case": case, + "processes": list(SIX_PROCESS_OUTPUTS), + "process_reports": per_process, + "card_mode": mode, + "card_fingerprint": expected["fingerprint"], + "card_safety_param": expected_param, + } + + +def decompress_zstd_bounded(raw: bytes) -> bytes: + """Bounded single-frame zstd decompress with exact EOF and trailing-data rejection.""" + try: + params = zstandard.get_frame_parameters(raw) + except zstandard.ZstdError as error: + raise FixtureError("input is malformed") from error + declared = params.content_size + if declared != ZSTD_CONTENTSIZE_UNKNOWN and declared > MAX_DECOMPRESSED_INPUT_BYTES: + raise FixtureError("input exceeds decompressed size limit") + + decompressor = zstandard.ZstdDecompressor().decompressobj() + chunks: list[bytes] = [] + total = 0 + offset = 0 + try: + while offset < len(raw): + end = min(offset + ZSTD_DECOMPRESS_CHUNK, len(raw)) + piece = decompressor.decompress(raw[offset:end]) + offset = end + if piece: + total += len(piece) + if total > MAX_DECOMPRESSED_INPUT_BYTES: + raise FixtureError("input exceeds decompressed size limit") + chunks.append(piece) + if decompressor.eof: + trailing = decompressor.unused_data + raw[offset:] + if trailing: + raise FixtureError("input is malformed") + break + else: + if not decompressor.eof: + raise FixtureError("input is malformed") + except FixtureError: + raise + except zstandard.ZstdError as error: + message = str(error).lower() + if "maximum" in message or "too large" in message or "max" in message: + raise FixtureError("input exceeds decompressed size limit") from error + raise FixtureError("input is malformed") from error + return b"".join(chunks) + + +def load_source_messages(raw: bytes) -> list[Any]: + if len(raw) > MAX_COMPRESSED_INPUT_BYTES: + raise FixtureError("input exceeds compressed size limit") + data = raw + try: + if raw.startswith(ZSTD_MAGIC): + data = decompress_zstd_bounded(raw) + elif raw.startswith(b"BZh"): + data = __import__("bz2").decompress(raw) + if len(data) > MAX_DECOMPRESSED_INPUT_BYTES: + raise FixtureError("input exceeds decompressed size limit") + except FixtureError: + raise + except MemoryError as error: + raise FixtureError("input exceeds decompressed size limit") from error + except (zstandard.ZstdError, OSError, ValueError) as error: + message = str(error).lower() + if "maximum" in message or "too large" in message or "max" in message: + raise FixtureError("input exceeds decompressed size limit") from error + raise FixtureError("input is malformed") from error + + try: + import warnings + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", RuntimeWarning) + messages = list(LogReader.from_bytes(data)) + if any(issubclass(item.category, RuntimeWarning) for item in caught) or (len(data) > 0 and not messages): + raise FixtureError("input is malformed") + except MemoryError as error: + raise FixtureError("input exceeds decompressed size limit") from error + except FixtureError: + raise + except (capnp.KjException, zstandard.ZstdError, ValueError, OSError, RuntimeError) as error: + raise FixtureError("input is malformed") from error + + if len(messages) > MAX_LOG_MESSAGES: + raise FixtureError("input exceeds message count limit") + return messages + + +def process( + input_path: str, + output_path: str | None, + report_path: str, + policy_path: str, + case: str, + deny_path: str | None, + sanitize: bool, + input_sha256: str | None = None, +) -> dict[str, Any]: + policy = parse_policy(policy_path) + if case not in policy["cases"]: + raise FixtureError("unknown fixture case") + + input_p = Path(input_path) + policy_p = Path(policy_path) + report_p = Path(report_path) + output_p = Path(output_path) if output_path else None + deny_p = Path(deny_path) if deny_path else None + + path_map = {"input": input_p, "policy": policy_p, "report": report_p} + if output_p is not None: + path_map["output"] = output_p + if deny_p is not None: + path_map["deny"] = deny_p + if sanitize: + if output_p is None: + raise FixtureError("sanitize requires --output") + if output_p == report_p: + raise FixtureError("output and report must be distinct") + resolved = validate_path_separation(path_map) + input_p = resolved["input"] + policy_p = resolved["policy"] + report_p = resolved["report"] + output_p = resolved.get("output") + deny_p = resolved.get("deny") + + deny: DenyManifest | None = None + if sanitize: + if deny_p is None: + raise FixtureError("sanitize requires a deny token file") + deny = load_deny(str(deny_p)) + elif deny_p is not None: + deny = load_deny(str(deny_p)) + + if input_sha256 is not None: + if not SHA256_RE.fullmatch(input_sha256): + raise FixtureError("input_sha256 must be lowercase hexadecimal SHA-256") + + raw = input_p.read_bytes() + if len(raw) > MAX_COMPRESSED_INPUT_BYTES: + raise FixtureError("input exceeds compressed size limit") + raw_hash = digest(raw) + + if sanitize: + assert deny is not None + if raw_hash != deny.raw_input_sha256: + raise FixtureError("input SHA-256 does not match deny raw_input_sha256") + if input_sha256 is not None and input_sha256 != deny.raw_input_sha256: + raise FixtureError("input_sha256 conflicts with deny raw_input_sha256") + else: + if deny is not None: + expected_fixture = deny.sanitized_output_sha256 + if expected_fixture is None: + raise FixtureError("private validate requires deny sanitized_output_sha256") + if raw_hash != expected_fixture: + raise FixtureError("input SHA-256 does not match deny sanitized_output_sha256") + if input_sha256 is not None and input_sha256 != expected_fixture: + raise FixtureError("input_sha256 conflicts with deny sanitized_output_sha256") + elif input_sha256 is not None and raw_hash != input_sha256: + raise FixtureError("input SHA-256 does not match --input-sha256") + + source_messages = load_source_messages(raw) + + temp_output: Path | None = None + try: + if sanitize: + assert output_p is not None and deny is not None + # Fail closed for stale destinations: clear any preexisting output before work. + if output_p.exists() or output_p.is_symlink(): + assert_not_symlink(output_p, "output") + output_p.unlink() + messages = [built for event in source_messages if (built := build_event(event, policy)) is not None] + verify_deny_inventory_completeness(source_messages, list(deny.tokens), deny.vin, policy) + fd, temp_output = open_private_temp(output_p.parent, suffix=".zst") + try: + output_bytes = serialize_log_bytes(messages) + write_fd_bytes(fd, output_bytes) + finally: + os.close(fd) + messages = list(LogReader.from_bytes(output_bytes)) + report = build_validation_report( + policy=policy, + case=case, + messages=messages, + fixture_bytes=output_bytes, + scope="private", + tokens=list(deny.tokens), + vin=deny.vin, + source_message_counts=dict(sorted(Counter(event.which() for event in source_messages).items())), + ) + output_digest = digest(output_bytes) + report["sanitized_output_sha256"] = output_digest + if deny.sanitized_output_sha256 is not None and deny.sanitized_output_sha256 != output_digest: + raise FixtureError("sanitized output SHA-256 does not match deny sanitized_output_sha256") + rejected = report.pop("_rejected_private_token", False) + write_json_atomic(report_p, report) + if rejected: + temp_output.unlink(missing_ok=True) + temp_output = None + output_p.unlink(missing_ok=True) + raise FixtureError("private token detected in retained output") + os.replace(temp_output, output_p) + temp_output = None + return public_status( + case=case, scope="private", output_sha256=output_digest, + output_bytes=len(output_bytes), performed=True, + ) + + scope = "private" if deny is not None else "public" + if scope == "private": + assert deny is not None + # Private validate of a sanitized fixture still requires the original VIN token. + if deny.vin.decode() == PLACEHOLDER_VIN: + raise FixtureError("private validate vin token must be the original private VIN") + report = build_validation_report( + policy=policy, + case=case, + messages=source_messages, + fixture_bytes=raw, + scope=scope, + tokens=list(deny.tokens) if deny is not None else None, + vin=deny.vin if deny is not None else None, + source_message_counts=None, + ) + rejected = report.pop("_rejected_private_token", False) + write_json_atomic(report_p, report) + if rejected: + raise FixtureError("private token detected in retained output") + return public_status( + case=case, + scope=scope, + output_sha256=digest(raw), + output_bytes=len(raw), + performed=(scope == "private"), + ) + except Exception: + if temp_output is not None: + temp_output.unlink(missing_ok=True) + if sanitize and output_p is not None: + output_p.unlink(missing_ok=True) + raise + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("sanitize", "validate"): + sub = subparsers.add_parser(command) + sub.add_argument("--policy", required=True) + sub.add_argument("--input-sha256") + sub.add_argument("--case", required=True) + sub.add_argument("--input", required=True) + sub.add_argument("--output", required=command == "sanitize") + sub.add_argument("--report", required=True) + sub.add_argument("--deny-token-file") + args = parser.parse_args(argv) + try: + status = process( + args.input, + args.output, + args.report, + args.policy, + args.case, + args.deny_token_file, + args.command == "sanitize", + args.input_sha256, + ) + print(json.dumps(status, sort_keys=True, separators=(",", ":"))) + return 0 + except (FixtureError, OSError, capnp.KjException, zstandard.ZstdError, json.JSONDecodeError, TypeError, ValueError, KeyError): + print("fixture validation failed", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/selfdrive/test/process_replay/nap_fixture_policy.json b/selfdrive/test/process_replay/nap_fixture_policy.json new file mode 100644 index 00000000000000..9cebca8708f2dd --- /dev/null +++ b/selfdrive/test/process_replay/nap_fixture_policy.json @@ -0,0 +1,471 @@ +{ + "schema_version": 1, + "allowed_services": { + "can": [ + "[].address", + "[].dat", + "[].src" + ], + "carParams": [ + "brand", + "carFingerprint", + "fuzzyFingerprint", + "pcmCruise", + "enableDsu", + "enableBsm", + "flags", + "alphaLongitudinalAvailable", + "minEnableSpeed", + "minSteerSpeed", + "steerAtStandstill", + "safetyConfigs[].safetyModel", + "safetyConfigs[].safetyParam", + "alternativeExperience", + "mass", + "wheelbase", + "centerToFront", + "steerRatio", + "steerRatioRear", + "rotationalInertia", + "tireStiffnessFactor", + "tireStiffnessFront", + "tireStiffnessRear", + "steerLimitAlert", + "steerLimitTimer", + "vEgoStopping", + "vEgoStarting", + "steerControlType", + "radarUnavailable", + "stopAccel", + "stoppingDecelRate", + "startAccel", + "startingState", + "steerActuatorDelay", + "longitudinalActuatorDelay", + "openpilotLongitudinalControl", + "carVin", + "dashcamOnly", + "passive", + "notCar", + "transmissionType", + "radarDelay", + "fingerprintSource", + "networkLocation", + "wheelSpeedFactor", + "secOcRequired", + "secOcKeyAvailable", + "maxLateralAccel", + "autoResumeSng" + ], + "carState": [ + "canValid", + "canTimeout", + "canErrorCounter", + "vEgo", + "vEgoRaw", + "aEgo", + "standstill", + "gasPressed", + "brake", + "brakePressed", + "steeringAngleDeg", + "steeringRateDeg", + "steeringTorque", + "steeringPressed", + "steeringDisengage", + "cruiseState.enabled", + "cruiseState.available", + "cruiseState.speed", + "cruiseState.standstill", + "cruiseState.nonAdaptive", + "gearShifter", + "buttonEvents[].type", + "buttonEvents[].pressed", + "wheelSpeeds.fl", + "wheelSpeeds.fr", + "wheelSpeeds.rl", + "wheelSpeeds.rr", + "vCruise", + "vCruiseCluster", + "leftBlinker", + "rightBlinker", + "doorOpen", + "seatbeltUnlatched", + "espDisabled", + "espActive", + "stockFcw", + "stockAeb", + "stockLkas", + "brakeHoldActive", + "parkingBrake", + "accFaulted", + "vehicleSensorsInvalid", + "invalidLkasSetting", + "lowSpeedAlert", + "buttonEnable", + "steerFaultTemporary", + "steerFaultPermanent", + "blockPcmEnable", + "regenBraking", + "pedalLongActive", + "pedalMaxRegen", + "leftBlindspot", + "rightBlindspot", + "teslaCCEngaged", + "teslaCCDisengaged", + "teslaCCNotArmed", + "pedalAuthorityFailed" + ], + "carControl": [ + "enabled", + "latActive", + "longActive", + "actuators.gas", + "actuators.brake", + "actuators.torque", + "actuators.steeringAngleDeg", + "actuators.accel", + "orientationNED[]", + "cruiseControl.cancel", + "leftBlinker", + "rightBlinker" + ], + "carOutput": [ + "actuatorsOutput.steeringAngleDeg", + "actuatorsOutput.torque", + "actuatorsOutput.accel" + ], + "controlsState": [ + "longControlState", + "forceDecel", + "desiredCurvature", + "curvature", + "longitudinalPlanMonoTime", + "lateralPlanMonoTime", + "lateralControlState.pidState.saturated", + "lateralControlState.pidState.active", + "lateralControlState.angleState.saturated", + "lateralControlState.angleState.active", + "lateralControlState.debugState.saturated", + "lateralControlState.debugState.active", + "lateralControlState.torqueState.saturated", + "lateralControlState.torqueState.active" + ], + "selfdriveState": [ + "enabled", + "active", + "engageable", + "state", + "personality", + "experimentalMode", + "alertHudVisual", + "alertStatus", + "alertSize" + ], + "liveCalibration": [ + "calStatus", + "calCycle", + "calPerc", + "rpyCalib[]", + "height[]", + "wideFromDeviceEuler[]" + ], + "liveParameters": [ + "valid", + "gyroBias", + "angleOffsetDeg", + "angleOffsetAverageDeg", + "stiffnessFactor", + "steerRatio", + "roll", + "posenetValid", + "angleOffsetValid", + "steerRatioValid", + "stiffnessFactorValid" + ], + "liveTorqueParameters": [ + "useParams", + "latAccelFactorFiltered", + "latAccelOffsetFiltered", + "frictionCoefficientFiltered" + ], + "liveDelay": [ + "lateralDelay" + ], + "longitudinalPlan": [ + "aTarget", + "shouldStop", + "hasLead", + "fcw", + "modelMonoTime" + ], + "radarState": [ + "mdMonoTime", + "carStateMonoTime", + "leadOne.status", + "leadOne.dRel", + "leadOne.vLead", + "leadOne.aLeadK", + "leadOne.aLeadTau", + "leadOne.modelProb", + "leadTwo.status", + "leadTwo.dRel", + "leadTwo.vLead", + "leadTwo.aLeadK", + "leadTwo.aLeadTau", + "radarErrors.canError", + "radarErrors.radarFault", + "radarErrors.wrongConfig", + "radarErrors.radarUnavailableTemporary" + ], + "liveTracks": [ + "points[].trackId", + "points[].dRel", + "points[].yRel", + "points[].vRel", + "points[].measured", + "errors.canError", + "errors.radarFault", + "errors.wrongConfig", + "errors.radarUnavailableTemporary" + ], + "modelV2": [ + "frameId", + "frameDropPerc", + "modelExecutionTime", + "position.x[]", + "velocity.x[]", + "acceleration.x[]", + "orientation.x[]", + "orientation.y[]", + "orientation.z[]", + "meta.laneChangeState", + "meta.laneChangeDirection", + "meta.hardBrakePredicted", + "meta.desirePrediction[]", + "meta.disengagePredictions.gasPressProbs[]", + "action.desiredCurvature", + "action.desiredAcceleration", + "action.shouldStop", + "laneLineProbs[]", + "laneLines[].y[]", + "leadsV3[].prob", + "leadsV3[].x[]", + "leadsV3[].xStd[]", + "leadsV3[].y[]", + "leadsV3[].yStd[]", + "leadsV3[].v[]", + "leadsV3[].vStd[]", + "leadsV3[].a[]" + ], + "driverMonitoringState": [ + "awarenessStatus", + "isActiveMode", + "isRHD", + "events[].name", + "events[].enable", + "events[].noEntry", + "events[].warning", + "events[].userDisable", + "events[].softDisable", + "events[].immediateDisable", + "events[].preEnable", + "events[].permanent", + "events[].overrideLongitudinal", + "events[].overrideLateral" + ], + "driverAssistance": [ + "leftLaneDeparture", + "rightLaneDeparture" + ], + "livePose": [ + "posenetOK", + "inputsOK", + "sensorsOK", + "orientationNED.x", + "orientationNED.y", + "orientationNED.z", + "orientationNED.xStd", + "orientationNED.yStd", + "orientationNED.zStd", + "velocityDevice.x", + "velocityDevice.y", + "velocityDevice.z", + "velocityDevice.xStd", + "velocityDevice.yStd", + "velocityDevice.zStd", + "accelerationDevice.x", + "accelerationDevice.y", + "accelerationDevice.z", + "accelerationDevice.xStd", + "accelerationDevice.yStd", + "accelerationDevice.zStd", + "angularVelocityDevice.x", + "angularVelocityDevice.y", + "angularVelocityDevice.z", + "angularVelocityDevice.xStd", + "angularVelocityDevice.yStd", + "angularVelocityDevice.zStd", + "angularVelocityDevice.valid" + ], + "onroadEvents": [ + "[].name", + "[].enable", + "[].noEntry", + "[].warning", + "[].userDisable", + "[].softDisable", + "[].immediateDisable", + "[].preEnable", + "[].permanent", + "[].overrideLongitudinal", + "[].overrideLateral" + ], + "pandaStates": [ + "[].pandaType", + "[].ignitionLine", + "[].controlsAllowed", + "[].faultStatus", + "[].safetyModel", + "[].safetyParam", + "[].alternativeExperience", + "[].faults[]", + "[].safetyRxChecksInvalid" + ], + "peripheralState": [ + "pandaType", + "voltage", + "current", + "fanSpeedRpm" + ], + "deviceState": [ + "freeSpacePercent", + "memoryUsagePercent", + "cpuUsagePercent[]", + "fanSpeedPercentDesired", + "started", + "thermalStatus", + "cpuTempC[]", + "gpuTempC[]", + "memoryTempC", + "maxTempC" + ], + "gpsLocation": [], + "accelerometer": [], + "gyroscope": [] + }, + "forbidden_services": [ + "clocks", + "initData", + "boot", + "sentinel", + "roadCameraState", + "driverCameraState", + "wideRoadCameraState", + "logMessage", + "errorLogMessage", + "androidLog", + "procLog", + "roadEncodeIdx", + "driverEncodeIdx", + "wideRoadEncodeIdx", + "qRoadEncodeIdx", + "livestreamRoadEncodeIdx", + "livestreamWideRoadEncodeIdx", + "livestreamDriverEncodeIdx", + "roadEncodeData", + "driverEncodeData", + "wideRoadEncodeData", + "qRoadEncodeData", + "livestreamRoadEncodeData", + "livestreamWideRoadEncodeData", + "livestreamDriverEncodeData", + "touch", + "qcomGnss", + "ubloxRaw", + "ubloxGnss", + "gnssMeasurements", + "gpsNMEA", + "uiDebug", + "navInstruction", + "navRoute", + "navThumbnail", + "mapRenderState", + "thumbnail", + "managerState", + "uploaderState", + "rawAudioData", + "soundPressure", + "audioFeedback", + "sendcan", + "alertDebug", + "userBookmark", + "bookmarkButton", + "gpsLocationExternal" + ], + "required_services": { + "can": 1, + "carParams": 1, + "carState": 1, + "carControl": 1, + "carOutput": 1, + "controlsState": 1, + "selfdriveState": 1, + "modelV2": 1, + "livePose": 1, + "liveCalibration": 1, + "liveParameters": 1, + "liveTorqueParameters": 1, + "liveDelay": 1, + "liveTracks": 1, + "radarState": 1, + "longitudinalPlan": 1, + "pandaStates": 1, + "peripheralState": 1, + "deviceState": 1, + "onroadEvents": 1, + "driverMonitoringState": 1, + "driverAssistance": 1, + "accelerometer": 1, + "gyroscope": 1, + "gpsLocation": 1 + }, + "selected_gps_service": "gpsLocation", + "freshness_services": [ + "accelerometer", + "gyroscope", + "gpsLocation" + ], + "pedal": { + "address": 1362, + "bus": 2, + "data_length": 6, + "counter_index": 4, + "counter_mask": 15, + "checksum_index": 5, + "checksum": "tesla_sum8_address", + "minimum_frames": 10, + "minimum_counters": 8 + }, + "sensitive_zero_fields": [ + "gpsLocation.latitude", + "gpsLocation.longitude", + "gpsLocation.altitude", + "gpsLocation.bearingDeg", + "gpsLocation.unixTimestampMillis", + "gpsLocation.vNED[]" + ], + "placeholder_vin": "00000000000000000", + "cases": { + "nap-preap-pedal-v1": { + "mode": "pedal", + "fingerprint": "TESLA_MODEL_S_PREAP", + "safety_param": 7 + }, + "nap-preap-no-pedal-v1": { + "mode": "no-pedal", + "fingerprint": "TESLA_MODEL_S_PREAP", + "safety_param": 6 + } + } +} diff --git a/selfdrive/test/process_replay/test_compare_logs.py b/selfdrive/test/process_replay/test_compare_logs.py new file mode 100644 index 00000000000000..4c2daa94a5f422 --- /dev/null +++ b/selfdrive/test/process_replay/test_compare_logs.py @@ -0,0 +1,46 @@ +from cereal import messaging + +from openpilot.selfdrive.test.process_replay.compare_logs import _diff_capnp_values + + +PATH = ("modelV2", "position", "x") + + +def _position_x_reader(values): + message = messaging.new_message("modelV2") + message.modelV2.position.x = values + return message.as_reader().modelV2.position.x + + +def test_equal_position_x_lists_have_no_diff(): + first = _position_x_reader([1, 2]) + second = _position_x_reader([1, 2]) + + assert list(_diff_capnp_values(first, second, PATH, 0)) == [] + + +def test_changed_position_x_element_reports_indexed_change(): + first = _position_x_reader([1, 2]) + second = _position_x_reader([1, 9]) + + assert list(_diff_capnp_values(first, second, PATH, 0)) == [ + ("change", "modelV2.position.x.1", (2, 9)), + ] + + +def test_grown_position_x_list_reports_indexed_additions(): + first = _position_x_reader([1, 2]) + second = _position_x_reader([1, 2, 3, 4]) + + assert list(_diff_capnp_values(first, second, PATH, 0)) == [ + ("add", "modelV2.position.x", [(2, 3), (3, 4)]), + ] + + +def test_shrunken_position_x_list_reports_reverse_indexed_removals(): + first = _position_x_reader([1, 2, 3, 4]) + second = _position_x_reader([1, 2]) + + assert list(_diff_capnp_values(first, second, PATH, 0)) == [ + ("remove", "modelV2.position.x", [(3, 4), (2, 3)]), + ] diff --git a/selfdrive/test/process_replay/test_fixture_tool.py b/selfdrive/test/process_replay/test_fixture_tool.py new file mode 100644 index 00000000000000..cf1bb77e527386 --- /dev/null +++ b/selfdrive/test/process_replay/test_fixture_tool.py @@ -0,0 +1,977 @@ +from __future__ import annotations + +from typing import Any +import hashlib +import json +import os +from pathlib import Path + +import pytest +import zstandard +from cereal import car, log as capnp_log +from openpilot.tools.lib.logreader import LogReader, save_log + +from openpilot.selfdrive.test.process_replay import fixture_tool + + +POLICY = Path(__file__).with_name("nap_fixture_policy.json") +PRIVATE_VIN = "1HGBH41JXMN109186" + + +def gas_sensor(counter: int) -> tuple[int, bytes, int]: + data = bytearray((1, 0xD6, 0, 0xFA, counter & 0x0F, 0)) + data[5] = (sum(data[:5]) + 0x52 + 0x05) & 0xFF + return 0x552, bytes(data), 2 + + +def _set_xyz(msg, x=0.0, y=0.0, z=0.0): + msg.x, msg.y, msg.z = x, y, z + msg.xStd = msg.yStd = msg.zStd = 0.0 + + +def make_car_params(*, pedal: bool = True, vin: str = PRIVATE_VIN, fw_secret: str | bytes | None = None) -> Any: + event = capnp_log.Event.new_message() + event.valid = True + event.logMonoTime = 1 + cp = event.init("carParams") + cp.brand = "tesla" + cp.carFingerprint = "TESLA_MODEL_S_PREAP" + cp.carVin = vin + cp.pcmCruise = not pedal + cp.openpilotLongitudinalControl = pedal + if fw_secret is not None: + fw = cp.init("carFw", 1)[0] + fw.ecu = car.CarParams.Ecu.unknown + fw.fwVersion = fw_secret if isinstance(fw_secret, (bytes, bytearray)) else fw_secret.encode() + fw.address = 0x7E0 + safety = cp.init("safetyConfigs", 1)[0] + safety.safetyModel = car.CarParams.SafetyModel.teslaPreap + safety.safetyParam = 7 if pedal else 6 + return event + + +def compact_stream(*, pedal: bool = True, private_vin: str = PRIVATE_VIN, + embed_private_text: bool = False, embed_private_data: bool = False, + split_vin_frames: bool = False, split_vin_events: bool = False, + interleaved_vin_frames: bool = False, undeclared_field: bool = False, + nonzero_vned: bool = False, wrong_safety_model: bool = False, + include_forbidden: bool = False, invalid_model: bool = False, + fw_secret: str | bytes | None = None, + duplicate_fw_in_can: bool = False) -> bytes: + events = [] + mono = 10 + + def stamp(event): + nonlocal mono + event.valid = True + event.logMonoTime = mono + mono += 10 + return event + + params = make_car_params(pedal=pedal, vin=private_vin, fw_secret=fw_secret) + if wrong_safety_model: + params.carParams.safetyConfigs[0].safetyModel = car.CarParams.SafetyModel.tesla + events.append(stamp(params)) + + can = capnp_log.Event.new_message() + frame_count = 10 if pedal else 1 + extras = 0 + if split_vin_frames: + extras += 2 + if interleaved_vin_frames: + extras += 3 + if embed_private_data: + extras += 1 + if duplicate_fw_in_can and fw_secret is not None: + extras += 1 + frames = can.init("can", frame_count + extras) + if pedal: + for index in range(10): + address, data, bus = gas_sensor(index) + frames[index].address, frames[index].dat, frames[index].src = address, data, bus + else: + frames[0].address, frames[0].dat, frames[0].src = 0x100, b"\x01\x02\x03\x04", 0 + cursor = frame_count + if split_vin_frames: + frames[cursor].address, frames[cursor].dat, frames[cursor].src = 0x100, private_vin[:3].encode(), 0 + frames[cursor + 1].address, frames[cursor + 1].dat, frames[cursor + 1].src = 0x101, private_vin[3:6].encode(), 0 + cursor += 2 + if interleaved_vin_frames: + frames[cursor].address, frames[cursor].dat, frames[cursor].src = 0x100, private_vin[:3].encode(), 0 + frames[cursor + 1].address, frames[cursor + 1].dat, frames[cursor + 1].src = 0x110, b"XX", 0 + frames[cursor + 2].address, frames[cursor + 2].dat, frames[cursor + 2].src = 0x101, private_vin[3:6].encode(), 0 + cursor += 3 + if embed_private_data: + frames[cursor].address, frames[cursor].dat, frames[cursor].src = 0x123, b"SECRETDATA_TOKEN", 0 + cursor += 1 + if duplicate_fw_in_can and fw_secret is not None: + fw_bytes = fw_secret if isinstance(fw_secret, (bytes, bytearray)) else fw_secret.encode() + frames[cursor].address, frames[cursor].dat, frames[cursor].src = 0x321, bytes(fw_bytes), 0 + cursor += 1 + events.append(stamp(can)) + + if split_vin_events: + first = capnp_log.Event.new_message() + f = first.init("can", 1)[0] + f.address, f.dat, f.src = 0x200, private_vin[:3].encode(), 1 + events.append(stamp(first)) + second = capnp_log.Event.new_message() + f = second.init("can", 1)[0] + f.address, f.dat, f.src = 0x201, private_vin[3:6].encode(), 1 + events.append(stamp(second)) + + # required services + cs = stamp(capnp_log.Event.new_message()) + state = cs.init("carState") + state.vEgo = 20.0 + state.aEgo = 0.1 + state.vCruise = 30.0 + state.standstill = False + state.steeringAngleDeg = 1.0 + state.canValid = True + state.cruiseState.enabled = True + state.leftBlindspot = True + state.rightBlindspot = False + state.teslaCCEngaged = False + state.teslaCCDisengaged = False + state.teslaCCNotArmed = False + state.pedalAuthorityFailed = False + events.append(cs) + + cc = stamp(capnp_log.Event.new_message()) + control = cc.init("carControl") + control.enabled = True + control.latActive = True + control.longActive = pedal + control.actuators.steeringAngleDeg = 1.0 + control.actuators.accel = 0.2 if pedal else 0.0 + control.orientationNED = [0.0, 0.01, 0.0] + control.leftBlinker = True + control.rightBlinker = False + events.append(cc) + + co = stamp(capnp_log.Event.new_message()) + output = co.init("carOutput") + output.actuatorsOutput.steeringAngleDeg = 1.0 + output.actuatorsOutput.torque = 0.1 + events.append(co) + + controls = stamp(capnp_log.Event.new_message()) + cst = controls.init("controlsState") + cst.desiredCurvature = 0.001 + cst.forceDecel = False + pid = cst.lateralControlState.init("pidState") + pid.saturated = False + pid.active = True + events.append(controls) + + sds = stamp(capnp_log.Event.new_message()) + ss = sds.init("selfdriveState") + ss.enabled = True + ss.active = True + ss.engageable = True + ss.state = capnp_log.SelfdriveState.OpenpilotState.enabled + if embed_private_text: + ss.alertText1 = "leak-token-value" + events.append(sds) + + model = stamp(capnp_log.Event.new_message()) + if invalid_model: + model.valid = False + mv = model.init("modelV2") + mv.frameId = 1 + mv.frameDropPerc = 0.0 + mv.position.x = [1.0, 2.0] + mv.velocity.x = [20.0, 20.0] + mv.acceleration.x = [0.1, 0.1] + mv.laneLineProbs = [0.9, 0.8, 0.7, 0.6] + lines = mv.init("laneLines", 4) + for line in lines: + line.y = [0.1, 0.2] + leads = mv.init("leadsV3", 2) + for lead in leads: + lead.prob = 0.5 + lead.x = [10.0] + lead.xStd = [1.0] + lead.y = [0.0] + lead.yStd = [0.1] + lead.v = [15.0] + lead.vStd = [0.5] + lead.a = [0.0] + mv.meta.laneChangeState = 0 + mv.meta.hardBrakePredicted = False + mv.meta.desirePrediction = [0.0] * 8 + mv.meta.disengagePredictions.gasPressProbs = [0.0, 0.01] + mv.action.desiredCurvature = 0.001 + mv.action.desiredAcceleration = 0.1 + mv.action.shouldStop = False + events.append(model) + + pose = stamp(capnp_log.Event.new_message()) + lp = pose.init("livePose") + lp.posenetOK = True + lp.inputsOK = True + _set_xyz(lp.orientationNED) + _set_xyz(lp.velocityDevice, x=20.0) + _set_xyz(lp.accelerationDevice) + _set_xyz(lp.angularVelocityDevice) + lp.angularVelocityDevice.valid = True + events.append(pose) + + calib = stamp(capnp_log.Event.new_message()) + lc = calib.init("liveCalibration") + lc.calStatus = capnp_log.LiveCalibrationData.Status.calibrated + lc.rpyCalib = [0.0, 0.0, 0.0] + events.append(calib) + + live_params = stamp(capnp_log.Event.new_message()) + lpd = live_params.init("liveParameters") + lpd.valid = True + lpd.angleOffsetDeg = 0.1 + lpd.steerRatio = 15.0 + lpd.stiffnessFactor = 1.0 + lpd.roll = 0.0 + events.append(live_params) + + torque = stamp(capnp_log.Event.new_message()) + lt = torque.init("liveTorqueParameters") + lt.useParams = False + events.append(torque) + + delay = stamp(capnp_log.Event.new_message()) + delay.init("liveDelay").lateralDelay = 0.05 + events.append(delay) + + tracks = stamp(capnp_log.Event.new_message()) + ltks = tracks.init("liveTracks") + pts = ltks.init("points", 1) + pts[0].trackId = 1 + pts[0].dRel = 20.0 + pts[0].yRel = 0.0 + pts[0].vRel = -1.0 + pts[0].measured = True + events.append(tracks) + + radar = stamp(capnp_log.Event.new_message()) + rs = radar.init("radarState") + rs.leadOne.status = True + rs.leadOne.dRel = 20.0 + rs.leadOne.vLead = 19.0 + rs.leadOne.aLeadK = 0.0 + rs.leadOne.aLeadTau = 1.5 + rs.leadOne.modelProb = 0.8 + rs.leadTwo.status = False + events.append(radar) + + plan = stamp(capnp_log.Event.new_message()) + lpplan = plan.init("longitudinalPlan") + lpplan.aTarget = 0.1 + lpplan.shouldStop = False + lpplan.hasLead = True + events.append(plan) + + panda = stamp(capnp_log.Event.new_message()) + ps = panda.init("pandaStates", 1)[0] + ps.pandaType = capnp_log.PandaState.PandaType.dos + ps.controlsAllowed = True + ps.safetyModel = car.CarParams.SafetyModel.teslaPreap + ps.safetyParam = 7 if pedal else 6 + ps.safetyRxChecksInvalid = False + events.append(panda) + + peri = stamp(capnp_log.Event.new_message()) + peri.init("peripheralState").pandaType = capnp_log.PandaState.PandaType.dos + events.append(peri) + + device = stamp(capnp_log.Event.new_message()) + ds = device.init("deviceState") + ds.freeSpacePercent = 50.0 + ds.memoryUsagePercent = 20 + ds.started = True + if undeclared_field: + ds.networkInfo.operator = "SECRET-OP" + events.append(device) + + onroad = stamp(capnp_log.Event.new_message()) + ev = onroad.init("onroadEvents", 1)[0] + ev.name = capnp_log.OnroadEvent.EventName.buttonEnable + events.append(onroad) + + dms = stamp(capnp_log.Event.new_message()) + dms.init("driverMonitoringState").awarenessStatus = 1.0 + events.append(dms) + + assist = stamp(capnp_log.Event.new_message()) + assist.init("driverAssistance") + events.append(assist) + + acc = stamp(capnp_log.Event.new_message()) + acc.init("accelerometer") + events.append(acc) + + gyro = stamp(capnp_log.Event.new_message()) + gyro.init("gyroscope") + events.append(gyro) + + gps = stamp(capnp_log.Event.new_message()) + loc = gps.init("gpsLocation") + if nonzero_vned: + loc.vNED = [1.0, 2.0, 3.0] + events.append(gps) + + if include_forbidden: + forbidden = stamp(capnp_log.Event.new_message()) + forbidden.init("initData") + events.append(forbidden) + + return b"".join(event.to_bytes() for event in events) + + +# fix typing import + +def read_messages(raw: bytes): + return list(LogReader.from_bytes(raw)) + + +def deny_token(label: str, value: str | bytes, encoding: str = "utf8") -> dict: + if encoding == "utf8": + encoded = value if isinstance(value, str) else value.decode("utf-8") + elif encoding == "hex": + raw_bytes = value if isinstance(value, (bytes, bytearray)) else value.encode("utf-8") + encoded = bytes(raw_bytes).hex() + elif encoding == "base64": + import base64 + raw_bytes = value if isinstance(value, (bytes, bytearray)) else value.encode("utf-8") + encoded = base64.b64encode(bytes(raw_bytes)).decode("ascii") + else: + raise AssertionError(encoding) + return {"label": label, "encoding": encoding, "value": encoded} + + +def write_deny(path: Path, raw: bytes, tokens: list[dict] | None = None, sha: str | None = None, + sanitized_output_sha256: str | None = None) -> Path: + payload = { + "raw_input_sha256": sha if sha is not None else hashlib.sha256(raw).hexdigest(), + "tokens": tokens if tokens is not None else [deny_token("vin", PRIVATE_VIN)], + } + if sanitized_output_sha256 is not None: + payload["sanitized_output_sha256"] = sanitized_output_sha256 + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def run_sanitize(tmp_path: Path, raw: bytes, deny: dict | None = None, case: str = "nap-preap-pedal-v1"): + source = tmp_path / "source.bin" + output = tmp_path / "output.zst" + report = tmp_path / "report.json" + source.write_bytes(raw) + deny_path = None + if deny is None: + deny_path = write_deny(tmp_path / "deny.json", raw) + else: + deny_path = tmp_path / "deny.json" + deny_path.write_text(json.dumps(deny), encoding="utf-8") + status = fixture_tool.process(str(source), str(output), str(report), str(POLICY), case, str(deny_path), True) + return output.read_bytes() if output.exists() else None, json.loads(report.read_text()), status + + +def test_sanitize_is_byte_deterministic_and_scrubs_vin_and_fw(tmp_path: Path): + raw = compact_stream() + first, report, status = run_sanitize(tmp_path, raw) + second, _, _ = run_sanitize(tmp_path, raw) + assert first == second + assert status["scope"] == "private" + assert status["private_token_scan"]["performed"] is True + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(first))) + params = next(message.carParams for message in messages if message.which() == "carParams") + assert params.carVin == "00000000000000000" + assert len(params.carFw) == 0 + assert fixture_tool.safety_model_ordinal(params.safetyConfigs[0].safetyModel) == int(car.CarParams.SafetyModel.teslaPreap) + assert report["can_evidence"]["valid_count"] == 10 + assert "token_matches" not in status + assert "raw_input_sha256" not in status + assert "raw_input_sha256" not in report + assert report["sanitized_output_sha256"] == hashlib.sha256(first).hexdigest() + assert params.safetyConfigs[0].safetyParam == 7 + + +def test_nested_list_fields_are_copied(tmp_path: Path): + raw = compact_stream() + output, _, _ = run_sanitize(tmp_path, raw) + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + model = next(message.modelV2 for message in messages if message.which() == "modelV2") + assert list(model.velocity.x) == [20.0, 20.0] + assert len(model.leadsV3) == 2 + assert list(model.leadsV3[0].x) == [10.0] + tracks = next(message.liveTracks for message in messages if message.which() == "liveTracks") + assert tracks.points[0].dRel == 20.0 + + +def test_required_service_rejection(tmp_path: Path): + messages = [event for event in read_messages(compact_stream()) if event.which() != "modelV2"] + with pytest.raises(fixture_tool.FixtureError, match="required valid service"): + fixture_tool.structural_checks(messages, fixture_tool.parse_policy(str(POLICY)), "nap-preap-pedal-v1") + + +def test_sanitize_skips_forbidden_and_unknown_source_services(tmp_path: Path): + raw = compact_stream(include_forbidden=True) + # also append an undeclared-but-not-forbidden service via raw event if possible: thumbnail is forbidden + output, report, _ = run_sanitize(tmp_path, raw) + assert output is not None + assert "initData" not in report["message_counts"] + assert "initData" in report["source_message_counts"] + + +@pytest.mark.parametrize( + "mutate", + ( + lambda p: p.__setitem__("schema_version", 99), + lambda p: p.__setitem__("placeholder_vin", "11111111111111111"), + lambda p: p["allowed_services"].__setitem__("sendcan", ["[].address"]), + lambda p: p["forbidden_services"].remove("clocks"), + lambda p: p["cases"].__setitem__("nap-preap-pedal-v1", {"mode": "weird", "fingerprint": "TESLA_MODEL_S_PREAP"}), + lambda p: p.__setitem__("extra", 1), + ), +) +def test_invalid_policy_classes_are_rejected(tmp_path: Path, mutate): + policy = json.loads(POLICY.read_text()) + mutate(policy) + path = tmp_path / "bad_policy.json" + path.write_text(json.dumps(policy), encoding="utf-8") + with pytest.raises(fixture_tool.FixtureError): + fixture_tool.parse_policy(str(path)) + + +def test_undeclared_service_rejected_on_validate(tmp_path: Path): + raw = compact_stream() + output, _, _ = run_sanitize(tmp_path, raw) + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + unexpected = capnp_log.Event.new_message() + unexpected.valid = True + unexpected.init("sendcan", 1) + path = tmp_path / "tampered.zst" + save_log(str(path), messages + [unexpected.as_reader()]) + report = tmp_path / "validate.json" + with pytest.raises(fixture_tool.FixtureError, match="undeclared service"): + fixture_tool.process(str(path), None, str(report), str(POLICY), "nap-preap-pedal-v1", None, False) + + +def test_undeclared_field_rejected(tmp_path: Path): + raw = compact_stream(undeclared_field=True) + # sanitize strips networkInfo because not allowed; validate against unsanitized tampered output + messages = read_messages(raw) + # force a sanitized-looking stream that still has networkInfo by manually building + policy = fixture_tool.parse_policy(str(POLICY)) + built = [fixture_tool.build_event(event, policy) for event in messages] + built = [event for event in built if event is not None] + # inject undeclared field into deviceState after sanitize rebuild + events = [] + for event in built: + if event.which() != "deviceState": + events.append(event) + continue + msg = event.as_builder() + msg.deviceState.networkInfo.operator = "SECRET-OP" + events.append(msg.as_reader()) + path = tmp_path / "field.zst" + save_log(str(path), events) + with pytest.raises(fixture_tool.FixtureError, match="undeclared populated field"): + fixture_tool.process(str(path), None, str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", None, False) + + +def test_nonzero_vned_rejected(tmp_path: Path): + raw = compact_stream(nonzero_vned=True) + policy = fixture_tool.parse_policy(str(POLICY)) + messages = [] + for event in read_messages(raw): + built = fixture_tool.build_event(event, policy) + if built is None: + continue + if built.which() == "gpsLocation": + msg = built.as_builder() + msg.gpsLocation.vNED = [1.0, 0.0, 0.0] + messages.append(msg.as_reader()) + else: + messages.append(built) + path = tmp_path / "vned.zst" + save_log(str(path), messages) + with pytest.raises(fixture_tool.FixtureError, match="sensitive field is not zero"): + fixture_tool.process(str(path), None, str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", None, False) + + +def test_exact_safety_model_required(tmp_path: Path): + output, _, _ = run_sanitize(tmp_path, compact_stream()) + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + rebuilt = [] + for event in messages: + if event.which() != "carParams": + rebuilt.append(event) + continue + msg = event.as_builder() + msg.carParams.safetyConfigs[0].safetyModel = car.CarParams.SafetyModel.tesla + rebuilt.append(msg.as_reader()) + with pytest.raises(fixture_tool.FixtureError, match="safetyModel"): + fixture_tool.structural_checks(rebuilt, fixture_tool.parse_policy(str(POLICY)), "nap-preap-pedal-v1") + + +def test_structural_checks_require_tesla_brand(): + messages = read_messages(compact_stream()) + rebuilt = [] + for event in messages: + if event.which() != "carParams": + rebuilt.append(event) + continue + msg = event.as_builder() + msg.carParams.carVin = "00000000000000000" + msg.carParams.carFw = [] + msg.carParams.brand = "private-brand" + rebuilt.append(msg.as_reader()) + + with pytest.raises(fixture_tool.FixtureError, match=r"carParams\.brand must be tesla"): + fixture_tool.structural_checks(rebuilt, fixture_tool.parse_policy(str(POLICY)), "nap-preap-pedal-v1") + +def test_text_token_outside_carparams_is_rejected(tmp_path: Path): + raw = compact_stream() + deny = { + "raw_input_sha256": hashlib.sha256(raw).hexdigest(), + "tokens": [ + deny_token("vin", PRIVATE_VIN), + deny_token("retained_public_text", "TESLA_MODEL_S_PREAP"), + ], + } + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, raw, deny) + # ensure no final output + output = tmp_path / "output.zst" + # run_sanitize may have left nothing; check report exists with matches + report = json.loads((tmp_path / "report.json").read_text()) + assert report["private_token_scan"]["performed"] is True + assert report["private_token_scan"]["token_matches"] + assert not output.exists() + + +def test_data_token_is_rejected(tmp_path: Path): + raw = compact_stream(embed_private_data=True) + deny = {"raw_input_sha256": hashlib.sha256(raw).hexdigest(), + "tokens": [deny_token("vin", PRIVATE_VIN), deny_token("secret", "SECRETDATA_TOKEN")]} + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, raw, deny) + assert not (tmp_path / "output.zst").exists() + report = json.loads((tmp_path / "report.json").read_text()) + assert any(row["label"] == "secret" for row in report["private_token_scan"]["token_matches"]) + + +def test_split_vin_across_frames_is_rejected(tmp_path: Path): + raw = compact_stream(split_vin_frames=True) + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, raw) + assert not (tmp_path / "output.zst").exists() + + +def test_split_vin_across_events_is_rejected(tmp_path: Path): + raw = compact_stream(split_vin_events=True) + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, raw) + assert not (tmp_path / "output.zst").exists() + + +def test_missing_empty_wrong_hash_deny(tmp_path: Path): + raw = compact_stream() + source = tmp_path / "source.bin" + source.write_bytes(raw) + output = tmp_path / "out.zst" + report = tmp_path / "report.json" + with pytest.raises(fixture_tool.FixtureError, match="deny"): + fixture_tool.process(str(source), str(output), str(report), str(POLICY), "nap-preap-pedal-v1", None, True) + empty = tmp_path / "empty.json" + empty.write_text(json.dumps({"raw_input_sha256": hashlib.sha256(raw).hexdigest(), "tokens": []}), encoding="utf-8") + with pytest.raises(fixture_tool.FixtureError, match="nonempty"): + fixture_tool.process(str(source), str(output), str(report), str(POLICY), "nap-preap-pedal-v1", str(empty), True) + wrong = write_deny(tmp_path / "wrong.json", raw, sha="0" * 64) + with pytest.raises(fixture_tool.FixtureError, match="SHA-256"): + fixture_tool.process(str(source), str(output), str(report), str(POLICY), "nap-preap-pedal-v1", str(wrong), True) + # conflicting --input-sha256 must not override deny + deny = write_deny(tmp_path / "deny.json", raw) + with pytest.raises(fixture_tool.FixtureError, match="conflicts"): + fixture_tool.process(str(source), str(output), str(report), str(POLICY), "nap-preap-pedal-v1", str(deny), True, "1" * 64) + + +def test_public_and_private_validate_scopes(tmp_path: Path): + raw = compact_stream() + output, private_report, _ = run_sanitize(tmp_path, raw) + public_report = tmp_path / "public.json" + status = fixture_tool.process(str(tmp_path / "output.zst"), None, str(public_report), str(POLICY), "nap-preap-pedal-v1", None, False) + assert status["scope"] == "public" + assert status["private_token_scan"]["performed"] is False + loaded = json.loads(public_report.read_text()) + assert loaded["scope"] == "public" + assert loaded["private_token_scan"]["performed"] is False + assert "token_matches" not in loaded + assert "token_matches" not in loaded.get("private_token_scan", {}) + + sanitized_deny = { + "raw_input_sha256": hashlib.sha256(raw).hexdigest(), + "sanitized_output_sha256": hashlib.sha256(output).hexdigest(), + "tokens": [deny_token("vin", PRIVATE_VIN)], + } + deny_path = tmp_path / "san_deny.json" + deny_path.write_text(json.dumps(sanitized_deny), encoding="utf-8") + private_path = tmp_path / "private.json" + status = fixture_tool.process(str(tmp_path / "output.zst"), None, str(private_path), str(POLICY), "nap-preap-pedal-v1", str(deny_path), False) + assert status["scope"] == "private" + assert status["private_token_scan"]["performed"] is True + loaded = json.loads(private_path.read_text()) + assert loaded["private_token_scan"]["performed"] is True + assert loaded["private_token_scan"]["token_matches"] + assert all(row["count"] == 0 for row in loaded["private_token_scan"]["token_matches"]) + + +def test_symlink_hardlink_and_same_path_collisions(tmp_path: Path): + raw = compact_stream() + source = tmp_path / "source.bin" + source.write_bytes(raw) + deny = write_deny(tmp_path / "deny.json", raw) + report = tmp_path / "report.json" + # same path output/report + with pytest.raises(fixture_tool.FixtureError, match="distinct|collision"): + fixture_tool.process(str(source), str(report), str(report), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + # symlink output + link = tmp_path / "link.zst" + target = tmp_path / "target.zst" + target.write_bytes(b"x") + link.symlink_to(target) + with pytest.raises(fixture_tool.FixtureError, match="symlink|alias|collision"): + fixture_tool.process(str(source), str(link), str(report), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + # hardlink alias between output and report destinations that already exist as same inode + hard_a = tmp_path / "hard_a.json" + hard_b = tmp_path / "hard_b.json" + hard_a.write_text("{}", encoding="utf-8") + os.link(hard_a, hard_b) + with pytest.raises(fixture_tool.FixtureError, match="inode|alias|collision"): + fixture_tool.process(str(source), str(hard_a), str(hard_b), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + + +def test_cli_malformed_input_returns_2(tmp_path: Path): + bad = tmp_path / "bad.bin" + bad.write_bytes(b"not-a-log") + deny = write_deny(tmp_path / "deny.json", b"not-a-log", tokens=[deny_token("tokenx", "abcdef")]) + code = fixture_tool.main([ + "sanitize", "--policy", str(POLICY), "--case", "nap-preap-pedal-v1", + "--input", str(bad), "--output", str(tmp_path / "out.zst"), "--report", str(tmp_path / "r.json"), + "--deny-token-file", str(deny), + ]) + assert code == 2 + + +def test_no_pedal_rejects_sensor_frames(tmp_path: Path): + # Build a no-pedal fixture then inject pedal sensor frames. + output, _, _ = run_sanitize(tmp_path, compact_stream(pedal=False, private_vin=PRIVATE_VIN), case="nap-preap-no-pedal-v1") + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + can = capnp_log.Event.new_message() + can.valid = True + can.logMonoTime = 999 + frame = can.init("can", 1)[0] + address, data, bus = gas_sensor(0) + frame.address, frame.dat, frame.src = address, data, bus + path = tmp_path / "nopedal_tampered.zst" + save_log(str(path), messages + [can.as_reader()]) + with pytest.raises(fixture_tool.FixtureError, match="no-pedal"): + fixture_tool.process(str(path), None, str(tmp_path / "nop.json"), str(POLICY), "nap-preap-no-pedal-v1", None, False) + + +def test_cli_public_validation_returns_failure_for_tampering(tmp_path: Path): + output, _, _ = run_sanitize(tmp_path, compact_stream()) + path = tmp_path / "fixture.zst" + path.write_bytes(output[:-1] + bytes((output[-1] ^ 1,))) + assert fixture_tool.main([ + "validate", "--policy", str(POLICY), "--case", "nap-preap-pedal-v1", + "--input", str(path), "--report", str(tmp_path / "rep.json"), + ]) != 0 + + +def test_interleaved_vin_across_frames_is_rejected(tmp_path: Path): + raw = compact_stream(interleaved_vin_frames=True) + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, raw) + assert not (tmp_path / "output.zst").exists() + + +def test_deny_requires_exact_source_vin(tmp_path: Path): + raw = compact_stream() + source = tmp_path / "source.bin" + source.write_bytes(raw) + deny = write_deny(tmp_path / "deny.json", raw, tokens=[deny_token("route", "abcdefgh")]) + with pytest.raises(fixture_tool.FixtureError, match="exactly one vin token"): + fixture_tool.process(str(source), str(tmp_path / "output.zst"), str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + + deny = write_deny(tmp_path / "deny2.json", raw, tokens=[deny_token("vin", "1HGBH41JXMN109187")]) + with pytest.raises(fixture_tool.FixtureError, match="does not match source"): + fixture_tool.process(str(source), str(tmp_path / "output2.zst"), str(tmp_path / "report2.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + + +def test_private_validate_reuses_raw_bound_deny(tmp_path: Path): + raw = compact_stream() + output, report, _ = run_sanitize(tmp_path, raw) + assert "raw_input_sha256" not in report + deny = write_deny( + tmp_path / "bound_deny.json", + raw, + sanitized_output_sha256=hashlib.sha256(output).hexdigest(), + ) + status = fixture_tool.process( + str(tmp_path / "output.zst"), None, str(tmp_path / "private.json"), str(POLICY), + "nap-preap-pedal-v1", str(deny), False, + ) + assert status["scope"] == "private" + loaded = json.loads((tmp_path / "private.json").read_text()) + assert "raw_input_sha256" not in loaded + assert all(row["count"] == 0 for row in loaded["private_token_scan"]["token_matches"]) + + +def test_stale_output_removed_on_rejection(tmp_path: Path): + raw = compact_stream() + output, _, _ = run_sanitize(tmp_path, raw) + assert output is not None + out_path = tmp_path / "output.zst" + assert out_path.exists() + # Re-run with VIN leak against the same destination. + with pytest.raises(fixture_tool.FixtureError, match="private token"): + run_sanitize(tmp_path, compact_stream(split_vin_frames=True)) + assert not out_path.exists() + + +def test_required_valid_service_minima(tmp_path: Path): + messages = read_messages(compact_stream(invalid_model=True)) + with pytest.raises(fixture_tool.FixtureError, match="required valid service"): + fixture_tool.structural_checks(messages, fixture_tool.parse_policy(str(POLICY)), "nap-preap-pedal-v1") + + +def test_exact_safety_param_required(tmp_path: Path): + output, _, _ = run_sanitize(tmp_path, compact_stream()) + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + rebuilt = [] + for message in messages: + if message.which() != "carParams": + rebuilt.append(message) + continue + event = message.as_builder() + event.carParams.safetyConfigs[0].safetyParam = 1 + rebuilt.append(event.as_reader()) + with pytest.raises(fixture_tool.FixtureError, match="safetyParam"): + fixture_tool.structural_checks(rebuilt, fixture_tool.parse_policy(str(POLICY)), "nap-preap-pedal-v1") + + +def test_retained_consumer_fields_survive_sanitize(tmp_path: Path): + output, _, _ = run_sanitize(tmp_path, compact_stream()) + messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + state = next(message.carState for message in messages if message.which() == "carState") + assert state.leftBlindspot is True + assert state.pedalAuthorityFailed is False + control = next(message.carControl for message in messages if message.which() == "carControl") + assert control.leftBlinker is True + assert control.rightBlinker is False + panda = next(message.pandaStates for message in messages if message.which() == "pandaStates") + assert panda[0].safetyRxChecksInvalid is False + + +def test_accept_six_process_outputs_helper(tmp_path: Path): + policy = fixture_tool.parse_policy(str(POLICY)) + raw = compact_stream() + output, _, _ = run_sanitize(tmp_path, raw) + fixture_messages = list(LogReader.from_bytes(zstandard.ZstdDecompressor().decompress(output))) + + def with_service(service: str, list_size: int | None = None, **fields): + event = capnp_log.Event.new_message() + event.valid = True + event.logMonoTime = 1 + payload = event.init(service) if list_size is None else event.init(service, list_size) + for key, value in fields.items(): + setattr(payload, key, value) + return event.as_reader() + + # Synthetic per-process outputs using fixture carParams for card mode evidence. + car_params = next(message for message in fixture_messages if message.which() == "carParams") + process_outputs = { + "card": [ + car_params, + next(message for message in fixture_messages if message.which() == "carState"), + next(message for message in fixture_messages if message.which() == "carOutput"), + next(message for message in fixture_messages if message.which() == "liveTracks"), + with_service("sendcan", list_size=1), + ], + "controlsd": [ + next(message for message in fixture_messages if message.which() == "carControl"), + next(message for message in fixture_messages if message.which() == "controlsState"), + ], + "selfdrived": [ + next(message for message in fixture_messages if message.which() == "selfdriveState"), + next(message for message in fixture_messages if message.which() == "onroadEvents"), + ], + "radard": [next(message for message in fixture_messages if message.which() == "radarState")], + "plannerd": [ + next(message for message in fixture_messages if message.which() == "longitudinalPlan"), + next(message for message in fixture_messages if message.which() == "driverAssistance"), + ], + "lagd": [next(message for message in fixture_messages if message.which() == "liveDelay")], + } + case_params = { + "NAPPedalEnabled": True, + "NAPRadarEnabled": True, + "NAPRadarBehindNosecone": True, + } + accepted = fixture_tool.accept_six_process_outputs( + case="nap-preap-pedal-v1", + policy=policy, + process_outputs=process_outputs, + case_params=case_params, + ) + assert accepted["status"] == "ok" + assert accepted["card_mode"] == "pedal" + assert accepted["card_safety_param"] == 7 + assert set(accepted["processes"]) == set(fixture_tool.SIX_PROCESS_OUTPUTS) + + with pytest.raises(fixture_tool.FixtureError, match="missing process outputs"): + fixture_tool.accept_six_process_outputs( + case="nap-preap-pedal-v1", + policy=policy, + process_outputs={k: v for k, v in process_outputs.items() if k != "lagd"}, + case_params=case_params, + ) + + +def test_selfdrived_replay_ignores_stripped_services(): + source = Path(__file__).resolve().parents[2] / "selfdrived" / "selfdrived.py" + text = source.read_text(encoding="utf-8") + assert "driverCameraState" in text + assert "managerState" in text + # REPLAY ignore list must include both stripped fixture services. + assert "ignore += ['roadCameraState', 'wideRoadCameraState', 'driverCameraState', 'managerState']" in text + + +def test_policy_attestation_rejects_weakened_contract(tmp_path: Path): + policy = json.loads(POLICY.read_text(encoding="utf-8")) + policy["pedal"]["minimum_frames"] = 0 + policy["pedal"]["minimum_counters"] = 0 + path = tmp_path / "weak.json" + path.write_text(json.dumps(policy), encoding="utf-8") + with pytest.raises(fixture_tool.FixtureError, match="attested release contract|pedal evidence minima"): + fixture_tool.parse_policy(str(path)) + + policy = json.loads(POLICY.read_text(encoding="utf-8")) + policy["allowed_services"]["carState"] = list(policy["allowed_services"]["carState"]) + ["cruiseState.speedOffline"] + path = tmp_path / "extra_leaf.json" + path.write_text(json.dumps(policy), encoding="utf-8") + with pytest.raises(fixture_tool.FixtureError, match="attested release contract"): + fixture_tool.parse_policy(str(path)) + + +def test_deny_inventory_completeness_requires_scrubbed_secrets(tmp_path: Path): + raw = compact_stream(fw_secret="SECRET_FW_VALUE") + source = tmp_path / "source.bin" + source.write_bytes(raw) + deny = write_deny(tmp_path / "deny.json", raw) # vin only; missing fw secret + with pytest.raises(fixture_tool.FixtureError, match="deny tokens incomplete"): + fixture_tool.process(str(source), str(tmp_path / "output.zst"), str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + + deny = write_deny( + tmp_path / "deny2.json", + raw, + tokens=[deny_token("vin", PRIVATE_VIN), deny_token("fw", "SECRET_FW_VALUE")], + ) + status = fixture_tool.process(str(source), str(tmp_path / "output2.zst"), str(tmp_path / "report2.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + assert status["status"] == "ok" + + +def test_deny_inventory_keeps_fw_when_duplicated_in_retained_can(tmp_path: Path): + # NAP-FIXTURE-001: private fwVersion bytes also present in retained can[].dat must + # still require an explicit deny token; set-subtraction must not exempt them. + secret = "SECRET_FW_VALUE" + raw = compact_stream(fw_secret=secret, duplicate_fw_in_can=True) + source = tmp_path / "source.bin" + source.write_bytes(raw) + out = tmp_path / "output.zst" + deny = write_deny(tmp_path / "deny.json", raw) # vin only + with pytest.raises(fixture_tool.FixtureError, match="deny tokens incomplete"): + fixture_tool.process(str(source), str(out), str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + assert not out.exists() + + deny = write_deny( + tmp_path / "deny2.json", + raw, + tokens=[deny_token("vin", PRIVATE_VIN), deny_token("fw", secret)], + ) + # Inventory complete, but retained can[].dat still carries the secret — scan rejects. + with pytest.raises(fixture_tool.FixtureError, match="private token"): + fixture_tool.process(str(source), str(tmp_path / "output2.zst"), str(tmp_path / "report2.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + assert not (tmp_path / "output2.zst").exists() + + # Explicit fw deny succeeds when the secret is scrubbed and not retained in CAN. + clean = compact_stream(fw_secret=secret, duplicate_fw_in_can=False) + clean_source = tmp_path / "clean.bin" + clean_source.write_bytes(clean) + deny = write_deny( + tmp_path / "deny3.json", + clean, + tokens=[deny_token("vin", PRIVATE_VIN), deny_token("fw", secret)], + ) + status = fixture_tool.process( + str(clean_source), str(tmp_path / "output3.zst"), str(tmp_path / "report3.json"), + str(POLICY), "nap-preap-pedal-v1", str(deny), True, + ) + assert status["status"] == "ok" + assert (tmp_path / "output3.zst").exists() + + +def test_binary_deny_token_hex_covers_non_utf8_fw(tmp_path: Path): + secret = b"BIN\xffFW\x00SECRETS" + raw = compact_stream(fw_secret=secret) + source = tmp_path / "source.bin" + source.write_bytes(raw) + deny = write_deny( + tmp_path / "deny.json", + raw, + tokens=[deny_token("vin", PRIVATE_VIN), deny_token("fw", secret, encoding="hex")], + ) + status = fixture_tool.process(str(source), str(tmp_path / "output.zst"), str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", str(deny), True) + assert status["status"] == "ok" + + legacy = { + "raw_input_sha256": hashlib.sha256(raw).hexdigest(), + "tokens": [{"label": "vin", "value": PRIVATE_VIN}], + } + legacy_path = tmp_path / "legacy.json" + legacy_path.write_text(json.dumps(legacy), encoding="utf-8") + with pytest.raises(fixture_tool.FixtureError, match="label, encoding, and value"): + fixture_tool.load_deny(str(legacy_path)) + + +def test_zstd_trailing_data_is_rejected(tmp_path: Path): + raw = compact_stream() + frame = zstandard.ZstdCompressor().compress(raw) + path = tmp_path / "trail.zst" + path.write_bytes(frame + b"TRAILING_JUNK") + with pytest.raises(fixture_tool.FixtureError, match="malformed"): + fixture_tool.process(str(path), None, str(tmp_path / "report.json"), str(POLICY), "nap-preap-pedal-v1", None, False) + + multi = frame + zstandard.ZstdCompressor().compress(b"NEXTFRAME") + path.write_bytes(multi) + with pytest.raises(fixture_tool.FixtureError, match="malformed"): + fixture_tool.process(str(path), None, str(tmp_path / "report2.json"), str(POLICY), "nap-preap-pedal-v1", None, False) + + +def test_nonexistent_canonical_path_aliases_are_rejected(tmp_path: Path): + missing = tmp_path / "out.zst" + aliased = tmp_path / "sub" / ".." / "out.zst" + with pytest.raises(fixture_tool.FixtureError, match="path collision"): + fixture_tool.validate_path_separation({"output": missing, "report": aliased}) + + +def test_bounded_zstd_expansion_rejects_bomb(tmp_path: Path): + # Tiny compressed payload that would expand past the fixture tool ceiling. + bomb = zstandard.ZstdCompressor(level=1).compress(b"A" * (fixture_tool.MAX_DECOMPRESSED_INPUT_BYTES + 1024)) + assert len(bomb) < fixture_tool.MAX_COMPRESSED_INPUT_BYTES + path = tmp_path / "bomb.zst" + path.write_bytes(bomb) + report = tmp_path / "report.json" + with pytest.raises(fixture_tool.FixtureError, match="decompressed size limit|malformed"): + fixture_tool.process(str(path), None, str(report), str(POLICY), "nap-preap-pedal-v1", None, False) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 11d07a75831539..62846f2516b08c 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -1,22 +1,56 @@ #!/usr/bin/env python3 import argparse import concurrent.futures +import hashlib +import json +import math import os +import re +import shutil +import stat import sys -from collections import defaultdict +import time +import traceback +from collections import Counter, defaultdict +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any, Literal +from urllib.request import Request, urlopen + +import zstandard as zstd from tqdm import tqdm -from typing import Any from opendbc.car.car_helpers import interface_names from openpilot.common.git import get_commit -from openpilot.tools.lib.openpilotci import get_url +from openpilot.common.params import ParamKeyType, Params +from openpilot.common.utils import atomic_write from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff -from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, replay_process, \ - check_most_messages_valid +from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, PROC_REPLAY_DIR, check_most_messages_valid, \ + replay_process from openpilot.tools.lib.filereader import FileReader from openpilot.tools.lib.logreader import LogReader, save_log -from openpilot.tools.lib.url_file import URLFile +from openpilot.tools.lib.openpilotci import get_url +from openpilot.tools.lib.url_file import URLFile, URLFileException +RESULTS_SCHEMA_VERSION = 1 +CASE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +EXCLUDED_PROCS = frozenset({"modeld", "dmonitoringmodeld"}) +ALWAYS_PROCS = ("card", "controlsd", "lagd") +EXTRA_PROCS = ( + "selfdrived", "radard", "plannerd", "calibrationd", "dmonitoringd", + "locationd", "paramsd", "ubloxd", "torqued", +) +PREAP_PROCS = ("card", "controlsd", "selfdrived", "radard", "plannerd", "lagd") +ALL_REPLAY_PROCS = tuple(cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS) +PROC_BY_NAME = {cfg.proc_name: cfg for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS} + +ARTIFACTS_REPO = "NotAutopilot/ci-artifacts" +ARTIFACTS_BRANCH = "process-replay" +ARTIFACTS_RAW = f"https://raw.githubusercontent.com/{ARTIFACTS_REPO}" + +# Keep this list compatible with regen_all.py (imports source_segments as segments). source_segments = [ ("HYUNDAI", "02c45f73a2e5c6e9|2021-01-01--19-08-22--1"), # HYUNDAI.HYUNDAI_SONATA ("HYUNDAI2", "d545129f3ca90f28|2022-11-07--20-43-08--3"), # HYUNDAI.HYUNDAI_KIA_EV6 (+ QCOM GPS) @@ -41,91 +75,1035 @@ #("VOLKSWAGEN2", "3cfdec54aa035f3f|2022-07-19--23-45-10--2"), # VOLKSWAGEN.VOLKSWAGEN_PASSAT_NMS ] -segments = [ - ("HYUNDAI", "regenAA0FC4ED71E|2025-04-08--22-57-50--0"), - ("HYUNDAI2", "regenAFB9780D823|2025-04-08--23-00-34--0"), - ("TOYOTA", "regen218A4DCFAA1|2025-04-08--22-57-51--0"), - # TODO: get new RAV4 route without enableDsu - # ("TOYOTA2", "regen107352E20EB|2025-04-08--22-57-46--0"), - ("TOYOTA3", "regen1455E3B4BDF|2025-04-09--03-26-06--0"), - ("HONDA", "regenB328FF8BA0A|2025-04-08--22-57-45--0"), - ("HONDA2", "regen6170C8C9A35|2025-04-08--22-57-46--0"), - ("CHRYSLER", "regen5B28FC2A437|2025-04-08--23-04-24--0"), - ("RAM", "regenBF81EA96E08|2025-04-08--23-06-54--0"), - ("SUBARU", "regen7366F13F6A1|2025-04-08--23-07-07--0"), - ("GM", "regen1271097D038|2025-04-09--03-26-00--0"), - ("NISSAN", "regen15D60604EAB|2025-04-08--23-06-59--0"), - ("VOLKSWAGEN", "regen0F2F06C9539|2025-04-08--23-06-56--0"), - ("MAZDA", "regenACF84CCF482|2024-08-30--03-21-55--0"), - ("FORD", "regen755D8CB1E1F|2025-04-08--23-13-43--0"), - ("RIVIAN", "regen5FCAC896BBE|2025-04-08--23-13-35--0"), - ("TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4"), -] - # dashcamOnly makes don't need to be tested until a full port is done. # "mg" is also excluded because xnor-tech's MG ZS EV port doesn't have test # routes in routes.py yet — inherited from the upstream merge and not used by NAP. excluded_interfaces = ["mock", "body", "psa", "mg"] -BASE_URL = "https://raw.githubusercontent.com/NotAutopilot/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") -EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} +REF_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") + +# Audited pre-manifest artifact tree SHA. Remove after the first manifest-backed release lands. +LEGACY_ARTIFACTS_COMMITS: frozenset[str] = frozenset({"2c9cdb739cd53df058130e480fd6a3a23e6c4387"}) + +MAX_GITHUB_REF_BODY_BYTES = 64 * 1024 +MAX_REF_COMMIT_FILE_BYTES = 41 # 40 ASCII hex + optional LF +MAX_MANIFEST_BYTES = 2 * 1024 * 1024 +MAX_SOURCE_COMPRESSED_BYTES = 64 * 1024 * 1024 +MAX_REFERENCE_COMPRESSED_BYTES = 64 * 1024 * 1024 +MAX_LOG_DECOMPRESSED_BYTES = 512 * 1024 * 1024 +MAX_LOG_MESSAGES = 2_000_000 + +_OPEN_NOFOLLOW_FLAGS = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + +ParamValue = bool | int | float +TaskStatus = Literal["pass", "diff", "error"] + + +class ManifestUnavailable(Exception): + """Exact missing/404 for a pinned artifacts object.""" + + +def _freeze_params(params: Mapping[str, ParamValue] | None) -> Mapping[str, ParamValue]: + return MappingProxyType(dict(params or {})) + + +def validate_case_id(case_id: str) -> str: + if not CASE_ID_RE.match(case_id): + raise ValueError(f"invalid case_id {case_id!r}; expected {CASE_ID_RE.pattern}") + return case_id + + +def validate_ref_commit(value: str) -> str: + if not isinstance(value, str) or REF_COMMIT_RE.fullmatch(value) is None: + raise ValueError(f"ref_commit must be a full lowercase SHA1, got {value!r}") + return value + + +def decode_ref_commit_bytes(raw: bytes) -> str: + text = raw.decode("utf-8") + if text.endswith("\n"): + text = text[:-1] + if "\n" in text or "\r" in text: + raise ValueError("ref_commit must be a single line") + return validate_ref_commit(text) + + +def validate_source_metadata( + case_id: str, + executable: bool, + source_bytes: int | None, + source_sha256: str | None, +) -> None: + has_bytes = source_bytes is not None + has_sha = source_sha256 is not None + if has_bytes ^ has_sha: + raise ValueError( + f"{case_id}: partial source metadata is rejected; provide both source_bytes and source_sha256, or neither" + ) + if executable: + if not (has_bytes and has_sha): + raise ValueError(f"{case_id}: executable ReplayCase requires both source_bytes and source_sha256") + if not isinstance(source_bytes, int) or isinstance(source_bytes, bool) or source_bytes <= 0: + raise ValueError(f"{case_id}: source_bytes must be a positive int") + if source_bytes > MAX_SOURCE_COMPRESSED_BYTES: + raise ValueError(f"{case_id}: source_bytes exceeds MAX_SOURCE_COMPRESSED_BYTES") + if not isinstance(source_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", source_sha256): + raise ValueError(f"{case_id}: source_sha256 must be 64 lowercase hex") + elif has_bytes or has_sha: + raise ValueError(f"{case_id}: pending non-executable descriptors must omit source_bytes and source_sha256") + + +def resolve_regular_member(root: str | os.PathLike[str], filename: str) -> Path: + if filename != os.path.basename(filename) or not filename or filename in {".", ".."}: + raise ValueError(f"ref member filename must be a basename, got {filename!r}") + root_path = Path(root).resolve(strict=True) + if not root_path.is_dir(): + raise ValueError(f"reference root is not a directory: {root_path}") + candidate = root_path / filename + st = os.lstat(candidate) + if stat.S_ISLNK(st.st_mode): + raise ValueError(f"symlink forbidden in reference tree: {candidate}") + if not stat.S_ISREG(st.st_mode): + raise ValueError(f"reference member must be a regular file: {candidate}") + resolved = candidate.resolve(strict=True) + try: + resolved.relative_to(root_path) + except ValueError as exc: + raise ValueError(f"reference member escapes root {root_path}: {resolved}") from exc + return resolved + + +def read_regular_member_bytes(root: str | os.PathLike[str], filename: str, *, max_bytes: int) -> bytes: + path = resolve_regular_member(root, filename) + fd = os.open(path, _OPEN_NOFOLLOW_FLAGS) + try: + st = os.fstat(fd) + if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode): + raise ValueError(f"reference member must stay a regular file: {path}") + data = os.read(fd, max_bytes + 1) + finally: + os.close(fd) + if len(data) > max_bytes: + raise ValueError(f"reference member exceeds {max_bytes} bytes: {path}") + return data + + +def _is_missing_artifact_error(exc: BaseException) -> bool: + if isinstance(exc, ManifestUnavailable): + return True + if isinstance(exc, (URLFileException, AssertionError, FileNotFoundError)): + msg = str(exc).lower() + return any(token in msg for token in ("doesn't exist", "does not exist", "not found", "404")) + return False + + +def read_url_bytes_bounded(url: str, *, max_bytes: int, url_file_cls: type = URLFile) -> bytes: + try: + handle = url_file_cls(url, cache=False) + data = handle.read(max_bytes + 1) + except Exception as exc: + if _is_missing_artifact_error(exc): + raise ManifestUnavailable(str(exc)) from exc + raise + if len(data) > max_bytes: + raise ValueError(f"remote artifact exceeds {max_bytes} bytes: {url}") + return data + + +def decompress_log_bytes(data: bytes, *, max_output_bytes: int = MAX_LOG_DECOMPRESSED_BYTES) -> bytes: + if data.startswith(b"\x28\xb5\x2f\xfd"): + dctx = zstd.ZstdDecompressor() + chunks: list[bytes] = [] + total = 0 + with dctx.stream_reader(data) as reader: + while True: + chunk = reader.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > max_output_bytes: + raise ValueError(f"decompressed log exceeds {max_output_bytes} bytes") + chunks.append(chunk) + return b"".join(chunks) + if data.startswith(b"BZh9"): + import bz2 + if len(data) > MAX_REFERENCE_COMPRESSED_BYTES: + raise ValueError("bz2 payload exceeds compressed bound") + out = bz2.decompress(data) + if len(out) > max_output_bytes: + raise ValueError(f"decompressed log exceeds {max_output_bytes} bytes") + return out + if len(data) > max_output_bytes: + raise ValueError(f"uncompressed log exceeds {max_output_bytes} bytes") + return data + + +def read_reference_bytes(locator: str, *, max_bytes: int = MAX_REFERENCE_COMPRESSED_BYTES, + url_file_cls: type = URLFile) -> bytes: + if locator.startswith(("http://", "https://")): + return read_url_bytes_bounded(locator, max_bytes=max_bytes, url_file_cls=url_file_cls) + root = os.path.dirname(locator) or "." + filename = os.path.basename(locator) + return read_regular_member_bytes(root, filename, max_bytes=max_bytes) + + +def load_ref_log_messages(locator: str, *, expected_bytes: int | None = None, + expected_sha256: str | None = None, + url_file_cls: type = URLFile) -> list[Any]: + max_bytes = expected_bytes if expected_bytes is not None else MAX_REFERENCE_COMPRESSED_BYTES + if expected_bytes is not None and expected_bytes > MAX_REFERENCE_COMPRESSED_BYTES: + raise ValueError("reference declared size exceeds MAX_REFERENCE_COMPRESSED_BYTES") + raw = read_reference_bytes(locator, max_bytes=max_bytes, url_file_cls=url_file_cls) + if expected_bytes is not None and len(raw) != expected_bytes: + raise ValueError(f"reference size mismatch expected={expected_bytes} actual={len(raw)}") + if expected_sha256 is not None: + digest = hashlib.sha256(raw).hexdigest() + if digest != expected_sha256: + raise ValueError(f"reference sha256 mismatch expected={expected_sha256} actual={digest}") + # Always decompress under a hard ceiling, then parse the bounded payload. + if raw.startswith((b"\x28\xb5\x2f\xfd", b"BZh9")): + payload = decompress_log_bytes(raw) + else: + payload = decompress_log_bytes(raw) + lr = LogReader.from_bytes(payload) + out: list[Any] = [] + for msg in lr: + out.append(msg) + if len(out) > MAX_LOG_MESSAGES: + raise ValueError(f"reference log exceeds MAX_LOG_MESSAGES ({MAX_LOG_MESSAGES})") + return out + + +@dataclass(frozen=True) +class ReplayCase: + __hash__ = None + + case_id: str + car_brand: str + source: str + processes: tuple[str, ...] + custom_params: Mapping[str, ParamValue] = field(default_factory=dict) + source_sha256: str | None = None + source_bytes: int | None = None + + def __post_init__(self): + object.__setattr__(self, "custom_params", _freeze_params(self.custom_params)) + object.__setattr__(self, "processes", tuple(self.processes)) + validate_case_id(self.case_id) + unknown = [p for p in self.processes if p not in PROC_BY_NAME] + if unknown: + raise ValueError(f"{self.case_id}: unknown processes {unknown}") + if len(set(self.processes)) != len(self.processes): + raise ValueError(f"{self.case_id}: duplicate processes") + validate_source_metadata(self.case_id, bool(self.source), self.source_bytes, self.source_sha256) + + @property + def executable(self) -> bool: + return bool(self.source) + + def source_url(self) -> str: + if not self.executable: + raise ValueError(f"{self.case_id}: source is not executable") + if self.source.startswith(("http://", "https://")): + return self.source + route, seg_num = self.source.rsplit("--", 1) + return get_url(route, seg_num, "rlog.zst") + + def to_record(self) -> dict[str, Any]: + return { + "case_id": self.case_id, + "car_brand": self.car_brand, + "source": self.source, + "processes": list(self.processes), + "custom_params": dict(self.custom_params), + "source_sha256": self.source_sha256, + "source_bytes": self.source_bytes, + "executable": self.executable, + "params_digest": params_digest(self.custom_params), + } + + +@dataclass(frozen=True) +class ReplayTask: + case: ReplayCase + process: str + + def __post_init__(self): + if self.process not in self.case.processes: + raise ValueError(f"{self.process} not in case {self.case.case_id}") + if self.process not in PROC_BY_NAME: + raise ValueError(f"unknown process {self.process}") + + @property + def task_id(self) -> str: + return f"{self.case.case_id}:{self.process}" + + +@dataclass(frozen=True) +class TaskResult: + status: TaskStatus + task_id: str + elapsed_s: float + diff_or_traceback: str + ref_path: str + new_path: str + message_counts: Mapping[str, int] = field(default_factory=dict) + + def __post_init__(self): + object.__setattr__(self, "message_counts", dict(self.message_counts)) + + def to_record(self) -> dict[str, Any]: + return { + "status": self.status, + "task_id": self.task_id, + "elapsed_s": self.elapsed_s, + "diff_or_traceback": self.diff_or_traceback, + "ref_path": self.ref_path, + "new_path": self.new_path, + "message_counts": dict(self.message_counts), + } + + +# Exact active-source digests measured from the current public route sources. +ACTIVE_SOURCE_DIGESTS: dict[str, dict[str, int | str]] = { + "chrysler": {"bytes": 11951101, "sha256": "a30f65eb1033afbd0cff70d965aa9c0a276cc5ceb73d28baa817430df178009a"}, + "ford": {"bytes": 11315727, "sha256": "786dfef08325d24e8205320862adc940706ff3475ae082ea104127fb5da62823"}, + "gm": {"bytes": 9846869, "sha256": "6c10ce899270e4fc8350730e44cb39dfbda451670dd598798207ffc929baa69a"}, + "honda": {"bytes": 10090147, "sha256": "8ea4ab5412a99bfc65fbe95cd169a54edb6bcb68f6fa20c97f2bc58ed4bba538"}, + "honda2": {"bytes": 10419375, "sha256": "fc9ab2b4fd9702ceb8e3342186076dc6a85dbd6a3bd859ac6b965234bcc65c69"}, + "hyundai": {"bytes": 11873030, "sha256": "0b6734026f7c775f7fd648ef76130f9b9a672c5236c6a328eff31ec9cc3c5343"}, + "hyundai2": {"bytes": 12527194, "sha256": "4cc51a77bf8d128d5720fd692e5a0994c887ee35fdd949cc0ce54d4215e2b7a2"}, + "mazda": {"bytes": 12309353, "sha256": "a84e9f84c39802f58a64031e7c56b362c0e745b5c216848ca2d561f5ae6d440a"}, + "nissan": {"bytes": 11775886, "sha256": "58f3822418e86d8c4afca79ef9058602c8825fadaedf8b63477ed53a2b9ba748"}, + "ram": {"bytes": 11420074, "sha256": "cc1d8bb9729a71c3166e04343e436a67fbc1ff7107a30201a5d13ea9b45ea143"}, + "rivian": {"bytes": 11022997, "sha256": "2f2574c057b7cb55fbb2e05c99c5f1b1d6d894222422c4b51d4363ad66e6c840"}, + "subaru": {"bytes": 9393878, "sha256": "f501c1c99bae27ca710f40f723a1960ea4e898836d7edfb57ea6397591e5aff2"}, + "tesla": {"bytes": 8287366, "sha256": "ca3bfc37b118f0e1987c2177b77b7446d44fbc7806cc5fae71c2ed79d4776fff"}, + "toyota": {"bytes": 10846201, "sha256": "6c028c0d40328fc00fa12acf91dba30d9f152e89d99a7089aadac62f6c7d5d64"}, + "toyota3": {"bytes": 9272538, "sha256": "876100021b13ddd1a39efad5fdb4e7376ad5b298f64bf390b5f58742ffc6e429"}, + "volkswagen": {"bytes": 10593610, "sha256": "bca512375f61ade1af793227f4aa9137ba1cbd0067d3b92f1242f5907f84a0af"}, +} + + +def _legacy_case(case_id: str, car_brand: str, segment: str, full: bool) -> ReplayCase: + processes = ALWAYS_PROCS + (EXTRA_PROCS if full else ()) + digest = ACTIVE_SOURCE_DIGESTS[case_id] + return ReplayCase( + case_id=case_id, + car_brand=car_brand, + source=segment, + processes=processes, + source_bytes=int(digest["bytes"]), + source_sha256=str(digest["sha256"]), + ) + + +ACTIVE_CASES: tuple[ReplayCase, ...] = ( + _legacy_case("hyundai", "HYUNDAI", "regenAA0FC4ED71E|2025-04-08--22-57-50--0", True), + _legacy_case("hyundai2", "HYUNDAI2", "regenAFB9780D823|2025-04-08--23-00-34--0", False), + _legacy_case("toyota", "TOYOTA", "regen218A4DCFAA1|2025-04-08--22-57-51--0", True), + # TODO: get new RAV4 route without enableDsu + # _legacy_case("toyota2", "TOYOTA2", "regen107352E20EB|2025-04-08--22-57-46--0", False), + _legacy_case("toyota3", "TOYOTA3", "regen1455E3B4BDF|2025-04-09--03-26-06--0", False), + _legacy_case("honda", "HONDA", "regenB328FF8BA0A|2025-04-08--22-57-45--0", False), + _legacy_case("honda2", "HONDA2", "regen6170C8C9A35|2025-04-08--22-57-46--0", False), + _legacy_case("chrysler", "CHRYSLER", "regen5B28FC2A437|2025-04-08--23-04-24--0", False), + _legacy_case("ram", "RAM", "regenBF81EA96E08|2025-04-08--23-06-54--0", False), + _legacy_case("subaru", "SUBARU", "regen7366F13F6A1|2025-04-08--23-07-07--0", False), + _legacy_case("gm", "GM", "regen1271097D038|2025-04-09--03-26-00--0", False), + _legacy_case("nissan", "NISSAN", "regen15D60604EAB|2025-04-08--23-06-59--0", False), + _legacy_case("volkswagen", "VOLKSWAGEN", "regen0F2F06C9539|2025-04-08--23-06-56--0", False), + _legacy_case("mazda", "MAZDA", "regenACF84CCF482|2024-08-30--03-21-55--0", False), + _legacy_case("ford", "FORD", "regen755D8CB1E1F|2025-04-08--23-13-43--0", False), + _legacy_case("rivian", "RIVIAN", "regen5FCAC896BBE|2025-04-08--23-13-35--0", False), + _legacy_case("tesla", "TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4", False), +) + +# Compatibility alias for anything still iterating brand/segment pairs. +segments = [(c.car_brand, c.source) for c in ACTIVE_CASES] + +NAP_PREAP_PEDAL_PARAMS: dict[str, ParamValue] = { + "NAPForcePreAP": True, + "NAPAdaptiveAccel": True, + "NAPPedalEnabled": True, + "NAPFollowDistance": 7, + "NAPPedalProfile": 4, + "NAPPedalCanBus": 2, + "NAPPedalCalibDone": True, + "NAPPedalCalibMin": 0.73299810159, + "NAPPedalCalibMax": 110.23299810159, + "NAPPedalCalibFactor": 0.9478672985781991, + "NAPPedalCalibZero": 4.732998101589998, + "NAPRadarEnabled": True, + "NAPRadarBehindNosecone": True, + "NAPRadarOffset": -0.5, + "NAPiBoosterEnabled": False, + "NAPBrakeFactor": 1.0, +} + +NAP_PREAP_NO_PEDAL_PARAMS: dict[str, ParamValue] = { + **NAP_PREAP_PEDAL_PARAMS, + "NAPPedalEnabled": False, + "NAPFollowDistance": 5, + "NAPPedalCalibMin": 0.9382172261100007, + "NAPPedalCalibMax": 110.43821722611, + "NAPPedalCalibFactor": 0.946969696969697, + "NAPPedalCalibZero": 4.838217226109999, + "NAPRadarOffset": -0.25, +} + +# Temporary staged descriptors for rollout PR A. Sources stay non-executable until fixture publish. +PENDING_CASES: tuple[ReplayCase, ...] = ( + ReplayCase( + case_id="nap-preap-pedal-v1", + car_brand="TESLA", + source="", + processes=PREAP_PROCS, + custom_params=NAP_PREAP_PEDAL_PARAMS, + ), + ReplayCase( + case_id="nap-preap-no-pedal-v1", + car_brand="TESLA", + source="", + processes=PREAP_PROCS, + custom_params=NAP_PREAP_NO_PEDAL_PARAMS, + ), +) + +EMPTY_PARAMS_DIGEST = "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" +NAP_PREAP_PEDAL_PARAMS_DIGEST = "b5217b239c759d5a960872a99f2e9ff6cacdb37aff6e99e1ee1bd0c20dead809" +NAP_PREAP_NO_PEDAL_PARAMS_DIGEST = "96b5f9f76bdf20f24bf2a29d408d87e60ebf377eccc3858b4f0c0de7c3c2e786" + + +def params_type_name(value: ParamValue) -> str: + if type(value) is bool: + return "bool" + if type(value) is int: + return "int" + if type(value) is float: + return "float" + raise TypeError(f"unsupported param value type {type(value)!r}") + + +def canonical_params_manifest(params: Mapping[str, ParamValue]) -> bytes: + """Canonical JSON array sorted by key: key/type/value with fixed separators and trailing LF.""" + parts: list[str] = [] + for key in sorted(params): + value = params[key] + type_name = params_type_name(value) + if type_name == "bool": + encoded = "true" if value else "false" + elif type_name == "int": + encoded = str(value) + else: + if not isinstance(value, float) or not math.isfinite(value): + raise ValueError(f"param {key} must be a finite float") + encoded = repr(value) + parts.append(f'{{"key":"{key}","type":"{type_name}","value":{encoded}}}') + return ("[" + ",".join(parts) + "]\n").encode("utf-8") + + +def params_digest(params: Mapping[str, ParamValue]) -> str: + return hashlib.sha256(canonical_params_manifest(params)).hexdigest() + + +def expected_param_key_type(value: ParamValue) -> ParamKeyType: + return { + "bool": ParamKeyType.BOOL, + "int": ParamKeyType.INT, + "float": ParamKeyType.FLOAT, + }[params_type_name(value)] + + +def validate_params_against_header(params: Mapping[str, ParamValue], params_obj: Params | None = None) -> None: + checker = params_obj or Params() + for key, value in params.items(): + declared = checker.get_type(key) + expected = expected_param_key_type(value) + if declared != expected: + raise TypeError(f"param {key}: value type {type(value)!r} does not match header type {declared!r}") + + +def build_tasks(cases: Iterable[ReplayCase]) -> list[ReplayTask]: + return [ReplayTask(case=case, process=proc) for case in cases for proc in case.processes] + + +def select_cases(inventory: Literal["active", "staged"] = "active") -> tuple[ReplayCase, ...]: + if inventory == "active": + return ACTIVE_CASES + if inventory == "staged": + return ACTIVE_CASES + PENDING_CASES + raise ValueError(f"unknown inventory {inventory!r}") + + +def assert_inventory_invariants(cases: Iterable[ReplayCase]) -> None: + cases = tuple(cases) + case_ids = [c.case_id for c in cases] + if len(case_ids) != len(set(case_ids)): + raise AssertionError(f"duplicate case_ids: {case_ids}") + tasks = build_tasks(cases) + task_ids = [t.task_id for t in tasks] + if len(task_ids) != len(set(task_ids)): + raise AssertionError(f"duplicate task_ids: {task_ids}") + for case in cases: + validate_source_metadata(case.case_id, case.executable, case.source_bytes, case.source_sha256) + validate_params_against_header(case.custom_params) + for key in case.custom_params: + if not key: + raise AssertionError("empty param key") + +def unique_cases_by_id(tasks: Iterable[ReplayTask]) -> dict[str, ReplayCase]: + """Deterministic case_id deduplication; never hash ReplayCase (custom_params is unhashable).""" + out: dict[str, ReplayCase] = {} + for task in tasks: + existing = out.get(task.case.case_id) + if existing is None: + out[task.case.case_id] = task.case + elif existing != task.case: + raise AssertionError(f"conflicting ReplayCase records for case_id={task.case.case_id}") + return out -def run_test_process(data): - segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data - lr = LogReader.from_bytes(lr_dat) - res, log_msgs = test_process(cfg, lr, segment, ref_log_path, cur_log_fn, args.ignore_fields, args.ignore_msgs) - # save logs so we can update refs - save_log(cur_log_fn, log_msgs) - return (segment, cfg.proc_name, res) +def filter_tasks( + tasks: Iterable[ReplayTask], + *, + whitelist_procs: Iterable[str] | None = None, + blacklist_procs: Iterable[str] | None = None, + whitelist_cars: Iterable[str] | None = None, + blacklist_cars: Iterable[str] | None = None, +) -> list[ReplayTask]: + all_procs = {t.process for t in tasks} or set(ALL_REPLAY_PROCS) + all_cars = {t.case.car_brand for t in tasks} + tested_procs = set(whitelist_procs if whitelist_procs is not None else all_procs) - set(blacklist_procs or []) + tested_cars = {c.upper() for c in (whitelist_cars if whitelist_cars is not None else all_cars)} - {c.upper() for c in (blacklist_cars or [])} + return [t for t in tasks if t.process in tested_procs and t.case.car_brand.upper() in tested_cars] -def get_log_data(segment): - r, n = segment.rsplit("--", 1) - with FileReader(get_url(r, n, "rlog.zst")) as f: - return (segment, f.read()) +def shard_task_ids(tasks: Iterable[ReplayTask], shard: Literal["card", "controlsd", "lagd", "other"]) -> list[str]: + if shard == "other": + blocked = {"card", "controlsd", "lagd"} + return [t.task_id for t in tasks if t.process not in blocked] + return [t.task_id for t in tasks if t.process == shard] -def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=None, ignore_msgs=None): + +def ref_filename(case_id: str, process: str, ref_commit: str) -> str: + validate_case_id(case_id) + validate_ref_commit(ref_commit) + return f"{case_id}__{process}__{ref_commit}.zst" + + +def legacy_ref_filename(segment: str, process: str, ref_commit: str) -> str: + validate_ref_commit(ref_commit) + return f"{segment}_{process}_{ref_commit}.zst".replace("|", "_") + + +def pin_artifacts_commit(branch: str = ARTIFACTS_BRANCH, opener: Callable[..., Any] = urlopen) -> str: + req = Request(f"https://api.github.com/repos/{ARTIFACTS_REPO}/git/ref/heads/{branch}", headers={"Accept": "application/vnd.github+json"}) + with opener(req, timeout=30) as resp: + body = resp.read(MAX_GITHUB_REF_BODY_BYTES + 1) + if len(body) > MAX_GITHUB_REF_BODY_BYTES: + raise ValueError("GitHub ref response exceeds MAX_GITHUB_REF_BODY_BYTES") + payload = json.loads(body.decode("utf-8")) + sha = payload["object"]["sha"] + return validate_ref_commit(sha) + + +def artifacts_file_url(artifacts_commit: str, filename: str) -> str: + if not re.fullmatch(r"[0-9a-f]{40}", artifacts_commit): + raise ValueError(f"artifacts commit must be a full SHA, got {artifacts_commit!r}") + return f"{ARTIFACTS_RAW}/{artifacts_commit}/{filename}" + + +def resolve_ref_commit( + artifacts_commit: str | None = None, + ref_commit_path: str = REF_COMMIT_FN, + *, + reference_dir: str | None = None, + pin_fn: Callable[..., str] = pin_artifacts_commit, + url_file_cls: type = URLFile, +) -> tuple[str, str | None]: + if reference_dir is not None: + raw = read_regular_member_bytes(reference_dir, "ref_commit", max_bytes=MAX_REF_COMMIT_FILE_BYTES) + return decode_ref_commit_bytes(raw), artifacts_commit + + if os.path.lexists(ref_commit_path): + root = os.path.dirname(ref_commit_path) or "." + raw = read_regular_member_bytes(root, os.path.basename(ref_commit_path), max_bytes=MAX_REF_COMMIT_FILE_BYTES) + return decode_ref_commit_bytes(raw), artifacts_commit + + if artifacts_commit is None: + artifacts_commit = pin_fn() + validate_ref_commit(artifacts_commit) + raw = read_url_bytes_bounded( + artifacts_file_url(artifacts_commit, "ref_commit"), + max_bytes=MAX_REF_COMMIT_FILE_BYTES, + url_file_cls=url_file_cls, + ) + return decode_ref_commit_bytes(raw), artifacts_commit + + +def write_candidate_ref_commit(ref_commit: str, candidate_dir: str = FAKEDATA) -> str: + """Write candidate ref_commit beside generated refs so a copied tree is reusable with --reference-dir.""" + validate_ref_commit(ref_commit) + os.makedirs(candidate_dir, exist_ok=True) + path = os.path.join(candidate_dir, "ref_commit") + with atomic_write(path, mode="w", encoding="utf-8", overwrite=True) as f: + f.write(ref_commit) + f.write("\n") + return path + + +def parse_artifacts_manifest( + raw: bytes, + *, + expected_ref_commit: str, + selected_task_ids: Iterable[str] | None = None, +) -> dict[str, Any]: + if not raw: + raise ValueError("malformed artifacts manifest: empty body") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"malformed artifacts manifest: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError("malformed artifacts manifest: expected object") + schema_version = payload.get("schema_version") + if schema_version != 1: + raise ValueError(f"unsupported artifacts manifest schema_version: {schema_version!r}") + manifest_ref = payload.get("ref_commit", payload.get("openpilot_sha")) + if manifest_ref != expected_ref_commit: + raise ValueError( + f"artifacts manifest ref identity mismatch: expected {expected_ref_commit}, got {manifest_ref!r}" + ) + refs = payload.get("refs") + if not isinstance(refs, dict): + raise ValueError("malformed artifacts manifest: refs must be an object") + for task_id, meta in refs.items(): + if not isinstance(task_id, str) or ":" not in task_id: + raise ValueError(f"malformed artifacts manifest task id: {task_id!r}") + if not isinstance(meta, dict): + raise ValueError(f"malformed artifacts manifest refs[{task_id!r}]") + filename = meta.get("filename") + size = meta.get("size", meta.get("bytes")) + sha256 = meta.get("sha256") + if not isinstance(filename, str) or filename != os.path.basename(filename): + raise ValueError(f"malformed artifacts manifest filename for {task_id}") + if size is not None: + if not isinstance(size, int) or isinstance(size, bool) or size <= 0 or size > MAX_REFERENCE_COMPRESSED_BYTES: + raise ValueError(f"malformed artifacts manifest size for {task_id}") + if sha256 is not None and (not isinstance(sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", sha256)): + raise ValueError(f"malformed artifacts manifest sha256 for {task_id}") + if selected_task_ids is not None: + missing = [tid for tid in selected_task_ids if tid not in refs] + if missing: + raise ValueError(f"artifacts manifest missing task metadata for {missing[0]}") + return payload + + +def load_artifacts_manifest( + artifacts_commit: str, + *, + expected_ref_commit: str, + selected_task_ids: Iterable[str] | None = None, + url_file_cls: type = URLFile, +) -> dict[str, Any] | None: + """Load pinned manifest.json once. + + Returns None only for an exact missing manifest on an allowlisted pre-bootstrap commit. + """ + validate_ref_commit(artifacts_commit) + validate_ref_commit(expected_ref_commit) + url = artifacts_file_url(artifacts_commit, "manifest.json") + try: + raw = read_url_bytes_bounded(url, max_bytes=MAX_MANIFEST_BYTES, url_file_cls=url_file_cls) + except ManifestUnavailable as error: + if artifacts_commit in LEGACY_ARTIFACTS_COMMITS: + return None + raise ValueError(f"artifacts manifest missing for non-legacy commit {artifacts_commit}") from error + return parse_artifacts_manifest( + raw, + expected_ref_commit=expected_ref_commit, + selected_task_ids=selected_task_ids, + ) + + +def resolve_ref_path( + task: ReplayTask, + *, + ref_commit: str, + artifacts_commit: str | None, + reference_dir: str | None, + update_refs: bool, + artifacts_manifest: dict[str, Any] | None = None, + pin_fn: Callable[..., str] = pin_artifacts_commit, +) -> str: + if update_refs: + # Candidate generation does not read an existing reference. + return "" + + validate_ref_commit(ref_commit) + new_name = ref_filename(task.case.case_id, task.process, ref_commit) + legacy_name = None + if task.case.source and not task.case.source.startswith(("http://", "https://")): + legacy_name = legacy_ref_filename(task.case.source, task.process, ref_commit) + + def _local_regular(root: str, filename: str) -> str | None: + try: + return str(resolve_regular_member(root, filename)) + except FileNotFoundError: + return None + + # A local candidate tree is authoritative. In particular, do not consult + # FAKEDATA when --reference-dir points at a copied candidate tree. + if reference_dir is not None: + Path(reference_dir).resolve(strict=True) # must exist as a real directory + found = _local_regular(reference_dir, new_name) + if found is None and legacy_name is not None: + found = _local_regular(reference_dir, legacy_name) + if found is not None: + return found + # Caller asked for a local candidate tree; do not silently fall back to remote. + return str(Path(reference_dir).resolve(strict=True) / new_name) + + # Default workspace: only accept regular non-symlink members under FAKEDATA. + if os.path.isdir(FAKEDATA): + found = _local_regular(FAKEDATA, new_name) + if found is None and legacy_name is not None: + found = _local_regular(FAKEDATA, legacy_name) + if found is not None: + return found + + if artifacts_commit is None: + artifacts_commit = pin_fn() + validate_ref_commit(artifacts_commit) + + if artifacts_manifest is not None: + refs = artifacts_manifest.get("refs") + if not isinstance(refs, dict): + raise ValueError("malformed artifacts manifest: refs must be an object") + meta = refs.get(task.task_id) + if not isinstance(meta, dict): + raise ValueError(f"artifacts manifest missing task metadata for {task.task_id}") + filename = meta.get("filename") + if filename != new_name: + raise ValueError( + f"artifacts manifest has non-canonical filename for {task.task_id}: expected {new_name!r}, got {filename!r}" + ) + return artifacts_file_url(artifacts_commit, new_name) + + # Pre-bootstrap trees have no manifest. Only allowlisted commits may use + # the historical segment filename; once a manifest exists it is canonical. + if artifacts_commit not in LEGACY_ARTIFACTS_COMMITS: + raise ValueError( + f"artifacts commit {artifacts_commit} has no manifest and is not in LEGACY_ARTIFACTS_COMMITS" + ) + if legacy_name is not None: + return artifacts_file_url(artifacts_commit, legacy_name) + return artifacts_file_url(artifacts_commit, new_name) + + +def new_log_path(task: ReplayTask, tested_commit: str, output_dir: str | None = None) -> str: + # output_dir is accepted for tests only; normal main keeps generated logs in FAKEDATA. + validate_ref_commit(tested_commit) + base = output_dir or FAKEDATA + os.makedirs(base, exist_ok=True) + return os.path.join(base, ref_filename(task.case.case_id, task.process, tested_commit)) + + +def clear_stale_fakedata(path: str = FAKEDATA) -> None: + if os.path.isdir(path): + shutil.rmtree(path) + os.makedirs(path, exist_ok=True) + + +def download_case_bytes(case: ReplayCase) -> bytes: + if not case.executable: + raise ValueError(f"{case.case_id}: refusing to download non-executable pending source") + if case.source_bytes is None or case.source_sha256 is None: + raise ValueError(f"{case.case_id}: executable source requires size and sha256") + if case.source_bytes > MAX_SOURCE_COMPRESSED_BYTES: + raise ValueError(f"{case.case_id}: source_bytes exceeds MAX_SOURCE_COMPRESSED_BYTES") + with FileReader(case.source_url()) as f: + data = f.read(case.source_bytes + 1) + if len(data) != case.source_bytes: + raise ValueError(f"{case.case_id}: size mismatch expected={case.source_bytes} actual={len(data)}") + digest = hashlib.sha256(data).hexdigest() + if digest != case.source_sha256: + raise ValueError(f"{case.case_id}: sha256 mismatch expected={case.source_sha256} actual={digest}") + return data + + +def download_case_by_id(case_payload: dict[str, Any]) -> tuple[str, bytes | None, str | None]: + case_id = case_payload["case_id"] + try: + case = ReplayCase( + case_id=case_payload["case_id"], + car_brand=case_payload["car_brand"], + source=case_payload["source"], + processes=tuple(case_payload["processes"]), + custom_params=case_payload["custom_params"], + source_sha256=case_payload.get("source_sha256"), + source_bytes=case_payload.get("source_bytes"), + ) + return case_id, download_case_bytes(case), None + except Exception: + return case_id, None, traceback.format_exc() + + +def message_counts(log_msgs: Iterable[Any]) -> dict[str, int]: + return dict(Counter(m.which() for m in log_msgs)) + + +def test_process(cfg, lr, task: ReplayTask, ref_log_path: str, new_log_path: str, ignore_fields=None, ignore_msgs=None, + update_refs: bool = False) -> tuple[TaskStatus, str, list[Any]]: if ignore_fields is None: ignore_fields = [] if ignore_msgs is None: ignore_msgs = [] - ref_log_msgs = list(LogReader(ref_log_path)) - - try: - log_msgs = replay_process(cfg, lr, disable_progress=True) - except Exception as e: - raise Exception("failed on segment: " + segment) from e + log_msgs = replay_process(cfg, lr, disable_progress=True, custom_params=dict(task.case.custom_params)) + # Persist immediately so missing/corrupt refs still retain the generated log. + save_log(new_log_path, log_msgs) if not check_most_messages_valid(log_msgs): - return f"Route did not have enough valid messages: {new_log_path}", log_msgs + return "error", f"Route did not have enough valid messages: {new_log_path}", log_msgs - # skip this check if the segment is using qcom gps - if cfg.proc_name != 'ubloxd' or any(m.which() in cfg.pubs for m in lr): + if cfg.proc_name != "ubloxd" or any(m.which() in cfg.pubs for m in lr): seen_msgs = {m.which() for m in log_msgs} expected_msgs = set(cfg.subs) if seen_msgs != expected_msgs: - return f"Expected messages: {expected_msgs}, but got: {seen_msgs}", log_msgs + return "error", f"Expected messages: {expected_msgs}, but got: {seen_msgs}", log_msgs + + if update_refs or not ref_log_path: + return "pass", "", log_msgs try: - return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs + ref_log_msgs = load_ref_log_messages(ref_log_path) + diff = compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance) except Exception as e: - return str(e), log_msgs + return "error", str(e), log_msgs + if isinstance(diff, str) or len(diff): + return "diff", diff if isinstance(diff, str) else json.dumps([list(x) if not isinstance(x, str) else x for x in diff], default=str), log_msgs + return "pass", "", log_msgs -if __name__ == "__main__": - all_cars = {car for car, _ in segments} - all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS} +def run_test_process(data: dict[str, Any]) -> TaskResult: + task_id = data["task_id"] + ref_path = data.get("ref_log_path", "") + new_path = data.get("new_log_path", "") + started = time.perf_counter() + try: + case = ReplayCase( + case_id=data["case_id"], + car_brand=data["car_brand"], + source=data["source"], + processes=tuple(data["processes"]), + custom_params=data["custom_params"], + source_sha256=data.get("source_sha256"), + source_bytes=data.get("source_bytes"), + ) + task = ReplayTask(case=case, process=data["process"]) + cfg = PROC_BY_NAME[task.process] + raw = data["lr_dat"] + if not isinstance(raw, (bytes, bytearray)): + raise TypeError("lr_dat must be bytes") + if len(raw) > MAX_SOURCE_COMPRESSED_BYTES: + raise ValueError("source payload exceeds MAX_SOURCE_COMPRESSED_BYTES") + if raw.startswith((b"\x28\xb5\x2f\xfd", b"BZh9")): + lr = LogReader.from_bytes(decompress_log_bytes(bytes(raw))) + else: + if len(raw) > MAX_LOG_DECOMPRESSED_BYTES: + raise ValueError("uncompressed source exceeds MAX_LOG_DECOMPRESSED_BYTES") + lr = LogReader.from_bytes(bytes(raw)) + status, detail, log_msgs = test_process( + cfg, lr, task, ref_path, new_path, + ignore_fields=data.get("ignore_fields") or [], + ignore_msgs=data.get("ignore_msgs") or [], + update_refs=bool(data.get("update_refs")), + ) + if status == "diff" and not isinstance(detail, str): + detail = str(detail) + return TaskResult( + status=status, + task_id=task_id, + elapsed_s=time.perf_counter() - started, + diff_or_traceback=detail if isinstance(detail, str) else str(detail), + ref_path=ref_path, + new_path=new_path, + message_counts=message_counts(log_msgs), + ) + except Exception: + return TaskResult( + status="error", + task_id=task_id, + elapsed_s=time.perf_counter() - started, + diff_or_traceback=traceback.format_exc(), + ref_path=ref_path, + new_path=new_path, + message_counts={}, + ) + + +def make_task_payload(task: ReplayTask, *, lr_dat: bytes, ref_log_path: str, new_log_path: str, + ignore_fields: list[str], ignore_msgs: list[str], update_refs: bool) -> dict[str, Any]: + return { + "task_id": task.task_id, + "case_id": task.case.case_id, + "car_brand": task.case.car_brand, + "source": task.case.source, + "processes": list(task.case.processes), + "custom_params": dict(task.case.custom_params), + "source_sha256": task.case.source_sha256, + "source_bytes": task.case.source_bytes, + "process": task.process, + "lr_dat": lr_dat, + "ref_log_path": ref_log_path, + "new_log_path": new_log_path, + "ignore_fields": ignore_fields, + "ignore_msgs": ignore_msgs, + "update_refs": update_refs, + } + + +def format_diff_text(results: list[TaskResult], ref_commit: str) -> tuple[str, str, bool]: + legacy_results: dict[str, dict[str, Any]] = defaultdict(dict) + log_paths: dict[str, dict[str, dict[str, str]]] = defaultdict(lambda: defaultdict(dict)) + for result in results: + case_id, proc = result.task_id.split(":", 1) + if result.status == "pass": + legacy_results[case_id][proc] = [] + elif result.status == "diff": + legacy_results[case_id][proc] = result.diff_or_traceback + else: + legacy_results[case_id][proc] = result.diff_or_traceback + log_paths[case_id][proc]["ref"] = result.ref_path + log_paths[case_id][proc]["new"] = result.new_path + return format_diff(legacy_results, log_paths, ref_commit) + + +def build_results_document( + *, + tested_commit: str, + ref_commit: str, + artifacts_commit: str | None, + inventory: str, + cases: Iterable[ReplayCase], + expected_task_ids: Iterable[str], + results: Mapping[str, TaskResult], +) -> dict[str, Any]: + expected = sorted(expected_task_ids) + settled = [results[tid] for tid in expected if tid in results] + status_counts = Counter(r.status for r in settled) + incomplete = [tid for tid in expected if tid not in results] + return { + "schema_version": RESULTS_SCHEMA_VERSION, + "tested_commit": tested_commit, + "ref_commit": ref_commit, + "artifacts_commit": artifacts_commit, + "inventory": inventory, + "cases": [c.to_record() for c in cases], + "expected_task_ids": expected, + "completed_task_ids": sorted(results), + "incomplete_task_ids": incomplete, + "status_counts": { + "pass": status_counts.get("pass", 0), + "diff": status_counts.get("diff", 0), + "error": status_counts.get("error", 0), + "incomplete": len(incomplete), + }, + "results": [results[tid].to_record() for tid in sorted(results)], + } + + +def write_diagnostics(output_dir: str, document: dict[str, Any], results: list[TaskResult], ref_commit: str) -> None: + os.makedirs(output_dir, exist_ok=True) + results_path = os.path.join(output_dir, "results.json") + diff_path = os.path.join(output_dir, "diff.txt") + with atomic_write(results_path, mode="w", encoding="utf-8", overwrite=True) as f: + json.dump(document, f, indent=2, sort_keys=True) + f.write("\n") + diff_short, diff_long, _ = format_diff_text(results, ref_commit) + with atomic_write(diff_path, mode="w", encoding="utf-8", overwrite=True) as f: + f.write(diff_long) + if not diff_long.endswith("\n"): + f.write("\n") + return + + +def retain_failed_logs(results: Iterable[TaskResult]) -> None: + for result in results: + if not result.new_path: + continue + if result.status == "pass": + if os.path.exists(result.new_path): + os.remove(result.new_path) + # failed/diff logs are retained + + +def run_task_pool( + payloads: list[dict[str, Any]], + *, + jobs: int, + worker: Callable[[dict[str, Any]], TaskResult] = run_test_process, + on_settled: Callable[[TaskResult], None] | None = None, + executor_factory=concurrent.futures.ProcessPoolExecutor, +) -> list[TaskResult]: + results: list[TaskResult] = [] + if not payloads: + return results + + with executor_factory(max_workers=max(jobs, 1)) as pool: + future_map = {pool.submit(worker, payload): payload for payload in payloads} + for fut in concurrent.futures.as_completed(future_map): + payload = future_map[fut] + try: + result = fut.result() + except Exception: + result = TaskResult( + status="error", + task_id=payload["task_id"], + elapsed_s=0.0, + diff_or_traceback=traceback.format_exc(), + ref_path=payload.get("ref_log_path", ""), + new_path=payload.get("new_log_path", ""), + ) + results.append(result) + if on_settled is not None: + on_settled(result) + return results + + +def exit_code_for(document: dict[str, Any]) -> int: + counts = document["status_counts"] + if counts["diff"] or counts["error"] or counts["incomplete"]: + return 1 + return 0 + + +def main( + argv: list[str] | None = None, + *, + get_commit_fn: Callable[[], str | None] = get_commit, + pin_artifacts_commit_fn: Callable[..., str] = pin_artifacts_commit, + resolve_ref_commit_fn: Callable[..., tuple[str, str | None]] = resolve_ref_commit, + load_artifacts_manifest_fn: Callable[..., dict[str, Any] | None] = load_artifacts_manifest, + download_case_by_id_fn: Callable[[dict[str, Any]], tuple[str, bytes | None, str | None]] = download_case_by_id, + run_task_pool_fn: Callable[..., list[TaskResult]] = run_task_pool, + clear_stale_fakedata_fn: Callable[..., None] = clear_stale_fakedata, + write_candidate_ref_commit_fn: Callable[..., str] = write_candidate_ref_commit, + interface_names_fn: Callable[[], set[str]] | None = None, +) -> int: + all_cars = {c.car_brand for c in ACTIVE_CASES} + all_procs = set(ALL_REPLAY_PROCS) cpu_count = os.cpu_count() or 1 parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output") - parser.add_argument("--whitelist-procs", type=str, nargs="*", default=all_procs, + parser.add_argument("--whitelist-procs", type=str, nargs="*", default=None, help="Whitelist given processes from the test (e.g. controlsd)") - parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars, + parser.add_argument("--whitelist-cars", type=str, nargs="*", default=None, help="Whitelist given cars from the test (e.g. HONDA)") parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[], help="Blacklist given processes from the test (e.g. controlsd)") @@ -136,91 +1114,283 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[], help="Msgs to ignore (e.g. carEvents)") parser.add_argument("--update-refs", action="store_true", - help="Updates reference logs using current commit") + help="Write candidate reference logs for the selected tasks") + parser.add_argument("--inventory", choices=("active", "staged"), default="active", + help="Case inventory to use (staged includes PENDING_CASES)") + parser.add_argument("--list-cases", action="store_true", help="Print selected source case records as JSON and exit") + parser.add_argument("--list-tasks", action="store_true", help="Print selected canonical task IDs and exit") + parser.add_argument("--output-dir", type=str, default=PROC_REPLAY_DIR, + help="Directory for diff.txt and results.json") + parser.add_argument("--reference-dir", type=str, default=None, + help="Local directory of already-generated candidate refs") parser.add_argument("-j", "--jobs", type=int, default=max(cpu_count - 2, 1), help="Max amount of parallel jobs") - args = parser.parse_args() + args = parser.parse_args(argv) + + cases = select_cases(args.inventory) + assert_inventory_invariants(cases) + tasks = filter_tasks( + build_tasks(cases), + whitelist_procs=args.whitelist_procs if args.whitelist_procs is not None else all_procs, + blacklist_procs=args.blacklist_procs, + whitelist_cars=args.whitelist_cars if args.whitelist_cars is not None else all_cars, + blacklist_cars=args.blacklist_cars, + ) + + if args.list_cases: + print(json.dumps([c.to_record() for c in cases], indent=2, sort_keys=True)) + return 0 + if args.list_tasks: + for task_id in sorted(t.task_id for t in tasks): + print(task_id) + return 0 - tested_procs = set(args.whitelist_procs) - set(args.blacklist_procs) - tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars) - tested_cars = {c.upper() for c in tested_cars} + tested_procs = {t.process for t in tasks} + tested_cars = {t.case.car_brand for t in tasks} + full_test = ( + args.inventory == "active" + and tested_procs == all_procs + and tested_cars == all_cars + and not args.ignore_fields + and not args.ignore_msgs + and args.whitelist_procs is None + and args.whitelist_cars is None + and not args.blacklist_procs + and not args.blacklist_cars + ) + if args.update_refs and args.reference_dir is not None: + raise SystemExit("--update-refs cannot use --reference-dir") - full_test = (tested_procs == all_procs) and (tested_cars == all_cars) and all(len(x) == 0 for x in (args.ignore_fields, args.ignore_msgs)) - os.makedirs(os.path.dirname(FAKEDATA), exist_ok=True) + expected_task_ids = sorted(t.task_id for t in tasks) + settled: dict[str, TaskResult] = {} + cur_commit = "" + ref_commit = "" + artifacts_commit: str | None = None + artifacts_manifest: dict[str, Any] | None = None - if args.update_refs: - assert full_test, "Need to run full test when updating refs" + # Diagnostics are an artifact in their own right. Create them before any + # commit lookup, cleanup, or network operation so setup failures are visible. + os.makedirs(args.output_dir, exist_ok=True) + + def flush() -> dict[str, Any]: + doc = build_results_document( + tested_commit=cur_commit, + ref_commit=ref_commit, + artifacts_commit=artifacts_commit, + inventory=args.inventory, + cases=cases, + expected_task_ids=expected_task_ids, + results=settled, + ) + write_diagnostics(args.output_dir, doc, list(settled.values()), ref_commit) + return doc + + def settle(result: TaskResult) -> dict[str, Any]: + settled[result.task_id] = result + return flush() + + def settle_all(message: str) -> dict[str, Any]: + doc = None + for task in tasks: + if task.task_id in settled: + continue + doc = settle(TaskResult( + status="error", + task_id=task.task_id, + elapsed_s=0.0, + diff_or_traceback=message, + ref_path="", + new_path="", + )) + return doc if doc is not None else flush() + + document = flush() try: - with open(REF_COMMIT_FN) as f: - ref_commit = f.read().strip() - except FileNotFoundError: - ref_commit = URLFile(BASE_URL + "ref_commit", cache=False).read().decode().strip() + cur_commit = validate_ref_commit(get_commit_fn() or "") + # A selected reference directory owns its files and metadata. Never clear + # it as part of preparing the default candidate workspace. + if args.reference_dir is None: + clear_stale_fakedata_fn(FAKEDATA) + except Exception: + document = settle_all(traceback.format_exc()) + print("TEST FAILED") + return exit_code_for(document) + + try: + if args.update_refs: + ref_commit = cur_commit + write_candidate_ref_commit_fn(ref_commit, FAKEDATA) + elif args.reference_dir is not None: + ref_commit, artifacts_commit = resolve_ref_commit_fn( + artifacts_commit, + reference_dir=args.reference_dir, + pin_fn=pin_artifacts_commit_fn, + ) + else: + artifacts_commit = pin_artifacts_commit_fn() + ref_commit, artifacts_commit = resolve_ref_commit_fn( + artifacts_commit, + pin_fn=pin_artifacts_commit_fn, + ) + artifacts_manifest = load_artifacts_manifest_fn( + artifacts_commit, + expected_ref_commit=ref_commit, + selected_task_ids=expected_task_ids, + ) + except Exception: + document = settle_all(traceback.format_exc()) + print("TEST FAILED") + return exit_code_for(document) + + document = flush() - cur_commit = get_commit() - if not cur_commit: - raise Exception("Couldn't get current commit") print(f"***** testing against commit {ref_commit} *****") + if artifacts_commit: + print(f"***** artifacts pinned at {artifacts_commit} *****") - # check to make sure all car brands are tested - if full_test: - untested = (set(interface_names) - set(excluded_interfaces)) - {c.lower() for c in tested_cars} - assert len(untested) == 0, f"Cars missing routes: {str(untested)}" - - log_paths: defaultdict[str, dict[str, dict[str, str]]] = defaultdict(lambda: defaultdict(dict)) - with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool: - download_segments = [seg for car, seg in segments if car in tested_cars] - log_data: dict[str, LogReader] = {} - p1 = pool.map(get_log_data, download_segments) - for segment, lr in tqdm(p1, desc="Getting Logs", total=len(download_segments)): - log_data[segment] = lr - - pool_args: Any = [] - for car_brand, segment in segments: - if car_brand not in tested_cars: - continue + try: + if full_test: + names = interface_names_fn() if interface_names_fn is not None else set(interface_names) + untested = (names - set(excluded_interfaces)) - {c.lower() for c in tested_cars} + if untested: + raise AssertionError(f"Cars missing routes: {str(untested)}") + except Exception: + document = settle_all(traceback.format_exc()) + print("TEST FAILED") + return exit_code_for(document) + # Download each selected executable case once (case_id dedup; never hash ReplayCase). + case_by_id = unique_cases_by_id(tasks) + download_errors: dict[str, str] = {} + log_data: dict[str, bytes] = {} - for cfg in CONFIGS: - if cfg.proc_name not in tested_procs: - continue + download_payloads = [] + for case in case_by_id.values(): + if not case.executable: + download_errors[case.case_id] = f"{case.case_id}: pending case has no executable source" + continue + download_payloads.append({ + "case_id": case.case_id, + "car_brand": case.car_brand, + "source": case.source, + "processes": list(case.processes), + "custom_params": dict(case.custom_params), + "source_sha256": case.source_sha256, + "source_bytes": case.source_bytes, + }) - # to speed things up, we only test all segments on card - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and car_brand not in ('HYUNDAI', 'TOYOTA'): - continue + if download_payloads: + try: + with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool: + future_map = {pool.submit(download_case_by_id_fn, payload): payload for payload in download_payloads} + for fut in tqdm(concurrent.futures.as_completed(future_map), desc="Getting Logs", total=len(future_map)): + payload = future_map[fut] + try: + case_id, data, err = fut.result() + except Exception: + case_id, data, err = payload["case_id"], None, traceback.format_exc() + if err is not None or data is None: + download_errors[case_id] = err or "download failed" + else: + log_data[case_id] = data + except Exception: + setup_error = traceback.format_exc() + for payload in download_payloads: + download_errors.setdefault(payload["case_id"], setup_error) - cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) - if args.update_refs: # reference logs will not exist if routes were just regenerated - route, seg_num = segment.rsplit("--", 1) - ref_log_path = get_url(route, seg_num, "rlog.zst") - else: - ref_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{ref_commit}.zst".replace("|", "_")) - ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn) + # Materialize download failures as settled task errors before workers run. + for task in tasks: + if task.case.case_id in download_errors: + try: + failed_new_path = new_log_path(task, cur_commit) + except Exception: + failed_new_path = "" + document = settle(TaskResult( + status="error", + task_id=task.task_id, + elapsed_s=0.0, + diff_or_traceback=download_errors[task.case.case_id], + ref_path="", + new_path=failed_new_path, + )) - pool_args.append((segment, cfg, args, cur_log_fn, ref_log_path, log_data[segment])) - log_paths[segment][cfg.proc_name]['ref'] = ref_log_path - log_paths[segment][cfg.proc_name]['new'] = cur_log_fn + payloads: list[dict[str, Any]] = [] + for task in tasks: + if task.task_id in settled: + continue + ref_path = "" + new_path = "" + try: + ref_path = resolve_ref_path( + task, + ref_commit=ref_commit, + artifacts_commit=artifacts_commit, + reference_dir=args.reference_dir, + update_refs=args.update_refs, + artifacts_manifest=artifacts_manifest, + pin_fn=pin_artifacts_commit_fn, + ) + new_path = new_log_path(task, cur_commit) + payloads.append(make_task_payload( + task, + lr_dat=log_data[task.case.case_id], + ref_log_path=ref_path, + new_log_path=new_path, + ignore_fields=args.ignore_fields, + ignore_msgs=args.ignore_msgs, + update_refs=args.update_refs, + )) + except Exception: + document = settle(TaskResult( + status="error", + task_id=task.task_id, + elapsed_s=0.0, + diff_or_traceback=traceback.format_exc(), + ref_path=ref_path, + new_path=new_path, + )) + + def on_settled(result: TaskResult) -> None: + settle(result) + try: + pooled_results = run_task_pool_fn(payloads, jobs=args.jobs, on_settled=on_settled) + for result in pooled_results: + settled.setdefault(result.task_id, result) + missing_after_pool = set(expected_task_ids) - set(settled) + for task_id in missing_after_pool: + settled[task_id] = TaskResult( + status="error", + task_id=task_id, + elapsed_s=0.0, + diff_or_traceback="worker pool completed without a result", + ref_path="", + new_path="", + ) + except Exception: + document = settle_all(traceback.format_exc()) + print("TEST FAILED") + return exit_code_for(document) - results: Any = defaultdict(dict) - p2 = pool.map(run_test_process, pool_args) - for (segment, proc, result) in tqdm(p2, desc="Running Tests", total=len(pool_args)): - results[segment][proc] = result + document = flush() - diff_short, diff_long, failed = format_diff(results, log_paths, ref_commit) if not args.update_refs: - with open(os.path.join(PROC_REPLAY_DIR, "diff.txt"), "w") as f: - f.write(diff_long) + retain_failed_logs(settled.values()) + diff_short, _, failed = format_diff_text(list(settled.values()), ref_commit) print(diff_short) - - if failed: + if exit_code_for(document): print("TEST FAILED") else: print("TEST SUCCEEDED") - else: - with open(REF_COMMIT_FN, "w") as f: - f.write(cur_commit) - print(f"\n\nUpdated reference logs for commit: {cur_commit}") + # Ensure candidate tree remains reusable even if workers already wrote logs. + write_candidate_ref_commit_fn(cur_commit, FAKEDATA) + print(f"\n\nWrote candidate reference logs for commit: {cur_commit}") + # Candidate-only: do not claim acceptance by rewriting REF_COMMIT_FN. + + return exit_code_for(document) - sys.exit(int(failed)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/selfdrive/test/process_replay/test_processes_unit.py b/selfdrive/test/process_replay/test_processes_unit.py new file mode 100755 index 00000000000000..a6153ced925887 --- /dev/null +++ b/selfdrive/test/process_replay/test_processes_unit.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +import concurrent.futures +import json +import os +import tempfile +import pickle +from pathlib import Path +import pytest + +from openpilot.common.params import ParamKeyType, Params +from openpilot.common.prefix import OpenpilotPrefix +from openpilot.selfdrive.test.process_replay import test_processes as tp + + +def test_active_inventory_counts(): + cases = tp.select_cases("active") + tasks = tp.build_tasks(cases) + assert len(cases) == 16 + assert len(tasks) == 66 + assert len({c.case_id for c in cases}) == 16 + assert len({t.task_id for t in tasks}) == 66 + assert len(tp.shard_task_ids(tasks, "card")) == 16 + assert len(tp.shard_task_ids(tasks, "controlsd")) == 16 + assert len(tp.shard_task_ids(tasks, "lagd")) == 16 + assert len(tp.shard_task_ids(tasks, "other")) == 18 + tp.assert_inventory_invariants(cases) + + +def test_staged_inventory_counts_and_pending_not_executable(): + cases = tp.select_cases("staged") + tasks = tp.build_tasks(cases) + assert len(cases) == 18 + assert len(tasks) == 78 + assert len(tp.PENDING_CASES) == 2 + assert all(not c.executable for c in tp.PENDING_CASES) + assert len(tp.shard_task_ids(tasks, "card")) == 18 + assert len(tp.shard_task_ids(tasks, "controlsd")) == 18 + assert len(tp.shard_task_ids(tasks, "lagd")) == 18 + assert len(tp.shard_task_ids(tasks, "other")) == 24 + union: set[str] = set() + for shard in ("card", "controlsd", "lagd", "other"): + ids = tp.shard_task_ids(tasks, shard) + assert union.isdisjoint(ids) + union.update(ids) + assert union == {t.task_id for t in tasks} + tp.assert_inventory_invariants(cases) + + +def test_source_segments_compat(): + assert len(tp.source_segments) == 17 + assert tp.source_segments[0][0] == "HYUNDAI" + brands = [c for c, _ in tp.source_segments] + assert "TOYOTA2" in brands + assert len(tp.segments) == 16 + assert "TOYOTA2" not in [c for c, _ in tp.segments] + + +def test_filter_whitelist_blacklist_semantics(): + tasks = tp.build_tasks(tp.ACTIVE_CASES) + only_card = tp.filter_tasks(tasks, whitelist_procs=["card"], blacklist_procs=[], whitelist_cars=None, blacklist_cars=[]) + assert {t.process for t in only_card} == {"card"} + assert len(only_card) == 16 + + no_core = tp.filter_tasks( + tasks, + whitelist_procs=None, + blacklist_procs=["card", "controlsd", "lagd"], + whitelist_cars=None, + blacklist_cars=[], + ) + assert len(no_core) == 18 + assert {"card", "controlsd", "lagd"}.isdisjoint({t.process for t in no_core}) + + honda = tp.filter_tasks(tasks, whitelist_procs=["card"], blacklist_procs=[], whitelist_cars=["honda"], blacklist_cars=[]) + assert {t.case.car_brand for t in honda} == {"HONDA"} + assert [t.task_id for t in honda] == ["honda:card"] + + +def test_empty_and_fixed_digests(): + assert tp.params_digest({}) == tp.EMPTY_PARAMS_DIGEST + assert tp.params_digest(tp.NAP_PREAP_PEDAL_PARAMS) == tp.NAP_PREAP_PEDAL_PARAMS_DIGEST + assert tp.params_digest(tp.NAP_PREAP_NO_PEDAL_PARAMS) == tp.NAP_PREAP_NO_PEDAL_PARAMS_DIGEST + manifest = tp.canonical_params_manifest(tp.NAP_PREAP_PEDAL_PARAMS) + assert manifest.endswith(b"\n") + assert b" " not in manifest + + +def test_header_type_validation_and_rejection(): + tp.validate_params_against_header(tp.NAP_PREAP_PEDAL_PARAMS) + with pytest.raises(TypeError): + tp.validate_params_against_header({"NAPFollowDistance": 1.5}) + with pytest.raises(TypeError): + tp.validate_params_against_header({"NAPBrakeFactor": 1}) + with pytest.raises(TypeError): + tp.validate_params_against_header({"NAPForcePreAP": 1}) + + +def test_params_round_trip_native_types(): + with OpenpilotPrefix(): + params = Params() + for key, value in tp.NAP_PREAP_PEDAL_PARAMS.items(): + declared = params.get_type(key) + if declared == ParamKeyType.BOOL: + assert isinstance(value, bool) + params.put_bool(key, value) + assert params.get_bool(key) is value + elif declared == ParamKeyType.INT: + assert isinstance(value, int) + assert not isinstance(value, bool) + params.put(key, value) + assert params.get(key) == value + elif declared == ParamKeyType.FLOAT: + assert isinstance(value, float) + params.put(key, value) + assert params.get(key) == value + else: + raise AssertionError(f"unexpected type for {key}: {declared}") + + +def test_ref_filename_and_case_id_validation(): + sha = "a" * 40 + assert tp.ref_filename("hyundai", "card", sha) == f"hyundai__card__{sha}.zst" + with pytest.raises(ValueError): + tp.ref_filename("HYUNDAI", "card", sha) + with pytest.raises(ValueError): + tp.validate_case_id("bad_id") + with pytest.raises(ValueError): + tp.ref_filename("hyundai", "card", "abc") + + +def test_pin_and_artifacts_url_require_full_sha(): + class FakeResp: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self, *args, **kwargs): + return json.dumps({"object": {"sha": "a" * 40}}).encode() + + sha = tp.pin_artifacts_commit(opener=lambda *a, **k: FakeResp()) + assert sha == "a" * 40 + url = tp.artifacts_file_url(sha, "ref_commit") + assert url == f"https://raw.githubusercontent.com/NotAutopilot/ci-artifacts/{sha}/ref_commit" + with pytest.raises(ValueError): + tp.artifacts_file_url("process-replay", "ref_commit") + + +def test_resolve_ref_path_prefers_local_new_name_then_legacy(): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + ref = "d" * 40 + with tempfile.TemporaryDirectory() as td: + new_name = tp.ref_filename(case.case_id, "card", ref) + legacy = tp.legacy_ref_filename(case.source, "card", ref) + Path(td, new_name).write_bytes(b"new") + path = tp.resolve_ref_path(task, ref_commit=ref, artifacts_commit="b" * 40, reference_dir=td, update_refs=False) + assert Path(path).resolve() == (Path(td) / new_name).resolve() + os.remove(os.path.join(td, new_name)) + Path(td, legacy).write_bytes(b"legacy") + path = tp.resolve_ref_path(task, ref_commit=ref, artifacts_commit="b" * 40, reference_dir=td, update_refs=False) + assert Path(path).resolve() == (Path(td) / legacy).resolve() + + +def test_resolve_ref_path_remote_legacy_for_route_cases(monkeypatch): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + commit = "c" * 40 + ref = "d" * 40 + monkeypatch.setattr(tp, "LEGACY_ARTIFACTS_COMMITS", frozenset({commit})) + path = tp.resolve_ref_path(task, ref_commit=ref, artifacts_commit=commit, reference_dir=None, update_refs=False) + legacy = tp.legacy_ref_filename(case.source, "card", ref) + assert path == tp.artifacts_file_url(commit, legacy) + + +def test_resolve_ref_path_rejects_missing_manifest_outside_legacy(): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + with pytest.raises(ValueError, match="LEGACY_ARTIFACTS_COMMITS"): + tp.resolve_ref_path( + task, + ref_commit="d" * 40, + artifacts_commit="c" * 40, + reference_dir=None, + update_refs=False, + artifacts_manifest=None, + ) + + +def test_incremental_diagnostics_and_nonzero_incomplete(): + with tempfile.TemporaryDirectory() as td: + expected = ["hyundai:card", "hyundai:controlsd"] + settled = { + "hyundai:card": tp.TaskResult("pass", "hyundai:card", 0.1, "", "ref", "new", {"carState": 1}), + } + doc = tp.build_results_document( + tested_commit="t" * 40, + ref_commit="a" * 40, + artifacts_commit="a" * 40, + inventory="active", + cases=tp.ACTIVE_CASES[:1], + expected_task_ids=expected, + results=settled, + ) + assert doc["schema_version"] == tp.RESULTS_SCHEMA_VERSION + assert doc["status_counts"]["incomplete"] == 1 + assert tp.exit_code_for(doc) == 1 + tp.write_diagnostics(td, doc, list(settled.values()), "r" * 40) + assert os.path.exists(os.path.join(td, "results.json")) + assert os.path.exists(os.path.join(td, "diff.txt")) + loaded = json.loads(Path(td, "results.json").read_text()) + assert loaded["incomplete_task_ids"] == ["hyundai:controlsd"] + + +def test_injected_worker_error_then_later_success(): + payloads = [ + {"task_id": "a:card", "ref_log_path": "", "new_log_path": ""}, + {"task_id": "b:card", "ref_log_path": "", "new_log_path": ""}, + {"task_id": "c:card", "ref_log_path": "", "new_log_path": ""}, + ] + + def worker(payload): + if payload["task_id"] == "a:card": + raise RuntimeError("injected worker boom") + return tp.TaskResult( + status="pass", + task_id=payload["task_id"], + elapsed_s=0.01, + diff_or_traceback="", + ref_path="", + new_path="", + message_counts={}, + ) + + settled: list[tp.TaskResult] = [] + results = tp.run_task_pool( + payloads, + jobs=2, + worker=worker, + on_settled=settled.append, + executor_factory=concurrent.futures.ThreadPoolExecutor, + ) + by_id = {r.task_id: r for r in results} + assert set(by_id) == {"a:card", "b:card", "c:card"} + assert by_id["a:card"].status == "error" + assert "injected worker boom" in by_id["a:card"].diff_or_traceback + assert by_id["b:card"].status == "pass" + assert by_id["c:card"].status == "pass" + assert len(settled) == 3 + + +def test_run_test_process_serializes_exceptions(): + payload = { + "task_id": "hyundai:card", + "case_id": "hyundai", + "car_brand": "HYUNDAI", + "source": "not-a-real-segment|--0", + "processes": ["card"], + "custom_params": {}, + "process": "card", + "lr_dat": b"not-a-log", + "ref_log_path": "", + "new_log_path": os.path.join(tempfile.gettempdir(), "nap-replay-unit-new.zst"), + "ignore_fields": [], + "ignore_msgs": [], + "update_refs": True, + } + result = tp.run_test_process(payload) + assert isinstance(result, tp.TaskResult) + assert result.status == "error" + assert result.task_id == "hyundai:card" + assert result.diff_or_traceback + + +def test_download_digest_mismatch_is_error(monkeypatch): + case = tp.ReplayCase( + case_id="fixture-case", + car_brand="TESLA", + source="https://example.invalid/rlog.zst", + processes=("card",), + source_sha256="0" * 64, + source_bytes=4, + ) + + class FakeFile: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self, *args, **kwargs): + return b"abcd" + + monkeypatch.setattr(tp, "FileReader", lambda *a, **k: FakeFile()) + with pytest.raises(ValueError, match="sha256 mismatch"): + tp.download_case_bytes(case) + + +def test_duplicate_case_id_rejected(): + case = tp.ACTIVE_CASES[0] + with pytest.raises(AssertionError): + tp.assert_inventory_invariants([case, case]) + +def test_case_dedup_does_not_hash_custom_params(): + case = tp.ReplayCase( + case_id="dedup-case", + car_brand="TESLA", + source="", + processes=("card",), + custom_params={"NAPForcePreAP": True}, + ) + tasks = [tp.ReplayTask(case=case, process="card"), tp.ReplayTask(case=case, process="card")] + deduped = tp.unique_cases_by_id(tasks) + assert deduped == {"dedup-case": case} + + +def test_executable_source_metadata_is_exact(): + with pytest.raises(ValueError, match="requires both"): + tp.ReplayCase(case_id="missing-metadata", car_brand="TESLA", source="https://example.invalid/rlog.zst", + processes=("card",)) + with pytest.raises(ValueError, match="omit"): + tp.ReplayCase(case_id="pending-metadata", car_brand="TESLA", source="", + processes=("card",), source_bytes=1, source_sha256="0" * 64) + + +def test_reference_dir_owns_ref_commit_and_candidate_metadata(): + with tempfile.TemporaryDirectory() as td: + ref_commit = "a" * 40 + tp.write_candidate_ref_commit(ref_commit, td) + assert tp.resolve_ref_commit(reference_dir=td) == (ref_commit, None) + assert Path(td, "ref_commit").read_text() == f"{ref_commit}\n" + + +def test_manifest_requires_canonical_remote_filename(): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + with pytest.raises(ValueError, match="non-canonical filename"): + tp.resolve_ref_path( + task, + ref_commit="a" * 40, + artifacts_commit="a" * 40, + reference_dir=None, + update_refs=False, + artifacts_manifest={"refs": {task.task_id: {"filename": "legacy.zst"}}}, + ) + + +def test_main_records_all_tasks_when_setup_fails(): + with tempfile.TemporaryDirectory() as td: + output_dir = os.path.join(td, "diagnostics") + result = tp.main( + ["--whitelist-procs", "card", "--output-dir", output_dir], + get_commit_fn=lambda: (_ for _ in ()).throw(RuntimeError("setup failed")), + clear_stale_fakedata_fn=lambda *_: pytest.fail("cleanup must follow initial diagnostics"), + ) + assert result == 1 + document = json.loads(Path(output_dir, "results.json").read_text()) + assert document["status_counts"]["error"] == 16 + assert document["status_counts"]["incomplete"] == 0 + +def test_generated_log_is_saved_before_reference_read(monkeypatch): + from types import SimpleNamespace + + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + cfg = SimpleNamespace(proc_name="card", pubs=set(), subs={"testMsg"}, ignore=[], tolerance=0) + message = SimpleNamespace(which=lambda: "testMsg") + saved: list[str] = [] + monkeypatch.setattr(tp, "replay_process", lambda *args, **kwargs: [message]) + monkeypatch.setattr(tp, "check_most_messages_valid", lambda messages: True) + monkeypatch.setattr(tp, "save_log", lambda path, messages: saved.append(path)) + monkeypatch.setattr(tp, "load_ref_log_messages", lambda path: (_ for _ in ()).throw(OSError("missing ref"))) + + status, detail, _ = tp.test_process(cfg, [], task, "/missing/ref.zst", "/candidate/new.zst") + assert status == "error" + assert "missing ref" in detail + assert saved == ["/candidate/new.zst"] + + +def test_task_result_is_pickleable(): + result = tp.TaskResult("pass", "case:card", 1.25, "", "/ref.zst", "/new.zst", {"carState": 3}) + restored = pickle.loads(pickle.dumps(result)) + assert restored == result + assert restored.to_record()["message_counts"] == {"carState": 3} + + +def test_validate_ref_commit_rejects_malformed_identities(): + good = "ab" * 20 + assert tp.validate_ref_commit(good) == good + for bad in ("abc", "A" * 40, "ab" * 19, "ab" * 21, "g" * 40, "a" * 39 + "/", "a" * 39 + "?", "a" * 39 + "#", "a" * 39 + '\n', "a" * 39 + " "): + with pytest.raises(ValueError): + tp.validate_ref_commit(bad) + + +def test_same_dir_collision_prefers_canonical_over_legacy(): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + ref = "e" * 40 + with tempfile.TemporaryDirectory() as td: + new_name = tp.ref_filename(case.case_id, "card", ref) + legacy = tp.legacy_ref_filename(case.source, "card", ref) + Path(td, new_name).write_bytes(b"canonical") + Path(td, legacy).write_bytes(b"legacy") + path = tp.resolve_ref_path(task, ref_commit=ref, artifacts_commit="b" * 40, reference_dir=td, update_refs=False) + assert Path(path).name == new_name + assert Path(path).read_bytes() == b"canonical" + + +def test_symlink_and_outside_local_refs_are_rejected(): + case = tp.ACTIVE_CASES[0] + task = tp.ReplayTask(case=case, process="card") + ref = "f" * 40 + with tempfile.TemporaryDirectory() as td: + root = Path(td) + outside = root / "outside.txt" + outside.write_text("secret" + '\n', encoding="utf-8") + (root / "ref_commit").symlink_to(outside) + with pytest.raises(ValueError, match="symlink"): + tp.resolve_ref_commit(reference_dir=str(root)) + (root / "ref_commit").unlink() + (root / "ref_commit").write_text(ref + '\n', encoding="utf-8") + new_name = tp.ref_filename(case.case_id, "card", ref) + (root / new_name).symlink_to(outside) + with pytest.raises(ValueError, match="symlink"): + tp.resolve_ref_path(task, ref_commit=ref, artifacts_commit="b" * 40, reference_dir=str(root), update_refs=False) + (root / new_name).unlink() + with pytest.raises(ValueError, match="basename"): + tp.resolve_regular_member(str(root), "../outside.txt") + + +def test_per_task_ref_failure_continues_other_tasks(monkeypatch): + with tempfile.TemporaryDirectory() as td: + output_dir = os.path.join(td, "out") + ref_dir = os.path.join(td, "refs") + os.makedirs(ref_dir) + Path(ref_dir, "ref_commit").write_text(("a" * 40) + '\n', encoding="utf-8") + + real_resolve = tp.resolve_ref_path + + def fake_resolve(task, **kwargs): + if task.process == "controlsd": + raise ValueError(f"injected ref failure for {task.task_id}") + return real_resolve(task, **kwargs) + + monkeypatch.setattr(tp, "resolve_ref_path", fake_resolve) + + def fake_pool(payloads, jobs, on_settled=None, **kwargs): + results = [] + for payload in payloads: + result = tp.TaskResult("pass", payload["task_id"], 0.01, "", payload.get("ref_log_path", ""), payload.get("new_log_path", ""), {}) + results.append(result) + if on_settled is not None: + on_settled(result) + return results + + case = tp.ACTIVE_CASES[0] + + def fake_download(payload): + return payload["case_id"], b"not-used", None + + monkeypatch.setattr(tp.concurrent.futures, "ProcessPoolExecutor", concurrent.futures.ThreadPoolExecutor) + + code = tp.main( + ["--whitelist-procs", "card", "controlsd", "--whitelist-cars", case.car_brand, + "--output-dir", output_dir, "--reference-dir", ref_dir, "-j", "1"], + get_commit_fn=lambda: "b" * 40, + download_case_by_id_fn=fake_download, + run_task_pool_fn=fake_pool, + clear_stale_fakedata_fn=lambda *_: None, + ) + assert code == 1 + document = json.loads(Path(output_dir, "results.json").read_text()) + by_id = {row["task_id"]: row for row in document["results"]} + assert by_id[f"{case.case_id}:controlsd"]["status"] == "error" + assert "injected ref failure" in by_id[f"{case.case_id}:controlsd"]["diff_or_traceback"] + assert by_id[f"{case.case_id}:card"]["status"] == "pass" + + +def test_on_settled_flushes_diagnostics_each_callback(monkeypatch): + with tempfile.TemporaryDirectory() as td: + output_dir = os.path.join(td, "out") + snapshots = [] + + def fake_pool(payloads, jobs, on_settled=None, **kwargs): + assert on_settled is not None + first = tp.TaskResult("pass", payloads[0]["task_id"], 0.01, "", "", "", {}) + on_settled(first) + snapshots.append({row["task_id"] for row in json.loads(Path(output_dir, "results.json").read_text())["results"]}) + assert Path(output_dir, "diff.txt").exists() + second = tp.TaskResult("pass", payloads[1]["task_id"], 0.01, "", "", "", {}) + on_settled(second) + snapshots.append({row["task_id"] for row in json.loads(Path(output_dir, "results.json").read_text())["results"]}) + return [first, second] + + case = tp.ACTIVE_CASES[0] + monkeypatch.setattr(tp.concurrent.futures, "ProcessPoolExecutor", concurrent.futures.ThreadPoolExecutor) + code = tp.main( + ["--whitelist-procs", "card", "controlsd", "--whitelist-cars", case.car_brand, + "--output-dir", output_dir, "--update-refs", "-j", "1"], + get_commit_fn=lambda: "c" * 40, + download_case_by_id_fn=lambda payload: (payload["case_id"], b"abc", None), + run_task_pool_fn=fake_pool, + clear_stale_fakedata_fn=lambda *_: None, + write_candidate_ref_commit_fn=lambda *_: "", + ) + assert code == 0 + assert len(snapshots[0]) == 1 + assert len(snapshots[1]) == 2 + assert snapshots[0].issubset(snapshots[1]) + + +def test_manifest_missing_on_non_legacy_is_error(): + class Missing: + def __init__(self, *a, **k): + pass + + def read(self, *a, **k): + raise tp.URLFileException("Remote file is empty or doesn't exist: x") + + with pytest.raises(ValueError, match="non-legacy"): + tp.load_artifacts_manifest("a" * 40, expected_ref_commit="b" * 40, url_file_cls=Missing) + + +def test_manifest_empty_body_is_malformed_not_legacy(monkeypatch): + monkeypatch.setattr(tp, "LEGACY_ARTIFACTS_COMMITS", frozenset({"a" * 40})) + + class Empty: + def __init__(self, *a, **k): + pass + + def read(self, *a, **k): + return b"" + + with pytest.raises(ValueError, match="empty body"): + tp.load_artifacts_manifest("a" * 40, expected_ref_commit="b" * 40, url_file_cls=Empty) diff --git a/third_party/acados/build.sh b/third_party/acados/build.sh index 95b3913c4a8f4f..668f9fd79d24ce 100755 --- a/third_party/acados/build.sh +++ b/third_party/acados/build.sh @@ -13,52 +13,115 @@ if [ -f /TICI ]; then BLAS_TARGET="ARMV8A_ARM_CORTEX_A57" fi -ACADOS_FLAGS="-DACADOS_WITH_QPOASES=ON -UBLASFEO_TARGET -DBLASFEO_TARGET=$BLAS_TARGET" +ACADOS_FLAGS=( + "-DACADOS_WITH_QPOASES=ON" + "-UBLASFEO_TARGET" + "-DBLASFEO_TARGET=$BLAS_TARGET" +) + +if [[ "$OSTYPE" != "darwin"* ]]; then + ACADOS_FLAGS+=( + "-DCMAKE_C_FLAGS=-D_POSIX_C_SOURCE=200112L -include stdlib.h -Wno-error=incompatible-pointer-types" + "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-z,noexecstack" + "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-z,noexecstack" + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" + ) +fi if [[ "$OSTYPE" == "darwin"* ]]; then - ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_MACOSX_RPATH=1" + ACADOS_FLAGS+=( + "-DCMAKE_OSX_ARCHITECTURES=arm64" + "-DCMAKE_MACOSX_RPATH=1" + ) ARCHNAME="Darwin" fi -if [ ! -d acados_repo/ ]; then - git clone https://github.com/acados/acados.git $DIR/acados_repo - # git clone https://github.com/commaai/acados.git $DIR/acados_repo +if [ ! -d "$DIR/acados_repo/" ]; then + git clone https://github.com/acados/acados.git "$DIR/acados_repo" + # git clone https://github.com/commaai/acados.git "$DIR/acados_repo" fi -cd acados_repo +cd "$DIR/acados_repo" git fetch --all git checkout 8af9b0ad180940ef611884574a0b27a43504311d # v0.2.2 git submodule update --depth=1 --recursive --init -# build mkdir -p build cd build -cmake $ACADOS_FLAGS .. +cmake "${ACADOS_FLAGS[@]}" .. make -j20 install INSTALL_DIR="$DIR/$ARCHNAME" -rm -rf $INSTALL_DIR -mkdir -p $INSTALL_DIR +rm -rf "$INSTALL_DIR" +mkdir -p "$INSTALL_DIR" -rm $DIR/acados_repo/lib/*.json +rm "$DIR/acados_repo/lib/"*.json -rm -rf $DIR/include $DIR/acados_template -cp -r $DIR/acados_repo/include $DIR -cp -r $DIR/acados_repo/lib $INSTALL_DIR -cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/ -#pip3 install -e $DIR/acados/interfaces/acados_template +rm -rf "$DIR/include" "$DIR/acados_template" +cp -r "$DIR/acados_repo/include" "$DIR" +cp -r "$DIR/acados_repo/lib" "$INSTALL_DIR" +cp -r "$DIR/acados_repo/interfaces/acados_template/acados_template" "$DIR" + +if [[ "$OSTYPE" != "darwin"* ]]; then + emitted_libraries_file="$(mktemp)" + trap 'rm -f "$emitted_libraries_file"' EXIT + if find "$INSTALL_DIR" -type f \( -name '*.so' -o -name '*.so.*' \) -print0 >"$emitted_libraries_file"; then + : + else + echo "failed to discover acados shared libraries" >&2 + exit 1 + fi + mapfile -d '' emitted_libraries < "$emitted_libraries_file" + rm -f "$emitted_libraries_file" + trap - EXIT + if (( ${#emitted_libraries[@]} == 0 )); then + echo "acados build emitted no shared libraries" >&2 + exit 1 + fi + for library in "${emitted_libraries[@]}"; do + if readelf_output="$(readelf -lW "$library")"; then + : + else + echo "readelf failed for $library" >&2 + exit 1 + fi + if stack_status="$(awk '$1 == "GNU_STACK" { + found = 1 + for (i = 2; i <= NF; i++) { + if ($i ~ /^[RWE]+$/) { + if ($i ~ /E/) exit 2 + exit 0 + } + } + exit 3 + } + END { + if (!found) exit 3 + }' <<< "$readelf_output")"; then + : + else + status=$? + if (( status == 2 )); then + echo "executable GNU_STACK in $library" >&2 + else + echo "missing or malformed GNU_STACK in $library" >&2 + fi + exit 1 + fi + done +fi # skip macOS - sed is different :/ if [[ "$OSTYPE" != "darwin"* ]]; then # strip future_fstrings to avoid having to install the compatibility package - find $DIR/acados_template/ -type f -exec sed -i '/future.fstrings/d' {} + + find "$DIR/acados_template/" -type f -exec sed -i '/future.fstrings/d' {} + fi # build tera -cd $DIR/acados_repo/interfaces/acados_template/tera_renderer/ +cd "$DIR/acados_repo/interfaces/acados_template/tera_renderer/" if [[ "$OSTYPE" == "darwin"* ]]; then cargo build --verbose --release --target aarch64-apple-darwin cp target/aarch64-apple-darwin/release/t_renderer target/release/t_renderer else cargo build --verbose --release fi -cp target/release/t_renderer $INSTALL_DIR/ +cp target/release/t_renderer "$INSTALL_DIR/" diff --git a/third_party/acados/tests/test_noexecstack.py b/third_party/acados/tests/test_noexecstack.py new file mode 100644 index 00000000000000..b576adec673ba9 --- /dev/null +++ b/third_party/acados/tests/test_noexecstack.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + + +ACADOS_ROOT = Path(__file__).resolve().parents[1] + + +def _libraries(arch: str) -> list[Path]: + lib_dir = ACADOS_ROOT / arch / "lib" + if not lib_dir.is_dir(): + return [] + return sorted( + path for path in lib_dir.iterdir() + if path.is_file() and (path.name.endswith(".so") or ".so." in path.name) + ) + + +def _stack_flags(library: Path) -> str: + result = subprocess.run( + ["readelf", "-lW", str(library)], + check=True, + capture_output=True, + text=True, + ) + for line in result.stdout.splitlines(): + fields = line.split() + if fields and fields[0] == "GNU_STACK": + for field in fields[1:]: + if set(field) <= set("RWE"): + return field + break + pytest.fail(f"{library} has no GNU_STACK program header") + + +@pytest.mark.parametrize("arch", ("x86_64", "larch64")) +def test_tracked_libraries_request_non_executable_stack(arch: str) -> None: + libraries = _libraries(arch) + assert libraries, f"{arch} has no tracked acados libraries" + for library in libraries: + assert "E" not in _stack_flags(library), f"{library} requests an executable stack" diff --git a/third_party/acados/tests/test_solver_smoke.py b/third_party/acados/tests/test_solver_smoke.py new file mode 100644 index 00000000000000..ef28b8ff82d33a --- /dev/null +++ b/third_party/acados/tests/test_solver_smoke.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from cereal import messaging +from openpilot.selfdrive.controls.lib.drive_helpers import CAR_ROTATION_RADIUS +from openpilot.selfdrive.controls.lib.lateral_mpc_lib.lat_mpc import LateralMpc, N as LAT_MPC_N +from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import LongitudinalMpc + + +def test_lateral_mpc_solves_zero_reference() -> None: + mpc = LateralMpc() + mpc.set_weights(1.0, 0.1, 0.0, 0.05, 800.0) + mpc.run( + np.zeros(4), + np.column_stack((np.full(LAT_MPC_N + 1, 30.0), + np.full(LAT_MPC_N + 1, CAR_ROTATION_RADIUS))), + np.zeros(LAT_MPC_N + 1), + np.zeros(LAT_MPC_N + 1), + np.zeros(LAT_MPC_N + 1), + ) + assert mpc.solution_status == 0 + + +def test_longitudinal_mpc_solves_zero_reference() -> None: + mpc = LongitudinalMpc() + + def fail_reset() -> None: + pytest.fail("LongitudinalMpc.reset() was invoked during update") + + mpc.reset = fail_reset + radar_state = messaging.new_message("radarState").radarState + mpc.update(radar_state, 0.0, 1.0) + assert mpc.solution_status == 0 diff --git a/third_party/acados/x86_64/lib/libacados.so b/third_party/acados/x86_64/lib/libacados.so old mode 100644 new mode 100755 index 50d647a8621fdd..310e5c667fc241 --- a/third_party/acados/x86_64/lib/libacados.so +++ b/third_party/acados/x86_64/lib/libacados.so @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05a1ba3cf37fa929cdd56f892608b2f89c35a05ef1b07fedb86b2f0d76607263 -size 540488 +oid sha256:e5aa31ab510a3abd857c8bed05eba3bf7a9c1492a5d3af16a4a661f1a2594362 +size 524840 diff --git a/third_party/acados/x86_64/lib/libblasfeo.so b/third_party/acados/x86_64/lib/libblasfeo.so old mode 100644 new mode 100755 index a98f45abd2e47c..86f2b99977a1e5 --- a/third_party/acados/x86_64/lib/libblasfeo.so +++ b/third_party/acados/x86_64/lib/libblasfeo.so @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c0bf22898d9c59b672d3d0961f5f4c804b9957478125d99eb297de3091bedd15 -size 2416112 +oid sha256:bf9f20d8a35a9c6d950baa2dacddbde79a27802c6333b932e4dc861db5f25b55 +size 1826800 diff --git a/third_party/acados/x86_64/lib/libhpipm.so b/third_party/acados/x86_64/lib/libhpipm.so old mode 100644 new mode 100755 index f40cb487cd7cfa..248a52dae5e7f7 --- a/third_party/acados/x86_64/lib/libhpipm.so +++ b/third_party/acados/x86_64/lib/libhpipm.so @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5b6875fb47940764d4ebb916c2373cb0e04929229feb654b290676c28d48fa9d -size 1531024 +oid sha256:ca818463ac47da7eb4cb8a504d19b913e9882dcd338f864faa98a47c6e3791fc +size 1547672 diff --git a/third_party/acados/x86_64/lib/libqpOASES_e.so.3.1 b/third_party/acados/x86_64/lib/libqpOASES_e.so.3.1 old mode 100644 new mode 100755 index 81afd059f7ea12..2057e5676fbbbc --- a/third_party/acados/x86_64/lib/libqpOASES_e.so.3.1 +++ b/third_party/acados/x86_64/lib/libqpOASES_e.so.3.1 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:04be908c3f707e5c968022b9cdd79ab75ae7af46e7fa019ceee98f854ddd3f64 -size 262464 +oid sha256:964de7e4102fc72066df5acedff8d0a6f274f7176f85836697091346e6ffdd9d +size 270464 diff --git a/third_party/acados/x86_64/t_renderer b/third_party/acados/x86_64/t_renderer index d41f6c37255482..a083dfe6184e0a 100755 --- a/third_party/acados/x86_64/t_renderer +++ b/third_party/acados/x86_64/t_renderer @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a53ae46650c4df5b0ddb87a658f59a0422e41743e8bc2d822da0aefd1d280791 -size 5088536 +oid sha256:58b8e3ee9943ad2730604e6e61591afbfa880a7c6055581758bac56b7d92575f +size 5404496 diff --git a/tools/cabana/SConscript b/tools/cabana/SConscript index 1f7ba3eaef79d2..0b0594bb62a41a 100644 --- a/tools/cabana/SConscript +++ b/tools/cabana/SConscript @@ -77,8 +77,8 @@ cabana_env = qt_env.Clone() if arch == "Darwin": cabana_env['CPPPATH'] += [f"{brew_prefix}/include"] cabana_env['LIBPATH'] += [f"{brew_prefix}/lib"] - -cabana_libs = [cereal, messaging, visionipc, replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'bz2', 'zstd', 'yuv', 'usb-1.0'] + qt_libs +ffmpeg_platform_libs = [] if arch == "Darwin" else ['va-drm', 'va', 'drm'] +cabana_libs = [cereal, messaging, visionipc, replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil'] + ffmpeg_platform_libs + ['x264', 'z', 'bz2', 'zstd', 'yuv', 'usb-1.0'] + qt_libs opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % (cabana_env.Dir("../../opendbc/dbc").abspath) cabana_env['CXXFLAGS'] += [opendbc_path] diff --git a/tools/ci/Dockerfile.replay b/tools/ci/Dockerfile.replay new file mode 100644 index 00000000000000..0f36ba2f12bb50 --- /dev/null +++ b/tools/ci/Dockerfile.replay @@ -0,0 +1,74 @@ +# Digest-pinned base for trusted private-runner replay jobs. +# Build/runtime packages install as root; the image default USER is 1001:1001 with no sudo. +# Source is NOT COPY'd: nap-replay-job bind-mounts / ephemeral /src (not executed on the host). +# Because there is no sudo, one-shot setup_dependencies / uv sync must be started as root by +# the launcher (podman --user 0), then replay drops back to --user 1001:1001 --network=none. +# Python deps land in the mounted /src/.venv (VIRTUAL_ENV=/src/.venv); they are not baked in. +FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + OPENPILOT_PATH=/src \ + VIRTUAL_ENV=/src/.venv \ + PYTHONPATH=/src \ + HOME=/home/runner \ + PATH="/usr/local/bin:/src/.venv/bin:/home/runner/.local/bin:${PATH}" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + cmake \ + pkg-config \ + curl \ + libcurl4-openssl-dev \ + libssl-dev \ + libusb-1.0-0-dev \ + libzmq3-dev \ + libsqlite3-dev \ + libbz2-dev \ + zlib1g-dev \ + libva-dev \ + libva-drm2 \ + libdrm-dev \ + git \ + git-lfs \ + locales \ + python3 \ + python3-venv \ + xz-utils \ + zstd \ + time \ + xvfb \ + && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ + && locale-gen \ + && git lfs install --system \ + && UV_VERSION=0.12.2 \ + && UV_ARCH=x86_64-unknown-linux-gnu \ + && UV_TARBALL="uv-${UV_ARCH}.tar.gz" \ + && UV_SHA256=d66e96b5f1ca3b99806eee283a8125d33a0bd669e6e6d9bc4ab7ffda63c41bf4 \ + && curl -fsSL "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${UV_TARBALL}" -o "/tmp/${UV_TARBALL}" \ + && echo "${UV_SHA256} /tmp/${UV_TARBALL}" | sha256sum -c - \ + && tar -xzf "/tmp/${UV_TARBALL}" -C /tmp \ + && install -m 0755 "/tmp/uv-${UV_ARCH}/uv" /usr/local/bin/uv \ + && rm -rf "/tmp/${UV_TARBALL}" "/tmp/uv-${UV_ARCH}" \ + && test -x /usr/local/bin/uv \ + && uv --version | grep -F "uv ${UV_VERSION}" \ + && groupadd --gid 1001 runner \ + && useradd --uid 1001 --gid 1001 --create-home --home-dir /home/runner \ + --shell /usr/sbin/nologin runner \ + && passwd -l runner \ + && mkdir -p /src \ + && chown 1001:1001 /src \ + && rm -rf /var/lib/apt/lists/* \ + && ! command -v sudo >/dev/null + +WORKDIR /src + +# No sudoers entry. Default runtime is unprivileged; launcher may override to root for setup. +USER 1001:1001 + +# Controller invokes replay under --userns=keep-id:uid=1001,gid=1001 --network=none. +CMD ["bash", "-lc", "selfdrive/test/process_replay/test_processes.py --help"] diff --git a/tools/op.sh b/tools/op.sh index f21a285d175701..ea0512bfe4d57a 100755 --- a/tools/op.sh +++ b/tools/op.sh @@ -201,6 +201,9 @@ function op_before_cmd() { function op_setup() { op_get_openpilot_dir cd $OPENPILOT_ROOT + if [[ -n "${OPENPILOT_LFS_URL:-}" ]]; then + git config lfs.url "$OPENPILOT_LFS_URL" + fi op_check_openpilot_dir op_check_os diff --git a/tools/replay/SConscript b/tools/replay/SConscript index b12ede8b14a0c9..99728d9d291d5e 100644 --- a/tools/replay/SConscript +++ b/tools/replay/SConscript @@ -18,7 +18,8 @@ if arch != "Darwin": replay_lib_src.append("qcom_decoder.cc") replay_lib = replay_env.Library("replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks) Export('replay_lib') -replay_libs = [replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'bz2', 'zstd', 'yuv', 'ncurses'] + base_libs +ffmpeg_platform_libs = [] if arch == "Darwin" else ['va-drm', 'va', 'drm'] +replay_libs = [replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil'] + ffmpeg_platform_libs + ['x264', 'z', 'bz2', 'zstd', 'yuv', 'ncurses'] + base_libs replay_env.Program("replay", ["main.cc"], LIBS=replay_libs, FRAMEWORKS=base_frameworks) if GetOption('extras'):