diff --git a/app/validate.py b/app/validate.py index 2f6edee..70d8982 100644 --- a/app/validate.py +++ b/app/validate.py @@ -126,13 +126,25 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") -def _load(subdir: str) -> list[tuple[str, dict[str, Any]]]: +def _load(subdir: str, only: set[str] | None = None) -> list[tuple[str, dict[str, Any]]]: + """Load every ``*.json`` under ``subdir``; with ``only``, just those paths. + + ``only`` holds DATA_DIR-relative posix paths (as ``git diff`` prints them + minus the ``data/`` prefix), so a scoped run never walks the whole tree. + """ path = DATA_DIR / subdir if not path.exists(): return [] + if only is not None: + files = sorted( + f for rel in only + if rel.startswith(f"{subdir}/") and (f := DATA_DIR / rel).is_file() + ) + else: + files = sorted(path.rglob("*.json")) # recurse into brand subfolders return [ (str(f.relative_to(DATA_DIR)), json.loads(f.read_text(encoding="utf-8-sig"))) - for f in sorted(path.rglob("*.json")) # recurse into brand subfolders + for f in files ] diff --git a/app/verify/cli.py b/app/verify/cli.py index fb39657..01275a3 100644 --- a/app/verify/cli.py +++ b/app/verify/cli.py @@ -78,13 +78,13 @@ def _iter_selected( def cmd_score(args: argparse.Namespace) -> int: - records = load_all() + changed = _changed_data_slugs(args.base) if args.changed else None + records = load_all(only=changed) _, _, soc_release = foreign_key_sets(records) now_year = offline.now_year_today() ts = _now_iso() categories = tuple(args.category) if args.category else CATEGORIES - changed = _changed_data_slugs(args.base) if args.changed else None # The scores cache is a full-dataset snapshot; only rewrite it on a full run. full_scope = args.category is None and args.max is None and not args.changed @@ -622,7 +622,7 @@ def cmd_pr(args: argparse.Namespace) -> int: cross-reference) + Tier 3 (promotion decision, DRY-RUN — never writes). Network tiers run only over the records changed vs the PR merge base, capped by --max. """ - records = load_all() + records = load_all() # full: the baseline below scores everything (see status.json) _, _, soc_release = foreign_key_sets(records) now_year = offline.now_year_today() diff --git a/app/verify/common.py b/app/verify/common.py index cdef974..fb3d12d 100644 --- a/app/verify/common.py +++ b/app/verify/common.py @@ -70,14 +70,28 @@ def __repr__(self) -> str: # pragma: no cover - debug aid return f"Record({self.category}, {self.slug!r})" -def load_category(category: str) -> list[Record]: +# Categories other records point at (brand/soc FKs, SoC release dates). A scoped +# load keeps these whole (~2.3k files) so foreign-key checks stay exact. +FK_CATEGORIES = ("brand", "soc") + + +def load_category(category: str, only: set[str] | None = None) -> list[Record]: """Load one category's records as :class:`Record` objects.""" - return [Record(category, path, data) for path, data in _load(category)] + return [Record(category, path, data) for path, data in _load(category, only)] -def load_all(categories: Iterable[str] = CATEGORIES) -> dict[str, list[Record]]: - """Load every category into ``{category: [Record, ...]}``.""" - return {cat: load_category(cat) for cat in categories} +def load_all( + categories: Iterable[str] = CATEGORIES, only: set[str] | None = None +) -> dict[str, list[Record]]: + """Load every category into ``{category: [Record, ...]}``. + + ``only`` (data-relative paths) restricts non-FK categories to those files so + a PR touching a handful of records does not parse all ~190k. + """ + return { + cat: load_category(cat, None if cat in FK_CATEGORIES else only) + for cat in categories + } def foreign_key_sets( diff --git a/tests/verify/test_scoped_load.py b/tests/verify/test_scoped_load.py new file mode 100644 index 0000000..f0875c8 --- /dev/null +++ b/tests/verify/test_scoped_load.py @@ -0,0 +1,30 @@ +"""Scoped loading reads only the requested files (plus whole FK categories).""" + +import json + +from app import validate +from app.verify import common + + +def _write(root, rel, slug): + f = root / rel + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(json.dumps({"slug": slug}), encoding="utf-8") + + +def test_only_restricts_non_fk_categories(tmp_path, monkeypatch): + monkeypatch.setattr(validate, "DATA_DIR", tmp_path) + monkeypatch.setattr(common, "_load", validate._load) + _write(tmp_path, "cpu/a/1.json", "a1") + _write(tmp_path, "cpu/a/2.json", "a2") + _write(tmp_path, "soc/x/1.json", "s1") + _write(tmp_path, "soc/x/2.json", "s2") + + full = common.load_all(("cpu", "soc")) + assert [len(full[c]) for c in ("cpu", "soc")] == [2, 2] + + scoped = common.load_all(("cpu", "soc"), only={"cpu/a/2.json", "cpu/missing.json"}) + assert [r.slug for r in scoped["cpu"]] == ["a2"] + assert len(scoped["soc"]) == 2 # FK category stays whole + + assert common.load_all(("cpu",), only=set())["cpu"] == []