From 415192a87d1368c0c81c7e9a25e9860ae7be1287 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Mon, 10 Aug 2026 16:04:05 +0200 Subject: [PATCH] feat[next]: cache manager CLI for the translation caches gt4py.next keeps the output of the translation step -- the optimized SDFG for DaCe, the generated sources for gtfn -- in a cache keyed by the program and the gt4py version, not by the gt4py sources. Editing a transformation or an optimization pass therefore leaves the key intact, so the next run replays the cached translation and the pass never runs, while the build step still recompiles from it and refreshes the library's mtime. Clearing only the build folder and seeing a rebuilt library looks like proof that the change took effect. It is not, and reading it that way once cost four multi-node benchmark jobs. Add `gt4py-next-cache`, mirroring `gt4py.cartesian.gt_cache_manager`: stdlib argparse, library functions plus a `__main__` block. `path` resolves the cache directories for the current environment, which is worth asking first since the default session lifetime puts them in a temporary directory that is gone when the process exits. `list` shows the entries, or with `--by-program` one row per program next to its build folders -- where a cached translation without a usable build folder is called out, since that combination is the trap. `delete` removes entries by key, program glob or all of them, after listing what matched and asking; it only prompts on a terminal and otherwise requires `--yes`, so a job file neither blocks nor proceeds unasked, and it takes the same lock as the runtime because these caches are shared between ranks. Nothing here predicts. A hit is decided by a fingerprint taken at run time over the lowered program and its arguments, which no tool outside a run can compute, and the caches record only a program name. Counts are reported as counts: no entries for a program means it will be re-translated; entries present is a reason to delete, never evidence that a run replayed. Build folders are the exception, since their name records the build-cache version, so those no run here can hit are counted as stale. Whether a build also ran to completion is not checked: it would mean reading each folder, and in a real cache all 51 build folders were complete while every stale one was stale by version. Entries are read directly rather than through `FileCache`, whose `__getitem__` deletes what it cannot unpickle -- an inspection tool must not do that -- so a corrupt or version-skewed entry degrades to `` instead of crashing. `[project.scripts]` gains the distribution's first console script, so the tool is reachable wherever gt4py is installed, including an icon4py environment on a cluster, which is where this failure mode bites. It is named after the subpackage because `gt4py.cartesian` has a separate cache with its own manager. The accompanying skill states the anti-pattern for agents: a fresh library mtime is evidence of recompilation, never of re-translation. It is the second skill in the repo, so AGENTS.md now records where skills live -- and that Claude Code reaches them through the tracked `.claude/skills` symlink, which fails silently if that link is ever dropped. --- .agents/skills/translation-cache/SKILL.md | 80 +++ AGENTS.md | 3 + pyproject.toml | 3 + src/gt4py/next/gt_cache_manager.py | 620 ++++++++++++++++++ .../unit_tests/test_gt_cache_manager.py | 515 +++++++++++++++ 5 files changed, 1221 insertions(+) create mode 100644 .agents/skills/translation-cache/SKILL.md create mode 100644 src/gt4py/next/gt_cache_manager.py create mode 100644 tests/next_tests/unit_tests/test_gt_cache_manager.py diff --git a/.agents/skills/translation-cache/SKILL.md b/.agents/skills/translation-cache/SKILL.md new file mode 100644 index 0000000000..20572fb08d --- /dev/null +++ b/.agents/skills/translation-cache/SKILL.md @@ -0,0 +1,80 @@ +--- +name: translation-cache +description: Clear and verify the gt4py.next translation cache before measuring a change to SDFG or code generation. TRIGGER before benchmarking, profiling, or judging any edit to a dace transformation, an optimization pass, gt_auto_optimize, gtfn codegen, or anything else affecting how a program is translated — and whenever a change appears to have "no effect" on generated code. SKIP for changes to the program itself (icon4py source, static args, domain sizes), which invalidate the cache on their own. +--- + +# translation-cache + +gt4py.next caches the *translation* step — the optimized SDFG for dace, the +generated source for gtfn — keyed by the program and `gt4py.__version__`, not by +the gt4py sources. Edit a transformation or an optimization pass and that key is +unchanged: the next run replays the cached translation and the pass never runs, +while the build step still recompiles from it. Benchmark without clearing the +cache and you are measuring the old compiler. + +For an editable install a *commit* changes the version and invalidates the cache. +Uncommitted edits do not — the dirty marker is a constant suffix — and a +non-editable install never does. + +## Recipe + +Run in the environment and working directory of the run being measured: + +```bash +gt4py-next-cache delete --program '' --yes +gt4py-next-cache list --filter '' --fail-if-cached +``` + +`delete` reports what it removed and exits non-zero if the selector matched +nothing. The `list` call is the gate — non-zero while any entry for that glob is +still there — so require it to pass **before** launching an expensive job. + +Read the output in the one direction that holds: **no entries for a program** +means it will be re-translated; **entries present** is a reason to delete, never +evidence that a run replayed. Deleting an entry that would not have been hit +costs nothing. + +When the cache is out of reach (a compute node, a container), or to invalidate +everything at once, salt both caches instead: + +```bash +export GT4PY_BUILD_CACHE_VERSION_ID=$(git -C rev-parse HEAD) +``` + +Use a nonce rather than the commit hash when iterating on uncommitted changes. + +## The two caches + +Run `gt4py-next-cache path` for where they are in the current environment — +worth checking first, because under the default session lifetime the cache sits +in a temporary directory that is deleted when the process exits, so nothing lands +in `.gt4py_cache` unless `GT4PY_BUILD_CACHE_LIFETIME=persistent` is set. + +The build cache and the translation cache are separate and hit independently, so +clearing only the build folder is **not enough**: the build step recompiles from +the replayed translation, and the library is rebuilt while every transformation +is skipped. `gt4py-next-cache list --by-program` shows both side by side, and +flags the combination that hides this — a cached translation with no usable build +folder. + +`gt4py-next-cache --help` covers the remaining flags; the same tool runs as +`python -m gt4py.next.gt_cache_manager` when the console script is not on `PATH`. + +## Anti-pattern + +**A fresh library mtime proves recompilation, not re-translation.** Never cite it +as evidence that a pass ran. Two further tells that were misread once and cost +four multi-node benchmark jobs: + +- Successive SDFG dumps differing only in `guid` fields — the signature of + unpickling the same cached object twice, not of a regenerated SDFG. +- Debug logging added inside a pass producing no output — the pass was never + called, rather than never applicable. + +To prove a pass executed, assert on something it changes, or log inside it and +confirm the log appears. + +## Related + +- `scripts/python/dace_determinism.py` answers the neighbouring question: whether + dace codegen is deterministic across two runs. diff --git a/AGENTS.md b/AGENTS.md index c429eaa5ec..0dcf4af994 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,9 @@ If a command above is wrong for your environment, fix `pyproject.toml`, subsystem. Add new ADRs there, not in a flat `docs/adr/`. - `gt4py.next`-specific conventions and test framework: [`src/gt4py/next/AGENTS.md`](src/gt4py/next/AGENTS.md). +- Agent skills (task recipes agents load on demand): + [`.agents/skills/`](.agents/skills/). Claude Code finds them through the + tracked `.claude/skills` symlink — keep it, or discovery silently stops. - Dev-environment setup and CI infrastructure: [`docs/development/`](docs/development/). - User-facing docs: [`docs/user/cartesian/`](docs/user/cartesian/) and diff --git a/pyproject.toml b/pyproject.toml index fe67748eee..016097a093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,9 @@ rocm7 = ['cupy-rocm-7-0>=14.0'] standard = ['clang-format>=18.1', 'scipy>=1.16.1'] testing = ['hypothesis>=6.93', 'pytest>=7.0'] +[project.scripts] +gt4py-next-cache = 'gt4py.next.gt_cache_manager:main' + [project.urls] Documentation = 'https://gridtools.github.io/gt4py' Homepage = 'https://gridtools.github.io/' diff --git a/src/gt4py/next/gt_cache_manager.py b/src/gt4py/next/gt_cache_manager.py new file mode 100644 index 0000000000..98911ad7e4 --- /dev/null +++ b/src/gt4py/next/gt_cache_manager.py @@ -0,0 +1,620 @@ +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +"""Utils for inspecting and pruning the gt4py.next caches for generated code. + +gt4py.next keeps two kinds of persistent cache side by side under the cache base +directory (see `gt4py.next.otf.compilation.cache`): + +- **Build caches**, one folder per compiled program variant, holding the build + artifacts and the compiled library. +- **Translation caches**, one pickled file per translated program, holding the + output of the translation step: the optimized SDFG for the DaCe backend, the + generated source for the gtfn backend. + +The translation cache is keyed by a fingerprint of the program plus the gt4py +version, not by the gt4py sources. Editing a transformation or an optimization +pass without changing either therefore leaves the fingerprint intact: the cached +translation is replayed and the pass never runs, while the build step still +recompiles and refreshes the library's mtime. A fresh library is thus evidence of +recompilation, never of re-translation. The two caches hit and miss independently, +so `list --by-program` reports both side by side. + +Run it in the environment that produced the cache, from the working directory the +cached run used (the default cache base is `/.gt4py_cache`), either through +the installed `gt4py-next-cache` command or as a module:: + + gt4py-next-cache list --by-program --filter 'apply_diffusion_*' + python -m gt4py.next.gt_cache_manager delete --program 'apply_diffusion_*' --yes +""" + +from __future__ import annotations + +import argparse +import collections +import dataclasses +import datetime +import fnmatch +import json +import pathlib +import pickle +import re +import shutil +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Any, Final + +from gt4py._core import locking +from gt4py.next import config +from gt4py.next.otf import stages +from gt4py.next.otf.compilation import cache +from gt4py.next.otf.compilation.build_systems import compiledb + + +ENTRY_SUFFIX: Final[str] = ".pkl" + +EXIT_OK: Final[int] = 0 +#: Nothing was done: a selector matched nothing, the user declined, or a gate tripped. +EXIT_NOTHING_DONE: Final[int] = 1 +#: The command could not run: bad arguments, or a confirmation that cannot be obtained. +EXIT_ERROR: Final[int] = 2 + +_BUILD_FOLDER_RE: Final[re.Pattern[str]] = re.compile(cache.CACHE_FOLDER_NAME_PATTERN) + + +class CacheDirError(RuntimeError): + """The given path is not usable as a gt4py.next cache base directory.""" + + +@dataclasses.dataclass(frozen=True) +class Entry: + """One translation cache entry, with what could be recovered from it. + + `program` is `None` exactly when the payload could not be unpickled, in which + case `error` describes why. Such an entry is dead weight rather than a + replay risk: the runtime cannot read it either and recomputes. + """ + + backend: str + key: str + path: pathlib.Path + size: int + mtime: float + program: str | None + error: str | None + + +@dataclasses.dataclass(frozen=True) +class BuildDir: + """One build cache folder, with what its name and contents say about it.""" + + program: str + path: pathlib.Path + version_id: str + + @property + def usable(self) -> bool: + """Whether a run in this environment could still be served by this folder. + + A folder built under a different build-cache version can never be hit + again: the version salts the folder name, so the next run looks for a + different folder and builds it. + """ + return self.version_id == config.BUILD_CACHE_VERSION_ID + + +@dataclasses.dataclass(frozen=True) +class ProgramSummary: + """What both caches hold for one program name. + + Counts, not predictions: a program name is all the caches record about their + contents, while a hit is decided by a fingerprint taken at run time over the + lowered program and its arguments. So an entry counted here may well be + unreachable, e.g. after its program's source was edited. + """ + + program: str + entries: Mapping[str, int] + build_dirs: int + usable_build_dirs: int + + @property + def entry_count(self) -> int: + return sum(self.entries.values()) + + +def get_cache_base(cache_dir: pathlib.Path | None = None) -> pathlib.Path: + """Return the cache base directory, or `cache_dir` if one is given. + + Args: + cache_dir: Explicit cache base. It is validated to look like a gt4py.next + cache, since the delete commands remove files underneath it. + + Returns: + The resolved cache base directory. It does not necessarily exist: a + session-lifetime cache base is created lazily. + + Raises: + CacheDirError: If `cache_dir` does not exist or does not look like a + gt4py.next cache base. + """ + if cache_dir is None: + return cache.get_cache_base_path(config.BUILD_CACHE_LIFETIME) + + path = cache_dir.expanduser().resolve() + if not path.is_dir(): + raise CacheDirError(f"'{path}' is not a directory.") + if not ( + path.name == config.BUILD_CACHE_DIR.name + or (path / cache.TRANSLATION_CACHE_DIR_NAME).is_dir() + or any(_BUILD_FOLDER_RE.fullmatch(child.name) for child in path.iterdir() if child.is_dir()) + ): + raise CacheDirError( + f"'{path}' does not look like a gt4py.next cache base directory: it is not named" + f" '{config.BUILD_CACHE_DIR.name}' and contains neither a translation cache" + f" ('{cache.TRANSLATION_CACHE_DIR_NAME}') nor a build cache folder." + ) + return path + + +def get_translation_cache_dirs( + cache_base: pathlib.Path, backends: Sequence[str] +) -> dict[str, pathlib.Path]: + """Return the translation cache directory of each of `backends`.""" + return { + backend: cache.get_translation_cache_folder(cache_base, backend) for backend in backends + } + + +def read_entry(path: pathlib.Path) -> stages.ProgramSource: + """Unpickle one translation cache entry. + + Reads the file directly instead of going through `filecache.FileCache`, which + deletes entries it fails to unpickle. + """ + with path.open("rb") as fp: + return pickle.load(fp) + + +def find_entries( + cache_base: pathlib.Path, backends: Sequence[str], *, program: str | None = None +) -> list[Entry]: + """Collect the translation cache entries of `backends`, optionally filtered. + + Args: + cache_base: Cache base directory. Missing cache directories are empty + caches, not errors. + backends: Backend names to scan. + program: If given, keep only entries whose program name matches this glob. + Unreadable entries have no program name and are dropped by any filter. + """ + entries = [] + for backend, cache_dir in get_translation_cache_dirs(cache_base, backends).items(): + if not cache_dir.is_dir(): + continue + for path in sorted(cache_dir.glob(f"*{ENTRY_SUFFIX}")): + file_stat = path.stat() + try: + program_name: str | None = read_entry(path).entry_point.name + error: str | None = None + except Exception as e: + program_name = None + error = f"{type(e).__name__}: {e}" + if program is not None and ( + program_name is None or not fnmatch.fnmatchcase(program_name, program) + ): + continue + entries.append( + Entry( + backend=backend, + key=path.stem, + path=path, + size=file_stat.st_size, + mtime=file_stat.st_mtime, + program=program_name, + error=error, + ) + ) + return entries + + +def find_build_dirs(cache_base: pathlib.Path, *, program: str | None = None) -> list[BuildDir]: + """Collect the build cache folders, optionally filtered by a program glob. + + The shared compiledb folder is named like a program's build folder but belongs + to no program, so it is left out and never deleted along with one. + """ + if not cache_base.is_dir(): + return [] + build_dirs = [] + for path in sorted(cache_base.iterdir()): + if not path.is_dir() or not (match := _BUILD_FOLDER_RE.fullmatch(path.name)): + continue + name = match.group("name").removesuffix(cache.BINDINGS_NAME_SUFFIX) + if name.startswith(compiledb.COMPILEDB_PROTOTYPE_NAME_PREFIX): + continue + if program is None or fnmatch.fnmatchcase(name, program): + build_dirs.append( + BuildDir(program=name, path=path, version_id=match.group("version_id")) + ) + return build_dirs + + +def summarize_programs( + cache_base: pathlib.Path, backends: Sequence[str], *, program: str | None = None +) -> tuple[list[ProgramSummary], list[Entry]]: + """Count what both caches hold, per program name. + + Returns: + A summary of every program with a translation cache entry or a build + folder, sorted by name, and the entries that could not be read. + """ + entries = find_entries(cache_base, backends, program=program) + build_dirs = find_build_dirs(cache_base, program=program) + + counts: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter) + for entry in entries: + if entry.program is not None: + counts[entry.program][entry.backend] += 1 + build_counts = collections.Counter(build_dir.program for build_dir in build_dirs) + usable_build_counts = collections.Counter( + build_dir.program for build_dir in build_dirs if build_dir.usable + ) + + return [ + ProgramSummary( + program=name, + entries=counts.get(name, {}), + build_dirs=build_counts.get(name, 0), + usable_build_dirs=usable_build_counts.get(name, 0), + ) + for name in sorted(counts.keys() | build_counts.keys()) + ], [entry for entry in entries if entry.program is None] + + +def delete_entries(entries: Sequence[Entry], cache_dirs: Sequence[pathlib.Path]) -> None: + """Remove translation cache entries, holding the same lock as the runtime. + + The caches are shared between concurrent processes (e.g. MPI ranks), so an + unlocked unlink can race a writer. + + Raises: + CacheDirError: If an entry is not a cache entry inside `cache_dirs`. + """ + for entry in entries: + if entry.path.suffix != ENTRY_SUFFIX or entry.path.parent not in cache_dirs: + raise CacheDirError( + f"refusing to remove '{entry.path}': not a translation cache entry." + ) + with locking.lock(entry.path): + entry.path.unlink(missing_ok=True) + + +def delete_build_dirs(build_dirs: Sequence[BuildDir], cache_base: pathlib.Path) -> None: + """Remove build cache folders, holding the same lock as the runtime. + + Raises: + CacheDirError: If a folder is not a build cache folder of `cache_base`. + """ + for build_dir in build_dirs: + if build_dir.path.parent != cache_base or not _BUILD_FOLDER_RE.fullmatch( + build_dir.path.name + ): + raise CacheDirError(f"refusing to remove '{build_dir.path}': not a build cache folder.") + with locking.lock(build_dir.path): + shutil.rmtree(build_dir.path) + + +def _format_size(size: int) -> str: + if size < 1024: + return f"{size} B" + scaled = size / 1024 + if scaled < 1024: + return f"{scaled:.1f} KiB" + return f"{scaled / 1024:.1f} MiB" + + +def _format_mtime(mtime: float) -> str: + return datetime.datetime.fromtimestamp(mtime).isoformat(sep=" ", timespec="seconds") + + +def _print_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> None: + widths = [max(len(str(row[i])) for row in [headers, *rows]) for i in range(len(headers))] + for row in [headers, *rows]: + print(" ".join(str(cell).ljust(width) for cell, width in zip(row, widths)).rstrip()) + + +def _cmd_path(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + rows = [("lifetime", config.BUILD_CACHE_LIFETIME.name.lower()), ("cache base", str(cache_base))] + rows += [ + (f"translation ({backend})", str(cache_dir)) + for backend, cache_dir in get_translation_cache_dirs(cache_base, args.backends).items() + ] + # Build folders are named per program variant, so name the pattern rather than a path. + rows.append( + ( + "build folders", + f"{cache_base}/{cache.BINDINGS_NAME_SUFFIX}__", + ) + ) + width = max(len(label) for label, _ in rows) + 2 + for label, value in rows: + print(f"{label + ':':{width}}{value}") + return EXIT_OK + + +def _cmd_list(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + if args.by_program: + return _list_by_program(args, cache_base) + return _list_entries(args, cache_base) + + +def _list_entries(args: argparse.Namespace, cache_base: pathlib.Path) -> int: + entries = find_entries(cache_base, args.backends, program=args.filter) + sort_keys: dict[str, Callable[[Entry], Any]] = { + "mtime": lambda e: e.mtime, + "size": lambda e: e.size, + "program": lambda e: (e.program or "", e.key), + "key": lambda e: e.key, + } + entries.sort(key=sort_keys[args.sort]) + + if args.json: + print( + json.dumps( + [{**dataclasses.asdict(e), "path": str(e.path)} for e in entries], + indent=2, + default=str, + ) + ) + elif not entries: + print("no entries") + else: + _print_table( + ("BACKEND", "KEY", "PROGRAM", "SIZE", "MTIME"), + [ + ( + e.backend, + e.key, + e.program if e.program is not None else "", + _format_size(e.size), + _format_mtime(e.mtime), + ) + for e in entries + ], + ) + return _fail_if_cached(args, len(entries)) + + +def _list_by_program(args: argparse.Namespace, cache_base: pathlib.Path) -> int: + summaries, unreadable = summarize_programs(cache_base, args.backends, program=args.filter) + + if args.json: + print(json.dumps([dataclasses.asdict(s) for s in summaries], indent=2, default=str)) + elif not summaries: + print("no cached programs") + else: + _print_table( + ("PROGRAM", "ENTRIES", "BUILDS"), + [ + ( + summary.program, + ", ".join(f"{n} {backend}" for backend, n in sorted(summary.entries.items())) + or "-", + f"{summary.usable_build_dirs}" + if summary.build_dirs == summary.usable_build_dirs + else f"{summary.usable_build_dirs}" + f" (+{summary.build_dirs - summary.usable_build_dirs} stale)", + ) + for summary in summaries + ], + ) + sys.stdout.flush() # keep the table above the notes when both go to a terminal + + if trap := [s for s in summaries if s.entry_count and not s.usable_build_dirs]: + print( + f"\nWARNING: {len(trap)} program(s) have a cached translation but no usable build" + " folder. Their next run rebuilds the library while the cached translation is" + " replayed, so a fresh library there would NOT mean a changed pass ran.", + file=sys.stderr, + ) + if stale := sum(s.build_dirs - s.usable_build_dirs for s in summaries): + print( + f"\nnote: {stale} build folder(s) cannot be hit by this environment: they were" + f" built under a build-cache version other than '{config.BUILD_CACHE_VERSION_ID}'." + " The same version salts the translation cache, but entries do not record it, so" + " entries left by those runs are still counted above although they cannot be hit" + " either.", + file=sys.stderr, + ) + if unreadable: + print( + f"\nnote: {len(unreadable)} entr(ies) could not be read; the runtime discards those" + " too, so they cannot be hit either.", + file=sys.stderr, + ) + return _fail_if_cached(args, sum(s.entry_count for s in summaries)) + + +def _fail_if_cached(args: argparse.Namespace, matched: int) -> int: + if args.fail_if_cached and matched: + print(f"error: {matched} cached entr(ies) matched.", file=sys.stderr) + return EXIT_NOTHING_DONE + return EXIT_OK + + +def _confirm(question: str) -> bool: + """Ask a yes/no question on an interactive terminal, defaulting to no.""" + try: + answer = input(f"{question} [y/N] ") + except (EOFError, KeyboardInterrupt): + print() + return False + return answer.strip().casefold() in ("y", "yes") + + +def _cmd_delete(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + cache_dirs = get_translation_cache_dirs(cache_base, args.backends) + + entries = find_entries(cache_base, args.backends, program=args.program) + if args.key: + entries = [entry for entry in entries if entry.key in args.key] + + build_dirs: list[BuildDir] = [] + if args.include_build_dirs: + if args.key: + programs = {entry.program for entry in entries if entry.program is not None} + build_dirs = [b for b in find_build_dirs(cache_base) if b.program in programs] + else: + build_dirs = find_build_dirs(cache_base, program=args.program) + + if not entries and not build_dirs: + print("error: nothing matched the given selector.", file=sys.stderr) + return EXIT_NOTHING_DONE + + print(f"from {cache_base}:") + for entry in entries: + print(f" {entry.path.relative_to(cache_base)} ({entry.program or ''})") + for build_dir in build_dirs: + print(f" {build_dir.path.relative_to(cache_base)}/ [build]") + summary = f"{len(entries)} entr(ies) and {len(build_dirs)} build folder(s)" + + if args.dry_run: + print(f"\ndry run: nothing was removed ({summary} matched).") + return EXIT_OK + if not args.yes: + if not sys.stdin.isatty(): + print( + f"error: refusing to remove {summary} without a confirmation. Pass --yes to" + " confirm, or --dry-run to preview.", + file=sys.stderr, + ) + return EXIT_ERROR + if not _confirm(f"\nRemove {summary}?"): + print("aborted, nothing was removed.") + return EXIT_NOTHING_DONE + + delete_entries(entries, list(cache_dirs.values())) + delete_build_dirs(build_dirs, cache_base) + print(f"\nremoved {summary}.") + return EXIT_OK + + +def _add_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--cache-dir", + type=pathlib.Path, + default=None, + metavar="PATH", + help="Cache base directory to operate on (default: the one the current environment uses).", + ) + parser.add_argument( + "--backend", + dest="backends", + choices=[*cache.TRANSLATION_CACHE_BACKENDS, "all"], + default="all", + help="Translation cache(s) to operate on (default: all).", + ) + + +def _make_parser(prog: str | None = None) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=prog, + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + path_parser = subparsers.add_parser("path", help="Print the resolved cache directories.") + path_parser.set_defaults(func=_cmd_path) + + list_parser = subparsers.add_parser( + "list", + help="List what the caches hold.", + description=( + "List the translation cache entries, or with --by-program one row per program" + " together with its build folders. Everything reported is a count of what is on" + " disk, not a prediction: a hit is decided by a fingerprint taken at run time" + " over the lowered program and its arguments, which this tool does not have, so" + " an entry listed here may already be unreachable — after its program's source" + " was edited, for instance. Build folders record the build-cache version in" + " their name, so those that no run here can hit are counted as stale;" + " translation cache entries record no version and cannot be told apart that way." + ), + ) + list_parser.add_argument( + "--filter", metavar="GLOB", default=None, help="Only entries whose program name matches." + ) + list_parser.add_argument( + "--by-program", + action="store_true", + help="One row per program, with its build folders, instead of one row per entry.", + ) + list_parser.add_argument("--sort", choices=["mtime", "size", "program", "key"], default="mtime") + list_parser.add_argument("--json", action="store_true", help="Print as JSON.") + list_parser.add_argument( + "--fail-if-cached", + action="store_true", + help="Exit with a non-zero status if anything matched, to gate a run on an empty cache.", + ) + list_parser.set_defaults(func=_cmd_list) + + delete_parser = subparsers.add_parser( + "delete", + help="Remove translation cache entries.", + description=( + "List what the selector matches, then remove it after a confirmation. The" + " confirmation is only asked for on an interactive terminal; elsewhere --yes is" + " required, so a script never blocks on a prompt." + ), + ) + selectors = delete_parser.add_mutually_exclusive_group(required=True) + selectors.add_argument("--key", action="append", metavar="KEY", help="Entry key; repeatable.") + selectors.add_argument("--program", metavar="GLOB", help="Entries whose program name matches.") + selectors.add_argument("--all", action="store_true", help="All entries.") + confirmation = delete_parser.add_mutually_exclusive_group() + confirmation.add_argument( + "--yes", "-y", action="store_true", help="Remove without asking for confirmation." + ) + confirmation.add_argument( + "--dry-run", + "-n", + action="store_true", + help="Only describe what would be removed, then exit.", + ) + delete_parser.add_argument( + "--include-build-dirs", + action="store_true", + help="Also remove the build cache folders of the selected programs.", + ) + delete_parser.set_defaults(func=_cmd_delete) + + for subparser in (path_parser, list_parser, delete_parser): + _add_common_arguments(subparser) + return parser + + +def main(argv: Sequence[str] | None = None, *, prog: str | None = None) -> int: + """Run the CLI. Entry point of the `gt4py-next-cache` command.""" + args = _make_parser(prog).parse_args(argv) + args.backends = ( + list(cache.TRANSLATION_CACHE_BACKENDS) if args.backends == "all" else [args.backends] + ) + try: + return args.func(args) + except CacheDirError as e: + print(f"error: {e}", file=sys.stderr) + return EXIT_ERROR + + +if __name__ == "__main__": + # `python -m` leaves argparse with the module file as the program name. + sys.exit(main(prog="python -m gt4py.next.gt_cache_manager")) diff --git a/tests/next_tests/unit_tests/test_gt_cache_manager.py b/tests/next_tests/unit_tests/test_gt_cache_manager.py new file mode 100644 index 0000000000..e5f62b44e5 --- /dev/null +++ b/tests/next_tests/unit_tests/test_gt_cache_manager.py @@ -0,0 +1,515 @@ +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import pathlib + +import pytest + +from gt4py._core import filecache +from gt4py.next import config, gt_cache_manager +from gt4py.next.otf import code_specs, stages +from gt4py.next.otf.binding import interface +from gt4py.next.otf.compilation import cache +from gt4py.next.otf.compilation.build_systems import compiledb + + +def make_program_source(name: str, backend: str) -> stages.ProgramSource: + """Build a payload of the shape the translation step of `backend` caches.""" + if backend == "dace": + code_spec = code_specs.SDFGCodeSpec() + source_code = {"type": "SDFG", "attributes": {"name": name}, "nodes": []} + else: + code_spec = code_specs.CPPCodeSpec() + source_code = f"// generated code of {name}" + return stages.ProgramSource( + entry_point=interface.Function(name, ()), + source_code=source_code, + library_deps=(), + code_spec=code_spec, + ) + + +def write_entry(cache_base: pathlib.Path, backend: str, name: str, salt: str = "") -> str: + """Write one translation cache entry through `FileCache`, as the runtime does.""" + file_cache = filecache.FileCache(cache.get_translation_cache_folder(cache_base, backend)) + key = f"{backend}-{name}-{salt}" + file_cache[key] = make_program_source(name, backend) + return file_cache._get_path(key).stem + + +def write_build_dir( + cache_base: pathlib.Path, name: str, salt: str = "0", *, version_id: str | None = None +) -> pathlib.Path: + """Create a folder named the way `cache.get_cache_folder` names them.""" + version_id = config.BUILD_CACHE_VERSION_ID if version_id is None else version_id + folder = cache_base / f"{name}{cache.BINDINGS_NAME_SUFFIX}_{salt * 16}_{version_id}" + folder.mkdir(parents=True) + (folder / "libprogram.so").write_text("not really a library") + return folder + + +class FakeStdin: + """Stand-in for `sys.stdin` that reports whether it is a terminal.""" + + def __init__(self, interactive: bool) -> None: + self._interactive = interactive + + def isatty(self) -> bool: + return self._interactive + + +@pytest.fixture +def cache_base(tmp_path): + base = tmp_path / config.BUILD_CACHE_DIR.name + base.mkdir() + return base + + +def test_get_cache_base_follows_persistent_config(monkeypatch, tmp_path): + monkeypatch.setattr(config, "BUILD_CACHE_DIR", tmp_path / ".gt4py_cache") + monkeypatch.setattr(config, "BUILD_CACHE_LIFETIME", config.BuildCacheLifetime.PERSISTENT) + assert gt_cache_manager.get_cache_base() == tmp_path / ".gt4py_cache" + + +def test_get_cache_base_follows_session_config(monkeypatch): + monkeypatch.setattr(config, "BUILD_CACHE_LIFETIME", config.BuildCacheLifetime.SESSION) + assert gt_cache_manager.get_cache_base() == cache.get_cache_base_path( + config.BuildCacheLifetime.SESSION + ) + + +def test_get_cache_base_accepts_cache_like_dirs(tmp_path, cache_base): + assert gt_cache_manager.get_cache_base(cache_base) == cache_base.resolve() + + by_translation_cache = tmp_path / "elsewhere" + cache.get_translation_cache_folder(by_translation_cache, "dace").mkdir(parents=True) + assert gt_cache_manager.get_cache_base(by_translation_cache) == by_translation_cache.resolve() + + by_build_dir = tmp_path / "somewhere" + by_build_dir.mkdir() + write_build_dir(by_build_dir, "prog") + assert gt_cache_manager.get_cache_base(by_build_dir) == by_build_dir.resolve() + + +def test_get_cache_base_rejects_other_dirs(tmp_path): + (tmp_path / "unrelated").mkdir() + (tmp_path / "unrelated" / "notes.txt").write_text("hello") + with pytest.raises(gt_cache_manager.CacheDirError, match="does not look like"): + gt_cache_manager.get_cache_base(tmp_path / "unrelated") + + with pytest.raises(gt_cache_manager.CacheDirError, match="not a directory"): + gt_cache_manager.get_cache_base(tmp_path / "missing") + + +def test_missing_cache_dirs_are_an_empty_cache(cache_base): + assert gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"]) == [] + assert gt_cache_manager.find_build_dirs(cache_base) == [] + assert gt_cache_manager.summarize_programs(cache_base, ["dace", "gtfn"]) == ([], []) + + +def test_find_entries_reads_program_and_backend(cache_base): + dace_key = write_entry(cache_base, "dace", "foo") + gtfn_key = write_entry(cache_base, "gtfn", "bar") + + entries = gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"]) + + assert {(e.backend, e.key, e.program) for e in entries} == { + ("dace", dace_key, "foo"), + ("gtfn", gtfn_key, "bar"), + } + assert all(e.error is None and e.size > 0 for e in entries) + + +def test_find_entries_filters_by_program_and_backend(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "foobar") + write_entry(cache_base, "gtfn", "foo") + + assert len(gt_cache_manager.find_entries(cache_base, ["dace"], program="foo*")) == 2 + assert len(gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"], program="foo")) == 2 + assert gt_cache_manager.find_entries(cache_base, ["gtfn"], program="nope") == [] + + +def test_find_entries_ignores_lock_files(cache_base): + key = write_entry(cache_base, "dace", "foo") + cache_dir = cache.get_translation_cache_folder(cache_base, "dace") + # A writer killed mid-write leaves its lock file behind. + (cache_dir / f"{key}.filelock_FileLock.lock").touch() + + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_find_build_dirs_recovers_program_name(cache_base): + write_build_dir(cache_base, "foo") + (cache_base / "not-a-build-dir").mkdir() + + build_dirs = gt_cache_manager.find_build_dirs(cache_base) + + assert [b.program for b in build_dirs] == ["foo"] + assert gt_cache_manager.find_build_dirs(cache_base, program="ba*") == [] + + +def test_find_build_dirs_skips_the_shared_compiledb(cache_base): + write_build_dir(cache_base, "foo") + compiledb_dir = write_build_dir( + cache_base, f"{compiledb.COMPILEDB_PROTOTYPE_NAME_PREFIX}_gridtools_cpu_Release", salt="1" + ) + + assert [b.program for b in gt_cache_manager.find_build_dirs(cache_base)] == ["foo"] + + gt_cache_manager.main( + ["delete", "--all", "--include-build-dirs", "--yes", "--cache-dir", str(cache_base)] + ) + + assert compiledb_dir.is_dir() + + +def test_corrupt_entry_degrades_instead_of_crashing(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + corrupt = cache.get_translation_cache_folder(cache_base, "dace") / "deadbeefdeadbeef.pkl" + corrupt.write_bytes(b"not a pickle") + + entries = gt_cache_manager.find_entries(cache_base, ["dace"]) + corrupt_entries = [e for e in entries if e.key == corrupt.stem] + + assert len(entries) == 2 + assert corrupt_entries[0].program is None + assert corrupt_entries[0].error + assert corrupt.exists() # inspection must not delete what it cannot read + + assert gt_cache_manager.main(["list", "--cache-dir", str(cache_base)]) == 0 + assert "" in capsys.readouterr().out + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + assert "could not be read" in capsys.readouterr().err + + +def test_summarize_groups_entries_and_builds_by_program(cache_base): + write_entry(cache_base, "dace", "cached") + write_entry(cache_base, "dace", "cached", salt="second") + write_entry(cache_base, "gtfn", "cached") + write_build_dir(cache_base, "built_only") + + summaries, unreadable = gt_cache_manager.summarize_programs(cache_base, ["dace", "gtfn"]) + + assert unreadable == [] + assert [s.program for s in summaries] == ["built_only", "cached"] + assert summaries[0].entry_count == 0 + assert summaries[0].build_dirs == 1 + assert summaries[1].entries == {"dace": 2, "gtfn": 1} + assert summaries[1].build_dirs == 0 + + +def test_summarize_counts_the_two_caches_independently(cache_base): + write_entry(cache_base, "dace", "warm") + write_build_dir(cache_base, "warm") + write_entry(cache_base, "dace", "translated_only") + write_build_dir(cache_base, "built_only") + + summaries = {s.program: s for s in gt_cache_manager.summarize_programs(cache_base, ["dace"])[0]} + + assert (summaries["warm"].entry_count, summaries["warm"].usable_build_dirs) == (1, 1) + assert ( + summaries["translated_only"].entry_count, + summaries["translated_only"].usable_build_dirs, + ) == (1, 0) + assert (summaries["built_only"].entry_count, summaries["built_only"].usable_build_dirs) == ( + 0, + 1, + ) + + +def test_build_from_another_version_is_stale(cache_base): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo", version_id="0.0.1+ancient") + + (summary,) = gt_cache_manager.summarize_programs(cache_base, ["dace"])[0] + + assert summary.build_dirs == 1 + assert summary.usable_build_dirs == 0 + + +def test_list_by_program_warns_about_a_cached_translation_without_a_build(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + captured = capsys.readouterr() + assert "1 dace" in captured.out + assert "would NOT mean a changed pass ran" in captured.err + + +def test_list_by_program_does_not_warn_when_both_caches_are_warm(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + assert "WARNING" not in capsys.readouterr().err + + +def test_list_by_program_notes_that_entries_may_be_stale_too(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo", version_id="0.0.1+ancient") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + assert "cannot be hit either" in capsys.readouterr().err + + +@pytest.mark.parametrize("extra", [[], ["--by-program"]]) +def test_list_fail_if_cached(cache_base, extra): + write_build_dir(cache_base, "foo") + argv = ["list", "--cache-dir", str(cache_base), "--fail-if-cached", *extra] + + assert gt_cache_manager.main(argv) == 0 # a build folder alone is not a cached translation + + write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(argv) == gt_cache_manager.EXIT_NOTHING_DONE + + +@pytest.mark.parametrize("extra", [[], ["--by-program"]]) +def test_list_filter(cache_base, capsys, extra): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + gt_cache_manager.main(["list", "--cache-dir", str(cache_base), "--filter", "fo*", *extra]) + + captured = capsys.readouterr() + assert "foo" in captured.out + assert "bar" not in captured.out + + +def test_delete_dry_run_keeps_entries(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + + assert ( + gt_cache_manager.main(["delete", "--all", "--dry-run", "--cache-dir", str(cache_base)]) == 0 + ) + + assert "dry run" in capsys.readouterr().out + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_asks_before_removing_on_a_terminal(cache_base, monkeypatch, capsys): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + monkeypatch.setattr("builtins.input", lambda prompt="": "y") + + assert gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) == 0 + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + + +@pytest.mark.parametrize("answer", ["n", "", "no", "whatever"]) +def test_delete_declined_removes_nothing(cache_base, monkeypatch, answer): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + monkeypatch.setattr("builtins.input", lambda prompt="": answer) + + result = gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + + assert result == gt_cache_manager.EXIT_NOTHING_DONE + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_never_prompts_without_a_terminal(cache_base, monkeypatch, capsys): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=False)) + + def _no_prompting(prompt=""): + raise AssertionError("a non-interactive run must not block on a prompt") + + monkeypatch.setattr("builtins.input", _no_prompting) + + result = gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + + assert result == gt_cache_manager.EXIT_ERROR + assert "--yes" in capsys.readouterr().err + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_declined_at_eof_removes_nothing(cache_base, monkeypatch): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + + def _eof(prompt=""): + raise EOFError + + monkeypatch.setattr("builtins.input", _eof) + + assert ( + gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + == gt_cache_manager.EXIT_NOTHING_DONE + ) + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_removes_selected_entries(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + assert ( + gt_cache_manager.main( + ["delete", "--program", "foo", "--yes", "--cache-dir", str(cache_base)] + ) + == 0 + ) + + assert [e.program for e in gt_cache_manager.find_entries(cache_base, ["dace"])] == ["bar"] + + +def test_delete_by_key(cache_base): + key = write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + assert ( + gt_cache_manager.main(["delete", "--key", key, "--yes", "--cache-dir", str(cache_base)]) + == 0 + ) + + assert [e.program for e in gt_cache_manager.find_entries(cache_base, ["dace"])] == ["bar"] + + +def test_delete_without_match_exits_non_zero(cache_base): + write_entry(cache_base, "dace", "foo") + + assert ( + gt_cache_manager.main( + ["delete", "--program", "nope", "--yes", "--cache-dir", str(cache_base)] + ) + == gt_cache_manager.EXIT_NOTHING_DONE + ) + + +def test_delete_leaves_build_dirs_alone(cache_base): + write_entry(cache_base, "dace", "foo") + build_dir = write_build_dir(cache_base, "foo") + + assert gt_cache_manager.main(["delete", "--all", "--yes", "--cache-dir", str(cache_base)]) == 0 + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert build_dir.is_dir() + + +def test_delete_include_build_dirs(cache_base): + write_entry(cache_base, "dace", "foo") + foo_build_dir = write_build_dir(cache_base, "foo") + bar_build_dir = write_build_dir(cache_base, "bar") + + assert ( + gt_cache_manager.main( + [ + "delete", + "--program", + "foo", + "--include-build-dirs", + "--yes", + "--cache-dir", + str(cache_base), + ] + ) + == 0 + ) + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert not foo_build_dir.exists() + assert bar_build_dir.is_dir() + + +def test_delete_backend_selection(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "gtfn", "foo") + + assert ( + gt_cache_manager.main( + ["delete", "--all", "--yes", "--backend", "dace", "--cache-dir", str(cache_base)] + ) + == 0 + ) + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert len(gt_cache_manager.find_entries(cache_base, ["gtfn"])) == 1 + + +def test_delete_refuses_paths_outside_the_cache(cache_base, tmp_path): + outsider = tmp_path / "precious.pkl" + outsider.write_text("keep me") + entry = gt_cache_manager.Entry( + backend="dace", + key=outsider.stem, + path=outsider, + size=0, + mtime=0.0, + program="foo", + error=None, + ) + + with pytest.raises(gt_cache_manager.CacheDirError, match="refusing to remove"): + gt_cache_manager.delete_entries( + [entry], [cache.get_translation_cache_folder(cache_base, "dace")] + ) + + assert outsider.exists() + + +def test_delete_refuses_build_dirs_outside_the_cache(cache_base, tmp_path): + outsider = tmp_path / "precious" + outsider.mkdir() + build_dir = gt_cache_manager.BuildDir( + program="foo", path=outsider, version_id=config.BUILD_CACHE_VERSION_ID + ) + + with pytest.raises(gt_cache_manager.CacheDirError, match="refusing to remove"): + gt_cache_manager.delete_build_dirs([build_dir], cache_base) + + assert outsider.is_dir() + + +def test_path_command_reports_both_caches_and_the_build_folders(cache_base, capsys): + assert gt_cache_manager.main(["path", "--cache-dir", str(cache_base)]) == 0 + + out = capsys.readouterr().out + assert str(cache_base.resolve()) in out + for backend in cache.TRANSLATION_CACHE_BACKENDS: + assert str(cache.get_translation_cache_folder(cache_base.resolve(), backend)) in out + # the confusion this answers: build folders are siblings of the translation + # caches, not something inside them + assert cache.BINDINGS_NAME_SUFFIX in out + + +def test_list_json_output(cache_base, capsys): + import json + + key = write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(["list", "--json", "--cache-dir", str(cache_base)]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert [(e["key"], e["program"], e["backend"]) for e in payload] == [(key, "foo", "dace")] + + +def test_console_script_points_at_main(): + import importlib.metadata + + entry_points = importlib.metadata.distribution("gt4py").entry_points + console_scripts = {ep.name: ep for ep in entry_points if ep.group == "console_scripts"} + + assert "gt4py-next-cache" in console_scripts + assert console_scripts["gt4py-next-cache"].load() is gt_cache_manager.main + + +def test_cli_reports_cache_dir_error(tmp_path, capsys): + (tmp_path / "unrelated").mkdir() + + assert ( + gt_cache_manager.main(["list", "--cache-dir", str(tmp_path / "unrelated")]) + == gt_cache_manager.EXIT_ERROR + ) + assert "does not look like" in capsys.readouterr().err