diff --git a/.agents/skills/translation-cache/SKILL.md b/.agents/skills/translation-cache/SKILL.md new file mode 100644 index 0000000000..20572fb08d --- /dev/null +++ b/.agents/skills/translation-cache/SKILL.md @@ -0,0 +1,80 @@ +--- +name: translation-cache +description: Clear and verify the gt4py.next translation cache before measuring a change to SDFG or code generation. TRIGGER before benchmarking, profiling, or judging any edit to a dace transformation, an optimization pass, gt_auto_optimize, gtfn codegen, or anything else affecting how a program is translated — and whenever a change appears to have "no effect" on generated code. SKIP for changes to the program itself (icon4py source, static args, domain sizes), which invalidate the cache on their own. +--- + +# translation-cache + +gt4py.next caches the *translation* step — the optimized SDFG for dace, the +generated source for gtfn — keyed by the program and `gt4py.__version__`, not by +the gt4py sources. Edit a transformation or an optimization pass and that key is +unchanged: the next run replays the cached translation and the pass never runs, +while the build step still recompiles from it. Benchmark without clearing the +cache and you are measuring the old compiler. + +For an editable install a *commit* changes the version and invalidates the cache. +Uncommitted edits do not — the dirty marker is a constant suffix — and a +non-editable install never does. + +## Recipe + +Run in the environment and working directory of the run being measured: + +```bash +gt4py-next-cache delete --program '' --yes +gt4py-next-cache list --filter '' --fail-if-cached +``` + +`delete` reports what it removed and exits non-zero if the selector matched +nothing. The `list` call is the gate — non-zero while any entry for that glob is +still there — so require it to pass **before** launching an expensive job. + +Read the output in the one direction that holds: **no entries for a program** +means it will be re-translated; **entries present** is a reason to delete, never +evidence that a run replayed. Deleting an entry that would not have been hit +costs nothing. + +When the cache is out of reach (a compute node, a container), or to invalidate +everything at once, salt both caches instead: + +```bash +export GT4PY_BUILD_CACHE_VERSION_ID=$(git -C rev-parse HEAD) +``` + +Use a nonce rather than the commit hash when iterating on uncommitted changes. + +## The two caches + +Run `gt4py-next-cache path` for where they are in the current environment — +worth checking first, because under the default session lifetime the cache sits +in a temporary directory that is deleted when the process exits, so nothing lands +in `.gt4py_cache` unless `GT4PY_BUILD_CACHE_LIFETIME=persistent` is set. + +The build cache and the translation cache are separate and hit independently, so +clearing only the build folder is **not enough**: the build step recompiles from +the replayed translation, and the library is rebuilt while every transformation +is skipped. `gt4py-next-cache list --by-program` shows both side by side, and +flags the combination that hides this — a cached translation with no usable build +folder. + +`gt4py-next-cache --help` covers the remaining flags; the same tool runs as +`python -m gt4py.next.gt_cache_manager` when the console script is not on `PATH`. + +## Anti-pattern + +**A fresh library mtime proves recompilation, not re-translation.** Never cite it +as evidence that a pass ran. Two further tells that were misread once and cost +four multi-node benchmark jobs: + +- Successive SDFG dumps differing only in `guid` fields — the signature of + unpickling the same cached object twice, not of a regenerated SDFG. +- Debug logging added inside a pass producing no output — the pass was never + called, rather than never applicable. + +To prove a pass executed, assert on something it changes, or log inside it and +confirm the log appears. + +## Related + +- `scripts/python/dace_determinism.py` answers the neighbouring question: whether + dace codegen is deterministic across two runs. diff --git a/AGENTS.md b/AGENTS.md index c429eaa5ec..0dcf4af994 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,9 @@ If a command above is wrong for your environment, fix `pyproject.toml`, subsystem. Add new ADRs there, not in a flat `docs/adr/`. - `gt4py.next`-specific conventions and test framework: [`src/gt4py/next/AGENTS.md`](src/gt4py/next/AGENTS.md). +- Agent skills (task recipes agents load on demand): + [`.agents/skills/`](.agents/skills/). Claude Code finds them through the + tracked `.claude/skills` symlink — keep it, or discovery silently stops. - Dev-environment setup and CI infrastructure: [`docs/development/`](docs/development/). - User-facing docs: [`docs/user/cartesian/`](docs/user/cartesian/) and diff --git a/pyproject.toml b/pyproject.toml index fe67748eee..016097a093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,9 @@ rocm7 = ['cupy-rocm-7-0>=14.0'] standard = ['clang-format>=18.1', 'scipy>=1.16.1'] testing = ['hypothesis>=6.93', 'pytest>=7.0'] +[project.scripts] +gt4py-next-cache = 'gt4py.next.gt_cache_manager:main' + [project.urls] Documentation = 'https://gridtools.github.io/gt4py' Homepage = 'https://gridtools.github.io/' diff --git a/src/gt4py/next/gt_cache_manager.py b/src/gt4py/next/gt_cache_manager.py new file mode 100644 index 0000000000..98911ad7e4 --- /dev/null +++ b/src/gt4py/next/gt_cache_manager.py @@ -0,0 +1,620 @@ +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +"""Utils for inspecting and pruning the gt4py.next caches for generated code. + +gt4py.next keeps two kinds of persistent cache side by side under the cache base +directory (see `gt4py.next.otf.compilation.cache`): + +- **Build caches**, one folder per compiled program variant, holding the build + artifacts and the compiled library. +- **Translation caches**, one pickled file per translated program, holding the + output of the translation step: the optimized SDFG for the DaCe backend, the + generated source for the gtfn backend. + +The translation cache is keyed by a fingerprint of the program plus the gt4py +version, not by the gt4py sources. Editing a transformation or an optimization +pass without changing either therefore leaves the fingerprint intact: the cached +translation is replayed and the pass never runs, while the build step still +recompiles and refreshes the library's mtime. A fresh library is thus evidence of +recompilation, never of re-translation. The two caches hit and miss independently, +so `list --by-program` reports both side by side. + +Run it in the environment that produced the cache, from the working directory the +cached run used (the default cache base is `/.gt4py_cache`), either through +the installed `gt4py-next-cache` command or as a module:: + + gt4py-next-cache list --by-program --filter 'apply_diffusion_*' + python -m gt4py.next.gt_cache_manager delete --program 'apply_diffusion_*' --yes +""" + +from __future__ import annotations + +import argparse +import collections +import dataclasses +import datetime +import fnmatch +import json +import pathlib +import pickle +import re +import shutil +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Any, Final + +from gt4py._core import locking +from gt4py.next import config +from gt4py.next.otf import stages +from gt4py.next.otf.compilation import cache +from gt4py.next.otf.compilation.build_systems import compiledb + + +ENTRY_SUFFIX: Final[str] = ".pkl" + +EXIT_OK: Final[int] = 0 +#: Nothing was done: a selector matched nothing, the user declined, or a gate tripped. +EXIT_NOTHING_DONE: Final[int] = 1 +#: The command could not run: bad arguments, or a confirmation that cannot be obtained. +EXIT_ERROR: Final[int] = 2 + +_BUILD_FOLDER_RE: Final[re.Pattern[str]] = re.compile(cache.CACHE_FOLDER_NAME_PATTERN) + + +class CacheDirError(RuntimeError): + """The given path is not usable as a gt4py.next cache base directory.""" + + +@dataclasses.dataclass(frozen=True) +class Entry: + """One translation cache entry, with what could be recovered from it. + + `program` is `None` exactly when the payload could not be unpickled, in which + case `error` describes why. Such an entry is dead weight rather than a + replay risk: the runtime cannot read it either and recomputes. + """ + + backend: str + key: str + path: pathlib.Path + size: int + mtime: float + program: str | None + error: str | None + + +@dataclasses.dataclass(frozen=True) +class BuildDir: + """One build cache folder, with what its name and contents say about it.""" + + program: str + path: pathlib.Path + version_id: str + + @property + def usable(self) -> bool: + """Whether a run in this environment could still be served by this folder. + + A folder built under a different build-cache version can never be hit + again: the version salts the folder name, so the next run looks for a + different folder and builds it. + """ + return self.version_id == config.BUILD_CACHE_VERSION_ID + + +@dataclasses.dataclass(frozen=True) +class ProgramSummary: + """What both caches hold for one program name. + + Counts, not predictions: a program name is all the caches record about their + contents, while a hit is decided by a fingerprint taken at run time over the + lowered program and its arguments. So an entry counted here may well be + unreachable, e.g. after its program's source was edited. + """ + + program: str + entries: Mapping[str, int] + build_dirs: int + usable_build_dirs: int + + @property + def entry_count(self) -> int: + return sum(self.entries.values()) + + +def get_cache_base(cache_dir: pathlib.Path | None = None) -> pathlib.Path: + """Return the cache base directory, or `cache_dir` if one is given. + + Args: + cache_dir: Explicit cache base. It is validated to look like a gt4py.next + cache, since the delete commands remove files underneath it. + + Returns: + The resolved cache base directory. It does not necessarily exist: a + session-lifetime cache base is created lazily. + + Raises: + CacheDirError: If `cache_dir` does not exist or does not look like a + gt4py.next cache base. + """ + if cache_dir is None: + return cache.get_cache_base_path(config.BUILD_CACHE_LIFETIME) + + path = cache_dir.expanduser().resolve() + if not path.is_dir(): + raise CacheDirError(f"'{path}' is not a directory.") + if not ( + path.name == config.BUILD_CACHE_DIR.name + or (path / cache.TRANSLATION_CACHE_DIR_NAME).is_dir() + or any(_BUILD_FOLDER_RE.fullmatch(child.name) for child in path.iterdir() if child.is_dir()) + ): + raise CacheDirError( + f"'{path}' does not look like a gt4py.next cache base directory: it is not named" + f" '{config.BUILD_CACHE_DIR.name}' and contains neither a translation cache" + f" ('{cache.TRANSLATION_CACHE_DIR_NAME}') nor a build cache folder." + ) + return path + + +def get_translation_cache_dirs( + cache_base: pathlib.Path, backends: Sequence[str] +) -> dict[str, pathlib.Path]: + """Return the translation cache directory of each of `backends`.""" + return { + backend: cache.get_translation_cache_folder(cache_base, backend) for backend in backends + } + + +def read_entry(path: pathlib.Path) -> stages.ProgramSource: + """Unpickle one translation cache entry. + + Reads the file directly instead of going through `filecache.FileCache`, which + deletes entries it fails to unpickle. + """ + with path.open("rb") as fp: + return pickle.load(fp) + + +def find_entries( + cache_base: pathlib.Path, backends: Sequence[str], *, program: str | None = None +) -> list[Entry]: + """Collect the translation cache entries of `backends`, optionally filtered. + + Args: + cache_base: Cache base directory. Missing cache directories are empty + caches, not errors. + backends: Backend names to scan. + program: If given, keep only entries whose program name matches this glob. + Unreadable entries have no program name and are dropped by any filter. + """ + entries = [] + for backend, cache_dir in get_translation_cache_dirs(cache_base, backends).items(): + if not cache_dir.is_dir(): + continue + for path in sorted(cache_dir.glob(f"*{ENTRY_SUFFIX}")): + file_stat = path.stat() + try: + program_name: str | None = read_entry(path).entry_point.name + error: str | None = None + except Exception as e: + program_name = None + error = f"{type(e).__name__}: {e}" + if program is not None and ( + program_name is None or not fnmatch.fnmatchcase(program_name, program) + ): + continue + entries.append( + Entry( + backend=backend, + key=path.stem, + path=path, + size=file_stat.st_size, + mtime=file_stat.st_mtime, + program=program_name, + error=error, + ) + ) + return entries + + +def find_build_dirs(cache_base: pathlib.Path, *, program: str | None = None) -> list[BuildDir]: + """Collect the build cache folders, optionally filtered by a program glob. + + The shared compiledb folder is named like a program's build folder but belongs + to no program, so it is left out and never deleted along with one. + """ + if not cache_base.is_dir(): + return [] + build_dirs = [] + for path in sorted(cache_base.iterdir()): + if not path.is_dir() or not (match := _BUILD_FOLDER_RE.fullmatch(path.name)): + continue + name = match.group("name").removesuffix(cache.BINDINGS_NAME_SUFFIX) + if name.startswith(compiledb.COMPILEDB_PROTOTYPE_NAME_PREFIX): + continue + if program is None or fnmatch.fnmatchcase(name, program): + build_dirs.append( + BuildDir(program=name, path=path, version_id=match.group("version_id")) + ) + return build_dirs + + +def summarize_programs( + cache_base: pathlib.Path, backends: Sequence[str], *, program: str | None = None +) -> tuple[list[ProgramSummary], list[Entry]]: + """Count what both caches hold, per program name. + + Returns: + A summary of every program with a translation cache entry or a build + folder, sorted by name, and the entries that could not be read. + """ + entries = find_entries(cache_base, backends, program=program) + build_dirs = find_build_dirs(cache_base, program=program) + + counts: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter) + for entry in entries: + if entry.program is not None: + counts[entry.program][entry.backend] += 1 + build_counts = collections.Counter(build_dir.program for build_dir in build_dirs) + usable_build_counts = collections.Counter( + build_dir.program for build_dir in build_dirs if build_dir.usable + ) + + return [ + ProgramSummary( + program=name, + entries=counts.get(name, {}), + build_dirs=build_counts.get(name, 0), + usable_build_dirs=usable_build_counts.get(name, 0), + ) + for name in sorted(counts.keys() | build_counts.keys()) + ], [entry for entry in entries if entry.program is None] + + +def delete_entries(entries: Sequence[Entry], cache_dirs: Sequence[pathlib.Path]) -> None: + """Remove translation cache entries, holding the same lock as the runtime. + + The caches are shared between concurrent processes (e.g. MPI ranks), so an + unlocked unlink can race a writer. + + Raises: + CacheDirError: If an entry is not a cache entry inside `cache_dirs`. + """ + for entry in entries: + if entry.path.suffix != ENTRY_SUFFIX or entry.path.parent not in cache_dirs: + raise CacheDirError( + f"refusing to remove '{entry.path}': not a translation cache entry." + ) + with locking.lock(entry.path): + entry.path.unlink(missing_ok=True) + + +def delete_build_dirs(build_dirs: Sequence[BuildDir], cache_base: pathlib.Path) -> None: + """Remove build cache folders, holding the same lock as the runtime. + + Raises: + CacheDirError: If a folder is not a build cache folder of `cache_base`. + """ + for build_dir in build_dirs: + if build_dir.path.parent != cache_base or not _BUILD_FOLDER_RE.fullmatch( + build_dir.path.name + ): + raise CacheDirError(f"refusing to remove '{build_dir.path}': not a build cache folder.") + with locking.lock(build_dir.path): + shutil.rmtree(build_dir.path) + + +def _format_size(size: int) -> str: + if size < 1024: + return f"{size} B" + scaled = size / 1024 + if scaled < 1024: + return f"{scaled:.1f} KiB" + return f"{scaled / 1024:.1f} MiB" + + +def _format_mtime(mtime: float) -> str: + return datetime.datetime.fromtimestamp(mtime).isoformat(sep=" ", timespec="seconds") + + +def _print_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> None: + widths = [max(len(str(row[i])) for row in [headers, *rows]) for i in range(len(headers))] + for row in [headers, *rows]: + print(" ".join(str(cell).ljust(width) for cell, width in zip(row, widths)).rstrip()) + + +def _cmd_path(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + rows = [("lifetime", config.BUILD_CACHE_LIFETIME.name.lower()), ("cache base", str(cache_base))] + rows += [ + (f"translation ({backend})", str(cache_dir)) + for backend, cache_dir in get_translation_cache_dirs(cache_base, args.backends).items() + ] + # Build folders are named per program variant, so name the pattern rather than a path. + rows.append( + ( + "build folders", + f"{cache_base}/{cache.BINDINGS_NAME_SUFFIX}__", + ) + ) + width = max(len(label) for label, _ in rows) + 2 + for label, value in rows: + print(f"{label + ':':{width}}{value}") + return EXIT_OK + + +def _cmd_list(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + if args.by_program: + return _list_by_program(args, cache_base) + return _list_entries(args, cache_base) + + +def _list_entries(args: argparse.Namespace, cache_base: pathlib.Path) -> int: + entries = find_entries(cache_base, args.backends, program=args.filter) + sort_keys: dict[str, Callable[[Entry], Any]] = { + "mtime": lambda e: e.mtime, + "size": lambda e: e.size, + "program": lambda e: (e.program or "", e.key), + "key": lambda e: e.key, + } + entries.sort(key=sort_keys[args.sort]) + + if args.json: + print( + json.dumps( + [{**dataclasses.asdict(e), "path": str(e.path)} for e in entries], + indent=2, + default=str, + ) + ) + elif not entries: + print("no entries") + else: + _print_table( + ("BACKEND", "KEY", "PROGRAM", "SIZE", "MTIME"), + [ + ( + e.backend, + e.key, + e.program if e.program is not None else "", + _format_size(e.size), + _format_mtime(e.mtime), + ) + for e in entries + ], + ) + return _fail_if_cached(args, len(entries)) + + +def _list_by_program(args: argparse.Namespace, cache_base: pathlib.Path) -> int: + summaries, unreadable = summarize_programs(cache_base, args.backends, program=args.filter) + + if args.json: + print(json.dumps([dataclasses.asdict(s) for s in summaries], indent=2, default=str)) + elif not summaries: + print("no cached programs") + else: + _print_table( + ("PROGRAM", "ENTRIES", "BUILDS"), + [ + ( + summary.program, + ", ".join(f"{n} {backend}" for backend, n in sorted(summary.entries.items())) + or "-", + f"{summary.usable_build_dirs}" + if summary.build_dirs == summary.usable_build_dirs + else f"{summary.usable_build_dirs}" + f" (+{summary.build_dirs - summary.usable_build_dirs} stale)", + ) + for summary in summaries + ], + ) + sys.stdout.flush() # keep the table above the notes when both go to a terminal + + if trap := [s for s in summaries if s.entry_count and not s.usable_build_dirs]: + print( + f"\nWARNING: {len(trap)} program(s) have a cached translation but no usable build" + " folder. Their next run rebuilds the library while the cached translation is" + " replayed, so a fresh library there would NOT mean a changed pass ran.", + file=sys.stderr, + ) + if stale := sum(s.build_dirs - s.usable_build_dirs for s in summaries): + print( + f"\nnote: {stale} build folder(s) cannot be hit by this environment: they were" + f" built under a build-cache version other than '{config.BUILD_CACHE_VERSION_ID}'." + " The same version salts the translation cache, but entries do not record it, so" + " entries left by those runs are still counted above although they cannot be hit" + " either.", + file=sys.stderr, + ) + if unreadable: + print( + f"\nnote: {len(unreadable)} entr(ies) could not be read; the runtime discards those" + " too, so they cannot be hit either.", + file=sys.stderr, + ) + return _fail_if_cached(args, sum(s.entry_count for s in summaries)) + + +def _fail_if_cached(args: argparse.Namespace, matched: int) -> int: + if args.fail_if_cached and matched: + print(f"error: {matched} cached entr(ies) matched.", file=sys.stderr) + return EXIT_NOTHING_DONE + return EXIT_OK + + +def _confirm(question: str) -> bool: + """Ask a yes/no question on an interactive terminal, defaulting to no.""" + try: + answer = input(f"{question} [y/N] ") + except (EOFError, KeyboardInterrupt): + print() + return False + return answer.strip().casefold() in ("y", "yes") + + +def _cmd_delete(args: argparse.Namespace) -> int: + cache_base = get_cache_base(args.cache_dir) + cache_dirs = get_translation_cache_dirs(cache_base, args.backends) + + entries = find_entries(cache_base, args.backends, program=args.program) + if args.key: + entries = [entry for entry in entries if entry.key in args.key] + + build_dirs: list[BuildDir] = [] + if args.include_build_dirs: + if args.key: + programs = {entry.program for entry in entries if entry.program is not None} + build_dirs = [b for b in find_build_dirs(cache_base) if b.program in programs] + else: + build_dirs = find_build_dirs(cache_base, program=args.program) + + if not entries and not build_dirs: + print("error: nothing matched the given selector.", file=sys.stderr) + return EXIT_NOTHING_DONE + + print(f"from {cache_base}:") + for entry in entries: + print(f" {entry.path.relative_to(cache_base)} ({entry.program or ''})") + for build_dir in build_dirs: + print(f" {build_dir.path.relative_to(cache_base)}/ [build]") + summary = f"{len(entries)} entr(ies) and {len(build_dirs)} build folder(s)" + + if args.dry_run: + print(f"\ndry run: nothing was removed ({summary} matched).") + return EXIT_OK + if not args.yes: + if not sys.stdin.isatty(): + print( + f"error: refusing to remove {summary} without a confirmation. Pass --yes to" + " confirm, or --dry-run to preview.", + file=sys.stderr, + ) + return EXIT_ERROR + if not _confirm(f"\nRemove {summary}?"): + print("aborted, nothing was removed.") + return EXIT_NOTHING_DONE + + delete_entries(entries, list(cache_dirs.values())) + delete_build_dirs(build_dirs, cache_base) + print(f"\nremoved {summary}.") + return EXIT_OK + + +def _add_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--cache-dir", + type=pathlib.Path, + default=None, + metavar="PATH", + help="Cache base directory to operate on (default: the one the current environment uses).", + ) + parser.add_argument( + "--backend", + dest="backends", + choices=[*cache.TRANSLATION_CACHE_BACKENDS, "all"], + default="all", + help="Translation cache(s) to operate on (default: all).", + ) + + +def _make_parser(prog: str | None = None) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=prog, + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + path_parser = subparsers.add_parser("path", help="Print the resolved cache directories.") + path_parser.set_defaults(func=_cmd_path) + + list_parser = subparsers.add_parser( + "list", + help="List what the caches hold.", + description=( + "List the translation cache entries, or with --by-program one row per program" + " together with its build folders. Everything reported is a count of what is on" + " disk, not a prediction: a hit is decided by a fingerprint taken at run time" + " over the lowered program and its arguments, which this tool does not have, so" + " an entry listed here may already be unreachable — after its program's source" + " was edited, for instance. Build folders record the build-cache version in" + " their name, so those that no run here can hit are counted as stale;" + " translation cache entries record no version and cannot be told apart that way." + ), + ) + list_parser.add_argument( + "--filter", metavar="GLOB", default=None, help="Only entries whose program name matches." + ) + list_parser.add_argument( + "--by-program", + action="store_true", + help="One row per program, with its build folders, instead of one row per entry.", + ) + list_parser.add_argument("--sort", choices=["mtime", "size", "program", "key"], default="mtime") + list_parser.add_argument("--json", action="store_true", help="Print as JSON.") + list_parser.add_argument( + "--fail-if-cached", + action="store_true", + help="Exit with a non-zero status if anything matched, to gate a run on an empty cache.", + ) + list_parser.set_defaults(func=_cmd_list) + + delete_parser = subparsers.add_parser( + "delete", + help="Remove translation cache entries.", + description=( + "List what the selector matches, then remove it after a confirmation. The" + " confirmation is only asked for on an interactive terminal; elsewhere --yes is" + " required, so a script never blocks on a prompt." + ), + ) + selectors = delete_parser.add_mutually_exclusive_group(required=True) + selectors.add_argument("--key", action="append", metavar="KEY", help="Entry key; repeatable.") + selectors.add_argument("--program", metavar="GLOB", help="Entries whose program name matches.") + selectors.add_argument("--all", action="store_true", help="All entries.") + confirmation = delete_parser.add_mutually_exclusive_group() + confirmation.add_argument( + "--yes", "-y", action="store_true", help="Remove without asking for confirmation." + ) + confirmation.add_argument( + "--dry-run", + "-n", + action="store_true", + help="Only describe what would be removed, then exit.", + ) + delete_parser.add_argument( + "--include-build-dirs", + action="store_true", + help="Also remove the build cache folders of the selected programs.", + ) + delete_parser.set_defaults(func=_cmd_delete) + + for subparser in (path_parser, list_parser, delete_parser): + _add_common_arguments(subparser) + return parser + + +def main(argv: Sequence[str] | None = None, *, prog: str | None = None) -> int: + """Run the CLI. Entry point of the `gt4py-next-cache` command.""" + args = _make_parser(prog).parse_args(argv) + args.backends = ( + list(cache.TRANSLATION_CACHE_BACKENDS) if args.backends == "all" else [args.backends] + ) + try: + return args.func(args) + except CacheDirError as e: + print(f"error: {e}", file=sys.stderr) + return EXIT_ERROR + + +if __name__ == "__main__": + # `python -m` leaves argparse with the module file as the program name. + sys.exit(main(prog="python -m gt4py.next.gt_cache_manager")) diff --git a/tests/next_tests/unit_tests/test_gt_cache_manager.py b/tests/next_tests/unit_tests/test_gt_cache_manager.py new file mode 100644 index 0000000000..e5f62b44e5 --- /dev/null +++ b/tests/next_tests/unit_tests/test_gt_cache_manager.py @@ -0,0 +1,515 @@ +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import pathlib + +import pytest + +from gt4py._core import filecache +from gt4py.next import config, gt_cache_manager +from gt4py.next.otf import code_specs, stages +from gt4py.next.otf.binding import interface +from gt4py.next.otf.compilation import cache +from gt4py.next.otf.compilation.build_systems import compiledb + + +def make_program_source(name: str, backend: str) -> stages.ProgramSource: + """Build a payload of the shape the translation step of `backend` caches.""" + if backend == "dace": + code_spec = code_specs.SDFGCodeSpec() + source_code = {"type": "SDFG", "attributes": {"name": name}, "nodes": []} + else: + code_spec = code_specs.CPPCodeSpec() + source_code = f"// generated code of {name}" + return stages.ProgramSource( + entry_point=interface.Function(name, ()), + source_code=source_code, + library_deps=(), + code_spec=code_spec, + ) + + +def write_entry(cache_base: pathlib.Path, backend: str, name: str, salt: str = "") -> str: + """Write one translation cache entry through `FileCache`, as the runtime does.""" + file_cache = filecache.FileCache(cache.get_translation_cache_folder(cache_base, backend)) + key = f"{backend}-{name}-{salt}" + file_cache[key] = make_program_source(name, backend) + return file_cache._get_path(key).stem + + +def write_build_dir( + cache_base: pathlib.Path, name: str, salt: str = "0", *, version_id: str | None = None +) -> pathlib.Path: + """Create a folder named the way `cache.get_cache_folder` names them.""" + version_id = config.BUILD_CACHE_VERSION_ID if version_id is None else version_id + folder = cache_base / f"{name}{cache.BINDINGS_NAME_SUFFIX}_{salt * 16}_{version_id}" + folder.mkdir(parents=True) + (folder / "libprogram.so").write_text("not really a library") + return folder + + +class FakeStdin: + """Stand-in for `sys.stdin` that reports whether it is a terminal.""" + + def __init__(self, interactive: bool) -> None: + self._interactive = interactive + + def isatty(self) -> bool: + return self._interactive + + +@pytest.fixture +def cache_base(tmp_path): + base = tmp_path / config.BUILD_CACHE_DIR.name + base.mkdir() + return base + + +def test_get_cache_base_follows_persistent_config(monkeypatch, tmp_path): + monkeypatch.setattr(config, "BUILD_CACHE_DIR", tmp_path / ".gt4py_cache") + monkeypatch.setattr(config, "BUILD_CACHE_LIFETIME", config.BuildCacheLifetime.PERSISTENT) + assert gt_cache_manager.get_cache_base() == tmp_path / ".gt4py_cache" + + +def test_get_cache_base_follows_session_config(monkeypatch): + monkeypatch.setattr(config, "BUILD_CACHE_LIFETIME", config.BuildCacheLifetime.SESSION) + assert gt_cache_manager.get_cache_base() == cache.get_cache_base_path( + config.BuildCacheLifetime.SESSION + ) + + +def test_get_cache_base_accepts_cache_like_dirs(tmp_path, cache_base): + assert gt_cache_manager.get_cache_base(cache_base) == cache_base.resolve() + + by_translation_cache = tmp_path / "elsewhere" + cache.get_translation_cache_folder(by_translation_cache, "dace").mkdir(parents=True) + assert gt_cache_manager.get_cache_base(by_translation_cache) == by_translation_cache.resolve() + + by_build_dir = tmp_path / "somewhere" + by_build_dir.mkdir() + write_build_dir(by_build_dir, "prog") + assert gt_cache_manager.get_cache_base(by_build_dir) == by_build_dir.resolve() + + +def test_get_cache_base_rejects_other_dirs(tmp_path): + (tmp_path / "unrelated").mkdir() + (tmp_path / "unrelated" / "notes.txt").write_text("hello") + with pytest.raises(gt_cache_manager.CacheDirError, match="does not look like"): + gt_cache_manager.get_cache_base(tmp_path / "unrelated") + + with pytest.raises(gt_cache_manager.CacheDirError, match="not a directory"): + gt_cache_manager.get_cache_base(tmp_path / "missing") + + +def test_missing_cache_dirs_are_an_empty_cache(cache_base): + assert gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"]) == [] + assert gt_cache_manager.find_build_dirs(cache_base) == [] + assert gt_cache_manager.summarize_programs(cache_base, ["dace", "gtfn"]) == ([], []) + + +def test_find_entries_reads_program_and_backend(cache_base): + dace_key = write_entry(cache_base, "dace", "foo") + gtfn_key = write_entry(cache_base, "gtfn", "bar") + + entries = gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"]) + + assert {(e.backend, e.key, e.program) for e in entries} == { + ("dace", dace_key, "foo"), + ("gtfn", gtfn_key, "bar"), + } + assert all(e.error is None and e.size > 0 for e in entries) + + +def test_find_entries_filters_by_program_and_backend(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "foobar") + write_entry(cache_base, "gtfn", "foo") + + assert len(gt_cache_manager.find_entries(cache_base, ["dace"], program="foo*")) == 2 + assert len(gt_cache_manager.find_entries(cache_base, ["dace", "gtfn"], program="foo")) == 2 + assert gt_cache_manager.find_entries(cache_base, ["gtfn"], program="nope") == [] + + +def test_find_entries_ignores_lock_files(cache_base): + key = write_entry(cache_base, "dace", "foo") + cache_dir = cache.get_translation_cache_folder(cache_base, "dace") + # A writer killed mid-write leaves its lock file behind. + (cache_dir / f"{key}.filelock_FileLock.lock").touch() + + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_find_build_dirs_recovers_program_name(cache_base): + write_build_dir(cache_base, "foo") + (cache_base / "not-a-build-dir").mkdir() + + build_dirs = gt_cache_manager.find_build_dirs(cache_base) + + assert [b.program for b in build_dirs] == ["foo"] + assert gt_cache_manager.find_build_dirs(cache_base, program="ba*") == [] + + +def test_find_build_dirs_skips_the_shared_compiledb(cache_base): + write_build_dir(cache_base, "foo") + compiledb_dir = write_build_dir( + cache_base, f"{compiledb.COMPILEDB_PROTOTYPE_NAME_PREFIX}_gridtools_cpu_Release", salt="1" + ) + + assert [b.program for b in gt_cache_manager.find_build_dirs(cache_base)] == ["foo"] + + gt_cache_manager.main( + ["delete", "--all", "--include-build-dirs", "--yes", "--cache-dir", str(cache_base)] + ) + + assert compiledb_dir.is_dir() + + +def test_corrupt_entry_degrades_instead_of_crashing(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + corrupt = cache.get_translation_cache_folder(cache_base, "dace") / "deadbeefdeadbeef.pkl" + corrupt.write_bytes(b"not a pickle") + + entries = gt_cache_manager.find_entries(cache_base, ["dace"]) + corrupt_entries = [e for e in entries if e.key == corrupt.stem] + + assert len(entries) == 2 + assert corrupt_entries[0].program is None + assert corrupt_entries[0].error + assert corrupt.exists() # inspection must not delete what it cannot read + + assert gt_cache_manager.main(["list", "--cache-dir", str(cache_base)]) == 0 + assert "" in capsys.readouterr().out + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + assert "could not be read" in capsys.readouterr().err + + +def test_summarize_groups_entries_and_builds_by_program(cache_base): + write_entry(cache_base, "dace", "cached") + write_entry(cache_base, "dace", "cached", salt="second") + write_entry(cache_base, "gtfn", "cached") + write_build_dir(cache_base, "built_only") + + summaries, unreadable = gt_cache_manager.summarize_programs(cache_base, ["dace", "gtfn"]) + + assert unreadable == [] + assert [s.program for s in summaries] == ["built_only", "cached"] + assert summaries[0].entry_count == 0 + assert summaries[0].build_dirs == 1 + assert summaries[1].entries == {"dace": 2, "gtfn": 1} + assert summaries[1].build_dirs == 0 + + +def test_summarize_counts_the_two_caches_independently(cache_base): + write_entry(cache_base, "dace", "warm") + write_build_dir(cache_base, "warm") + write_entry(cache_base, "dace", "translated_only") + write_build_dir(cache_base, "built_only") + + summaries = {s.program: s for s in gt_cache_manager.summarize_programs(cache_base, ["dace"])[0]} + + assert (summaries["warm"].entry_count, summaries["warm"].usable_build_dirs) == (1, 1) + assert ( + summaries["translated_only"].entry_count, + summaries["translated_only"].usable_build_dirs, + ) == (1, 0) + assert (summaries["built_only"].entry_count, summaries["built_only"].usable_build_dirs) == ( + 0, + 1, + ) + + +def test_build_from_another_version_is_stale(cache_base): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo", version_id="0.0.1+ancient") + + (summary,) = gt_cache_manager.summarize_programs(cache_base, ["dace"])[0] + + assert summary.build_dirs == 1 + assert summary.usable_build_dirs == 0 + + +def test_list_by_program_warns_about_a_cached_translation_without_a_build(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + captured = capsys.readouterr() + assert "1 dace" in captured.out + assert "would NOT mean a changed pass ran" in captured.err + + +def test_list_by_program_does_not_warn_when_both_caches_are_warm(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + assert "WARNING" not in capsys.readouterr().err + + +def test_list_by_program_notes_that_entries_may_be_stale_too(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + write_build_dir(cache_base, "foo", version_id="0.0.1+ancient") + + assert gt_cache_manager.main(["list", "--by-program", "--cache-dir", str(cache_base)]) == 0 + + assert "cannot be hit either" in capsys.readouterr().err + + +@pytest.mark.parametrize("extra", [[], ["--by-program"]]) +def test_list_fail_if_cached(cache_base, extra): + write_build_dir(cache_base, "foo") + argv = ["list", "--cache-dir", str(cache_base), "--fail-if-cached", *extra] + + assert gt_cache_manager.main(argv) == 0 # a build folder alone is not a cached translation + + write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(argv) == gt_cache_manager.EXIT_NOTHING_DONE + + +@pytest.mark.parametrize("extra", [[], ["--by-program"]]) +def test_list_filter(cache_base, capsys, extra): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + gt_cache_manager.main(["list", "--cache-dir", str(cache_base), "--filter", "fo*", *extra]) + + captured = capsys.readouterr() + assert "foo" in captured.out + assert "bar" not in captured.out + + +def test_delete_dry_run_keeps_entries(cache_base, capsys): + write_entry(cache_base, "dace", "foo") + + assert ( + gt_cache_manager.main(["delete", "--all", "--dry-run", "--cache-dir", str(cache_base)]) == 0 + ) + + assert "dry run" in capsys.readouterr().out + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_asks_before_removing_on_a_terminal(cache_base, monkeypatch, capsys): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + monkeypatch.setattr("builtins.input", lambda prompt="": "y") + + assert gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) == 0 + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + + +@pytest.mark.parametrize("answer", ["n", "", "no", "whatever"]) +def test_delete_declined_removes_nothing(cache_base, monkeypatch, answer): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + monkeypatch.setattr("builtins.input", lambda prompt="": answer) + + result = gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + + assert result == gt_cache_manager.EXIT_NOTHING_DONE + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_never_prompts_without_a_terminal(cache_base, monkeypatch, capsys): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=False)) + + def _no_prompting(prompt=""): + raise AssertionError("a non-interactive run must not block on a prompt") + + monkeypatch.setattr("builtins.input", _no_prompting) + + result = gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + + assert result == gt_cache_manager.EXIT_ERROR + assert "--yes" in capsys.readouterr().err + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_declined_at_eof_removes_nothing(cache_base, monkeypatch): + write_entry(cache_base, "dace", "foo") + monkeypatch.setattr(gt_cache_manager.sys, "stdin", FakeStdin(interactive=True)) + + def _eof(prompt=""): + raise EOFError + + monkeypatch.setattr("builtins.input", _eof) + + assert ( + gt_cache_manager.main(["delete", "--all", "--cache-dir", str(cache_base)]) + == gt_cache_manager.EXIT_NOTHING_DONE + ) + assert len(gt_cache_manager.find_entries(cache_base, ["dace"])) == 1 + + +def test_delete_removes_selected_entries(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + assert ( + gt_cache_manager.main( + ["delete", "--program", "foo", "--yes", "--cache-dir", str(cache_base)] + ) + == 0 + ) + + assert [e.program for e in gt_cache_manager.find_entries(cache_base, ["dace"])] == ["bar"] + + +def test_delete_by_key(cache_base): + key = write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "dace", "bar") + + assert ( + gt_cache_manager.main(["delete", "--key", key, "--yes", "--cache-dir", str(cache_base)]) + == 0 + ) + + assert [e.program for e in gt_cache_manager.find_entries(cache_base, ["dace"])] == ["bar"] + + +def test_delete_without_match_exits_non_zero(cache_base): + write_entry(cache_base, "dace", "foo") + + assert ( + gt_cache_manager.main( + ["delete", "--program", "nope", "--yes", "--cache-dir", str(cache_base)] + ) + == gt_cache_manager.EXIT_NOTHING_DONE + ) + + +def test_delete_leaves_build_dirs_alone(cache_base): + write_entry(cache_base, "dace", "foo") + build_dir = write_build_dir(cache_base, "foo") + + assert gt_cache_manager.main(["delete", "--all", "--yes", "--cache-dir", str(cache_base)]) == 0 + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert build_dir.is_dir() + + +def test_delete_include_build_dirs(cache_base): + write_entry(cache_base, "dace", "foo") + foo_build_dir = write_build_dir(cache_base, "foo") + bar_build_dir = write_build_dir(cache_base, "bar") + + assert ( + gt_cache_manager.main( + [ + "delete", + "--program", + "foo", + "--include-build-dirs", + "--yes", + "--cache-dir", + str(cache_base), + ] + ) + == 0 + ) + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert not foo_build_dir.exists() + assert bar_build_dir.is_dir() + + +def test_delete_backend_selection(cache_base): + write_entry(cache_base, "dace", "foo") + write_entry(cache_base, "gtfn", "foo") + + assert ( + gt_cache_manager.main( + ["delete", "--all", "--yes", "--backend", "dace", "--cache-dir", str(cache_base)] + ) + == 0 + ) + + assert gt_cache_manager.find_entries(cache_base, ["dace"]) == [] + assert len(gt_cache_manager.find_entries(cache_base, ["gtfn"])) == 1 + + +def test_delete_refuses_paths_outside_the_cache(cache_base, tmp_path): + outsider = tmp_path / "precious.pkl" + outsider.write_text("keep me") + entry = gt_cache_manager.Entry( + backend="dace", + key=outsider.stem, + path=outsider, + size=0, + mtime=0.0, + program="foo", + error=None, + ) + + with pytest.raises(gt_cache_manager.CacheDirError, match="refusing to remove"): + gt_cache_manager.delete_entries( + [entry], [cache.get_translation_cache_folder(cache_base, "dace")] + ) + + assert outsider.exists() + + +def test_delete_refuses_build_dirs_outside_the_cache(cache_base, tmp_path): + outsider = tmp_path / "precious" + outsider.mkdir() + build_dir = gt_cache_manager.BuildDir( + program="foo", path=outsider, version_id=config.BUILD_CACHE_VERSION_ID + ) + + with pytest.raises(gt_cache_manager.CacheDirError, match="refusing to remove"): + gt_cache_manager.delete_build_dirs([build_dir], cache_base) + + assert outsider.is_dir() + + +def test_path_command_reports_both_caches_and_the_build_folders(cache_base, capsys): + assert gt_cache_manager.main(["path", "--cache-dir", str(cache_base)]) == 0 + + out = capsys.readouterr().out + assert str(cache_base.resolve()) in out + for backend in cache.TRANSLATION_CACHE_BACKENDS: + assert str(cache.get_translation_cache_folder(cache_base.resolve(), backend)) in out + # the confusion this answers: build folders are siblings of the translation + # caches, not something inside them + assert cache.BINDINGS_NAME_SUFFIX in out + + +def test_list_json_output(cache_base, capsys): + import json + + key = write_entry(cache_base, "dace", "foo") + + assert gt_cache_manager.main(["list", "--json", "--cache-dir", str(cache_base)]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert [(e["key"], e["program"], e["backend"]) for e in payload] == [(key, "foo", "dace")] + + +def test_console_script_points_at_main(): + import importlib.metadata + + entry_points = importlib.metadata.distribution("gt4py").entry_points + console_scripts = {ep.name: ep for ep in entry_points if ep.group == "console_scripts"} + + assert "gt4py-next-cache" in console_scripts + assert console_scripts["gt4py-next-cache"].load() is gt_cache_manager.main + + +def test_cli_reports_cache_dir_error(tmp_path, capsys): + (tmp_path / "unrelated").mkdir() + + assert ( + gt_cache_manager.main(["list", "--cache-dir", str(tmp_path / "unrelated")]) + == gt_cache_manager.EXIT_ERROR + ) + assert "does not look like" in capsys.readouterr().err