diff --git a/bits b/bits index 8d96ec4..c28654b 100755 --- a/bits +++ b/bits @@ -20,6 +20,15 @@ fi ARGV=("$@"); ARGC=$# # ARGC must be a plain integer, not an array +# `bits preload …` MUST be handled before the global `--config` pre-parse below, +# which would otherwise swallow preload's own `--config ` (a name clash with +# the bits.rc `--config`). Post-publish CVMFS filebundle generator: traces the +# deployed tests (strace, optionally --docker) and writes .cvmfsbundle-* into one +# tar. Self-contained; needs no work dir. Uses the raw ARGV, unmangled. +if [[ "${ARGV[0]:-}" == "preload" ]]; then + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.preload_cmd import main; sys.exit(main())' "${ARGV[@]:1}" +fi + # Optional phase timing for diagnosing slow load/enter: set BITS_TIMING=1 to # print the wall time of each phase to stderr. Uses bash EPOCHREALTIME (>=5), # else GNU date; no overhead when unset. diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 0413a44..ed64f87 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -723,6 +723,9 @@ def storeHook(package, specs, defaults) -> bool: _HASH_EXCLUDED_META_KEYS = frozenset({ "license", "description", "url", "homepage", "acknowledgment", "acknowledgement", "source_url", "redistributable", + # preload: CVMFS filebundle test list, consumed post-publish by `bits preload`; + # it never affects the build, so editing it must not force a rebuild. + "preload", }) # Source-selection keys are ALSO dropped from the recipe TEXT hash — not because diff --git a/bits_helpers/cvmfs_publish.py b/bits_helpers/cvmfs_publish.py index 4bf101a..7bc8daa 100644 --- a/bits_helpers/cvmfs_publish.py +++ b/bits_helpers/cvmfs_publish.py @@ -363,6 +363,29 @@ def _publish_tar(ctx, path, tar, label, fp=None): return jid +def publish_overlay_tars(ctx, tars): + """Publish each overlay tar at the repo ROOT (path="") via the staged/ingest + path; return the job ids. + + An overlay tar carries files at their repo-relative locations (e.g. the + ``.cvmfsbundle-*`` files `bits preload` writes next to their triggers), so a + single root lease drops each file into place — unlike a package tar staged at + one subtree. The caller's tar is preserved: a temp copy is published + (``_publish_tar`` removes what it publishes). Each tar gets a distinct + ``job_id_base`` so their leases do not collide. + """ + import shutil + base = ctx.get("job_id_base", "local") + jids = [] + for i, tar in enumerate(tars): + fd, tmp = tempfile.mkstemp(suffix=".tar", dir=ctx.get("tmp_dir") or None) + os.close(fd) + shutil.copyfile(tar, tmp) + jids.append(_publish_tar(dict(ctx, job_id_base="%s-tar%d" % (base, i)), + "", tmp, os.path.basename(tar))) + return jids + + def publish_one(spec, ctx): """Full producer pipeline for ONE package, staged OR ingest path. Mirrors the CI loop body: locate tar -> untar -> resolve path -> relocate -> relativise -> tar -> @@ -537,7 +560,11 @@ def main(argv=None): help="print the content fingerprint of a directory tree and " "exit (the CI uses this to fingerprint its own relocated " "tree with the identical algorithm); other args ignored") - ap.add_argument("--manifest") # required unless --fingerprint (checked below) + ap.add_argument("--manifest") # required unless --fingerprint/--tar (checked below) + ap.add_argument("--tar", action="append", default=[], metavar="FILE", + help="publish overlay tar(s) at the repo root (e.g. the bundle " + "tar from 'bits preload') via the staged/ingest path, " + "instead of a build manifest. Repeatable.") ap.add_argument("--repo") ap.add_argument("--one", help="publish only this package (increment-1 test)") ap.add_argument("--tars-root", default=os.path.join( @@ -594,6 +621,23 @@ def main(argv=None): if a.fingerprint: print(tree_fingerprint(a.fingerprint)) return 0 + if a.tar: + if not a.repo: + ap.error("--tar requires --repo") + ctx = {"repo": a.repo, "stratum0_url": a.stratum0_url, + "prepub_url": a.prepub_url, "token": a.token, + "job_id_base": a.job_id_base, "build_id": a.build_id, + "swissknife": a.swissknife or None, "base_root": a.base_root or None, + "bearer_auth": a.bearer_auth, "no_stats_db": a.no_stats_db, + "no_prepare_lock": a.no_prepare_lock, + "replace_on_conflict": a.replace_on_conflict, + "publish_path": a.publish_path, "direct_s3": a.direct_s3, + "submit": not a.dry_run, + "tmp_dir": os.path.join(os.environ.get("BITS_WORK_DIR", "/tmp"), "tmp")} + os.makedirs(ctx["tmp_dir"], exist_ok=True) + for tar, jid in zip(a.tar, publish_overlay_tars(ctx, a.tar)): + print("published overlay %s -> %s" % (tar, jid)) + return 0 if not a.manifest or not a.repo: ap.error("--manifest and --repo are required") if a.dry_run: diff --git a/bits_helpers/preload_bundle.py b/bits_helpers/preload_bundle.py new file mode 100644 index 0000000..1fe2a3b --- /dev/null +++ b/bits_helpers/preload_bundle.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""CVMFS filebundle spec emitter for the post-publish `bits preload` tool. + +The tool straces a trigger binary that already lives on a deployed CVMFS tree, +so every opened path is already an absolute ``/cvmfs/.cern.ch/…`` path. +Turning that into a filebundle spec (per +https://cvmfs.readthedocs.io/en/stable/cpt-file-bundles/) is therefore just: + + * keep the opens under the repo mount (drop system files, /proc, …); + * strip the mount to get repository-root-absolute paths (``/lcg/releases/…``); + * emit ``/.cvmfsbundle-`` next to the trigger, listing those + paths as ``dependencies`` (the trigger itself excluded). + +No relocation and no cross-package resolution: the deployed paths are already +final. This module is the pure, unit-testable core; the tool driver +(`preload_cmd`) handles recipe parsing, env setup, strace, tar and publish. +""" + +import json +import os + + +SPEC_NAME = "CVMFS_BUNDLE" +SPEC_VERSION = "1.0.0" +SPEC_ENCODING = "UTF-8" + + +def repo_root_of(cvmfs_path): + """The repo mount root of a ``/cvmfs//…`` path, i.e. ``/cvmfs/``. + + E.g. ``/cvmfs/sft.cern.ch/lcg/releases`` -> ``/cvmfs/sft.cern.ch``. Returns + None when *cvmfs_path* is not under ``/cvmfs//``. + """ + parts = [p for p in (cvmfs_path or "").split("/") if p] + if len(parts) >= 2 and parts[0] == "cvmfs": + return "/cvmfs/" + parts[1] + return None + + +def to_repo_absolute(abs_path, repo_root): + """``/cvmfs//a/b`` -> ``/a/b``; None if *abs_path* is not under the repo. + + The result is repository-root-absolute (leading slash), exactly the form the + filebundle spec's ``dependencies`` want. + """ + root = (repo_root or "").rstrip("/") + # Canonicalise: env setup can leave '/./' or '//' in captured paths (e.g. the + # BITS_ARCH_PREFIX="." trick used to source a deployed init.sh). + abs_path = os.path.normpath(abs_path) + if not root or abs_path == root or not abs_path.startswith(root + "/"): + return None + return abs_path[len(root):] # keeps the leading '/' + + +def _is_safe_rel(rel): + """True if *rel* is a plain, in-tree relative path (no abs, no ``..``, no NUL).""" + if not rel or rel.startswith("/") or "\x00" in rel: + return False + return ".." not in rel.split("/") + + +def render_spec(dependencies): + """The versioned filebundle JSON document for *dependencies* (verbatim keys).""" + return { + "name": SPEC_NAME, + "version": SPEC_VERSION, + "encoding": SPEC_ENCODING, + "dependencies": list(dependencies), + } + + +def bundle_path_for(path): + """``/.cvmfsbundle-`` for a trigger path (any form: abs or rel).""" + d, base = os.path.split(path) + name = ".cvmfsbundle-" + base + return (d + "/" + name) if d else name + + +def build_bundle(trigger_abs, opened_abs, repo_root): + """Build one bundle from a trigger and the files its launch opened. + + Returns ``(tar_relpath, spec_dict)`` — *tar_relpath* is the bundle's location + relative to the repo root (no leading slash), for placement in the staging + tar; *spec_dict* is the filebundle JSON. Returns ``(None, None)`` when the + trigger is not under the repo or nothing under the repo was opened (no empty + bundle). Opens outside the repo (system libs, /proc) and the trigger itself + are excluded; the result is sorted and de-duplicated. + """ + trig_rel = to_repo_absolute(trigger_abs, repo_root) + if not trig_rel: + return None, None + deps = set() + for p in opened_abs: + if p == trigger_abs: + continue + r = to_repo_absolute(p, repo_root) + if r: + deps.add(r) + if not deps: + return None, None + bundle_abs = bundle_path_for(trig_rel) # '/…/bin/.cvmfsbundle-root' + return bundle_abs.lstrip("/"), render_spec(sorted(deps)) + + +def stage_bundle(staging_dir, tar_relpath, spec): + """Write *spec* as JSON to ``/`` (dirs created). + + *tar_relpath* must be a safe in-tree relative path. Returns the file path + written. The staging tree mirrors the repo layout so a single tar of it + drops each bundle next to its trigger on publish. + """ + if not _is_safe_rel(tar_relpath): + raise ValueError("unsafe bundle path: %r" % (tar_relpath,)) + dest = os.path.join(staging_dir, tar_relpath) + os.makedirs(os.path.dirname(dest) or staging_dir, exist_ok=True) + with open(dest, "w", encoding="utf-8") as fh: + json.dump(spec, fh, indent=2) + fh.write("\n") + return dest diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py new file mode 100644 index 0000000..923cfba --- /dev/null +++ b/bits_helpers/preload_cmd.py @@ -0,0 +1,426 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""`bits preload` — post-publish CVMFS filebundle generator. + +Given a list of packages and a deployed CVMFS path, for each package that +declares a ``Preload()`` in its recipe: locate it under the tree, set up its +deployed environment, strace the recipe's trigger (via a tool-provided +``cvmfs_preload`` so ``bits-recipe-tools`` is untouched), turn the captured +``/cvmfs`` opens into ``.cvmfsbundle-`` spec files, tar them, and +publish that one tar into CVMFS. + +This module keeps the pure, unit-testable steps (recipe `Preload` detection, +locating a package on the tree, parsing the tracer output, assembling bundles +and the tar). The strace, environment setup and publish are shelled out and are +validated on a host with a real mounted repo — they cannot run in CI. +""" + +import os +import re +import shlex +import tarfile + +from bits_helpers import preload_bundle as B + +# open("/path", ...) = / openat(AT_FDCWD, "/path", ...) = — capture +# the path AND the syscall return so failed probes (= -1 ENOENT) are dropped. +_OPEN_RE = re.compile( + r'(?:\bopen\(|\bopenat\(AT_FDCWD,\s*)"([^"]+)"[^=\n]*=\s*(-?\d+)') + + +def _norm_tests(raw): + """Normalise a ``preload:`` / config ``tests:`` list to ``[(exe, [args]), …]``. + + Accepts either mapping entries (``{exe: bin/x, args: [--v]}``) or bare + strings (``"bin/x --v"``, shlex-split). Entries without an exe are skipped. + """ + out = [] + for item in raw or []: + if isinstance(item, dict): + exe = item.get("exe") + args = item.get("args") or [] + if isinstance(args, str): + args = shlex.split(args) + args = [str(a) for a in args] + elif isinstance(item, str): + try: + toks = shlex.split(item) + except ValueError: + continue + exe, args = (toks[0] if toks else None), toks[1:] + else: + continue + if exe: + out.append((str(exe), args)) + return out + + +def recipe_tests(spec): + """Tests from a recipe's hash-excluded ``preload:`` section (via ``parseRecipe``). + + *spec* is the parsed YAML front-matter; ``spec['preload']`` is the test list. + Returns ``[(exe, [args]), …]`` (empty when the recipe declares none). + """ + if not isinstance(spec, dict): + return [] + return _norm_tests(spec.get("preload")) + + +def load_config(yaml_text): + """Parse ``config/preload.yaml`` into a normalised sweep config. + + Returns ``{arch: [..]|None, docker: bool, update: bool, packages: {name: + {tests: [(exe,[args])], versions: [..]}}}``. ``packages`` accepts a bare list + (names defer to the recipe), a mapping, or a list mixing names and single-key + mappings; ``arch`` accepts a scalar or list (None ⇒ discover). An empty/{} + ``packages`` means "all packages that carry a recipe preload:". + """ + from bits_helpers.utilities import yamlLoad + data = yamlLoad(yaml_text) or {} + arch = data.get("arch") + if isinstance(arch, str): + arch = [arch] + pkgs = {} + + def _add(name, spec): + spec = spec or {} + pkgs[name] = {"tests": _norm_tests(spec.get("tests")), + "versions": list(spec.get("versions") or [])} + + raw = data.get("packages") + if isinstance(raw, dict): + for name, spec in raw.items(): + _add(name, spec) + elif isinstance(raw, list): + for item in raw: + if isinstance(item, str): + _add(item, {}) + elif isinstance(item, dict) and len(item) == 1: + (name, spec), = item.items() + _add(name, spec) + return {"cvmfs": data.get("cvmfs") or None, + "arch": arch or None, + "docker": bool(data.get("docker", False)), + "update": bool(data.get("update", False)), + "packages": pkgs} + + +def resolve_tests(cfg_pkg, recipe_spec): + """Tests for a package: the config's ``tests`` if any, else the recipe's. + + *cfg_pkg* is a ``load_config`` package entry (or None); *recipe_spec* the + package's parsed recipe front-matter (or None). + """ + if cfg_pkg and cfg_pkg.get("tests"): + return list(cfg_pkg["tests"]) + return recipe_tests(recipe_spec) + + +def bundle_exists(pkg_dir, exe): + """True if ``//.cvmfsbundle-`` already exists.""" + return os.path.exists(os.path.join(pkg_dir, B.bundle_path_for(exe))) + + +def discover_archs(cvmfs_root): + """Deployed platforms under *cvmfs_root* (reuses cvmfs_inspect).""" + from bits_helpers import cvmfs_inspect as I + return I.list_platforms(cvmfs_root) + + +def package_dir(cvmfs_root, arch, pkg, verrev): + """``//Packages//`` — a deployed package dir.""" + return os.path.join(cvmfs_root, arch, "Packages", pkg, verrev) + + +def package_versions(cvmfs_root, arch, pkg, want=None): + """Deployed ```` dirs for *pkg* (reuses cvmfs_inspect), newest last. + + *want* is an optional list of version globs (from config ``versions:``); a + bare version matches its ``-`` dir too (``5.9.1`` ⇒ ``5.9.1-1``). + """ + import fnmatch + from bits_helpers import cvmfs_inspect as I + vers = I.list_packages(cvmfs_root, arch).get(pkg, []) + if want: + vers = [v for v in vers + if any(fnmatch.fnmatch(v, w) or fnmatch.fnmatch(v, w + "-*") + for w in want)] + return vers + + +def parse_strace_opens(strace_text): + """Absolute paths SUCCESSFULLY opened in ``strace -e open,openat`` output. + + Keeps only opens whose syscall returned a valid fd (``= N`` with N >= 0), so + the dynamic loader's failed probes (``= -1 ENOENT`` in ``glibc-hwcaps/``, + ``tls/`` and arch subdirs) are excluded — otherwise the bundle lists files + that do not exist. First-seen order, de-duplicated; relative paths (from an + already-chdir'd process) are dropped as they cannot be mapped to the repo. + """ + out, seen = [], set() + for path, ret in _OPEN_RE.findall(strace_text or ""): + if ret.startswith("-"): # failed syscall (-1 ENOENT, …) + continue + if path.startswith("/") and path not in seen: + seen.add(path) + out.append(path) + return out + + +def locate_package(cvmfs_path, pkg, ver=None): + """Deployed package directory ``…//`` under *cvmfs_path*. + + Finds a directory named *pkg* that itself contains a version directory; when + *ver* is given, the version dir must equal it or start with ``ver + '-'`` + (so ``6.24.06`` matches ``6.24.06-4``). Returns the newest matching verrev's + absolute path, or None. Does not follow symlinks out of the tree. + """ + hits = [] + for dirpath, dirnames, _ in os.walk(cvmfs_path): + if os.path.basename(dirpath) != pkg: + continue + for v in sorted(dirnames): + if ver and not (v == ver or v.startswith(ver + "-")): + continue + hits.append(os.path.join(dirpath, v)) + dirnames[:] = [] # a dir's children are versions, stop + if not hits: + return None + return sorted(hits)[-1] + + +def assemble_bundles(traces, repo_root, staging_dir): + """Stage a ``.cvmfsbundle-*`` file for every trace block that yields deps. + + *traces* is the parsed tracer output; each ``(trigger, opens)`` becomes a + bundle via :func:`preload_bundle.build_bundle`, written into *staging_dir* + at its repo-relative path. Returns the sorted list of staged tar-relative + paths (empty when nothing under the repo was opened). + """ + staged = [] + for trigger_abs, opened_abs in traces: + tar_rel, spec = B.build_bundle(trigger_abs, opened_abs, repo_root) + if not tar_rel: + continue + B.stage_bundle(staging_dir, tar_rel, spec) + staged.append(tar_rel) + return sorted(staged) + + +def packages_base(pkg_dir): + """The ``…//Packages`` base for a ``…/Packages//`` dir.""" + return os.path.dirname(os.path.dirname(pkg_dir)) + + +def build_trace_script(pkg_dir, exe, args, log_path): + """Bash that sets up the deployed env and straces one trigger to *log_path*. + + Sources the deployed ``init.sh`` with ``WORK_DIR=`` and + ``BITS_ARCH_PREFIX="."`` (so its ``$WORK_DIR/$BITS_ARCH_PREFIX/`` deps + resolve without doubling the arch — validated on the host), then runs + ``strace -f -e trace=open,openat``. No ``set -u``: init.sh references unset + vars by design. Returns the script text. + """ + import shlex + exe_abs = os.path.join(pkg_dir, exe) + quoted = " ".join(shlex.quote(a) for a in (args or [])) + return "\n".join([ + "set -o pipefail", + "export WORK_DIR=%s" % shlex.quote(packages_base(pkg_dir)), + 'export BITS_ARCH_PREFIX="."', + "INIT=%s/etc/profile.d/init.sh" % shlex.quote(pkg_dir), + '[ -r "$INIT" ] && . "$INIT" 2>/dev/null || true', + "strace -f -e trace=open,openat -o %s %s %s >/dev/null 2>&1 || true" + % (shlex.quote(log_path), shlex.quote(exe_abs), quoted), + ]) + "\n" + + +def make_tar(staging_dir, out_tar): + """Tar the staged bundle tree (paths relative to *staging_dir*) into *out_tar*. + + Deterministic ordering. Returns *out_tar*. An empty staging tree yields an + empty tar, which the caller should not publish. + """ + entries = [] + for dp, _dn, fns in os.walk(staging_dir): + for f in fns: + full = os.path.join(dp, f) + entries.append((full, os.path.relpath(full, staging_dir))) + with tarfile.open(out_tar, "w") as tf: + for full, arc in sorted(entries, key=lambda e: e[1]): + tf.add(full, arcname=arc, recursive=False) + return out_tar + + +# ── host-only trace executors (proven by run-preload-xrootd.sh) ─────────────── + +# A trigger that blocks (waits on stdin/network) must not stall the sweep. +_TRACE_TIMEOUT = 300 + + +def _trace_native(pkg_dir, exe, args, log_dir): + """Run the trace script on the host; return the strace log text ("" on error).""" + import subprocess + log = os.path.join(log_dir, "strace.log") + script = build_trace_script(pkg_dir, exe, args, log) + try: + subprocess.run(["bash", "-c", script], check=False, timeout=_TRACE_TIMEOUT) + with open(log) as fh: + return fh.read() + except Exception: # never let one trace abort the sweep + return "" + + +def _trace_docker(pkg_dir, exe, args, log_dir, image, cvmfs_mount="/cvmfs"): + """Run the trace script inside *image* with /cvmfs bind-mounted + SYS_PTRACE.""" + import subprocess + script = build_trace_script(pkg_dir, exe, args, "/out/strace.log") + cmd = ["docker", "run", "--rm", "-i", + "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", + "-v", "%s:%s:ro,rslave" % (cvmfs_mount, cvmfs_mount), + "-v", "%s:/out" % log_dir, image, "bash", "-s"] + try: + subprocess.run(cmd, input=script, text=True, check=False, + timeout=_TRACE_TIMEOUT) + with open(os.path.join(log_dir, "strace.log")) as fh: + return fh.read() + except Exception: # docker missing, timeout, bad image, … + return "" + + +def _default_tracer(docker=False, image=None): + """Return a tracer(pkg_dir, exe, args) -> [opened_abs] for the sweep.""" + import tempfile + + def tracer(pkg_dir, exe, args): + d = tempfile.mkdtemp(prefix="preload-") + try: + text = (_trace_docker(pkg_dir, exe, args, d, image) if docker + else _trace_native(pkg_dir, exe, args, d)) + finally: + __import__("shutil").rmtree(d, ignore_errors=True) + return parse_strace_opens(text) + return tracer + + +# ── the sweep ───────────────────────────────────────────────────────────────── + +def sweep(cvmfs_root, config, recipe_reader, staging_dir, + tracer, update=False, log=None, repo_root=None): + """Run the config-driven sweep, staging bundles into *staging_dir*. + + *recipe_reader(pkg)* returns a package's parsed recipe front-matter (or None); + *tracer(pkg_dir, exe, args)* returns the opened absolute paths (injected so + the loop is testable without strace). Skips a test whose bundle already exists + unless *update*. *repo_root* defaults to the mount of *cvmfs_root* (override + only in tests). Returns the sorted list of staged bundle tar-paths. + """ + log = log or (lambda *a: None) + repo_root = repo_root or B.repo_root_of(cvmfs_root) + if not repo_root: + raise ValueError("not a /cvmfs//… path: %s" % cvmfs_root) + archs = config.get("arch") or discover_archs(cvmfs_root) + staged = [] + for arch in archs: + for pkg, cfg_pkg in _sweep_packages(cvmfs_root, arch, config, recipe_reader): + tests = resolve_tests(cfg_pkg, recipe_reader(pkg)) + if not tests: + continue + for verrev in package_versions(cvmfs_root, arch, pkg, + (cfg_pkg or {}).get("versions")): + pdir = package_dir(cvmfs_root, arch, pkg, verrev) + for exe, args in tests: + if bundle_exists(pdir, exe) and not update: + log("skip %s/%s %s (bundle exists)", arch, verrev, exe) + continue + exe_abs = os.path.join(pdir, exe) + tar_rel, spec = B.build_bundle(exe_abs, tracer(pdir, exe, args), + repo_root) + if not tar_rel: + log("empty %s/%s %s (no in-repo opens)", arch, verrev, exe) + continue + B.stage_bundle(staging_dir, tar_rel, spec) + staged.append(tar_rel) + log("staged %s", tar_rel) + return sorted(staged) + + +def _sweep_packages(cvmfs_root, arch, config, recipe_reader): + """Yield ``(pkg, cfg_pkg)`` for the sweep: the config's packages, or — when + none are listed — every deployed package that carries a recipe ``preload:``.""" + pkgs = config.get("packages") or {} + if pkgs: + for name, cfg_pkg in pkgs.items(): + yield name, cfg_pkg + return + from bits_helpers import cvmfs_inspect as I + for name in sorted(I.list_packages(cvmfs_root, arch)): + if recipe_tests(recipe_reader(name)): + yield name, None + + +def main(argv=None): + import argparse + ap = argparse.ArgumentParser(prog="bits preload", + description="Generate CVMFS filebundle prefetch " + "files for deployed packages.") + ap.add_argument("--cvmfs", metavar="ROOT", + help="deployed tree root, e.g. /cvmfs//lcg/bits " + "(overrides the config's cvmfs: key; required if the " + "config has none)") + ap.add_argument("--config", metavar="YAML", + help="preload.yaml: may fully specify cvmfs/arch/packages/tests " + "(recipe-independent), or just select scope/policy") + ap.add_argument("--config-dir", default=".", metavar="DIR", + help="recipe directory (for preload: sections). Default: .") + ap.add_argument("--arch", action="append", metavar="ARCH", + help="platform (repeatable); default: discover from the tree") + ap.add_argument("--update", action="store_true", + help="regenerate even if a .cvmfsbundle-* already exists") + ap.add_argument("--docker", action="store_true", help="trace inside a container") + ap.add_argument("--docker-image", metavar="IMAGE", help="image for --docker") + ap.add_argument("--output", metavar="TAR", default="preload-bundles.tar", + help="write the bundle tar here. Default: %(default)s") + a = ap.parse_args(argv) + + if a.config and not os.path.isfile(a.config): + ap.error("config file not found: %s" % a.config) + config = load_config(open(a.config).read()) if a.config else { + "cvmfs": None, "arch": None, "docker": False, "update": False, "packages": {}} + if a.arch: + config["arch"] = a.arch + cvmfs = a.cvmfs or config.get("cvmfs") + if not cvmfs: + ap.error("no CVMFS tree given: pass --cvmfs or set cvmfs: in the config") + update = a.update or config.get("update") + docker = a.docker or config.get("docker") + if docker and not a.docker_image: + ap.error("--docker requires --docker-image IMAGE") + + def recipe_reader(pkg): + from bits_helpers.utilities import parseRecipe, FileReader + path = os.path.join(a.config_dir, pkg + ".sh") + if not os.path.isfile(path): + return None + try: + _err, spec, _body = parseRecipe(FileReader(path)) + return spec + except Exception: + return None + + import tempfile + staging = tempfile.mkdtemp(prefix="preload-stage-") + try: + staged = sweep(cvmfs, config, recipe_reader, staging, + _default_tracer(docker, a.docker_image), update=update, + log=lambda f, *ar: print(">> " + (f % ar))) + if not staged: + print("no bundles generated"); return 0 + make_tar(staging, a.output) + print("wrote %d bundle(s) to %s" % (len(staged), a.output)) + print("publish with: bits cvmfs-publish --tar %s --repo " + "--prepub-url [--stratum0-url ]" % a.output) + finally: + __import__("shutil").rmtree(staging, ignore_errors=True) + return 0 diff --git a/tests/test_cvmfs_publish.py b/tests/test_cvmfs_publish.py index 0063c7b..e39fa9c 100644 --- a/tests/test_cvmfs_publish.py +++ b/tests/test_cvmfs_publish.py @@ -622,3 +622,40 @@ def test_ingest_workers_gt1_does_not_require_stage_flags(self): if __name__ == "__main__": unittest.main() + + +class TestPublishOverlayTars(unittest.TestCase): + """publish_overlay_tars stages each tar at the repo root (path=''), gives each + a distinct job_id_base, and preserves the caller's tar (a temp copy is + published). The actual stage/submit is proven on a build host separately.""" + + def test_overlay_publish(self): + import bits_helpers.cvmfs_publish as P + calls = [] + + def fake_publish_tar(ctx, path, tar, label, fp=None): + # mimic _publish_tar: it removes the (temp) tar it publishes + calls.append((ctx["job_id_base"], path, label, os.path.exists(tar))) + os.remove(tar) + return "job-" + label + + orig = P._publish_tar + P._publish_tar = fake_publish_tar + try: + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + t1 = os.path.join(d, "a.tar"); open(t1, "w").write("x") + t2 = os.path.join(d, "b.tar"); open(t2, "w").write("y") + jids = P.publish_overlay_tars( + {"job_id_base": "base", "tmp_dir": d}, [t1, t2]) + self.assertEqual(jids, ["job-a.tar", "job-b.tar"]) + self.assertEqual([c[1] for c in calls], ["", ""]) # root path + self.assertEqual([c[0] for c in calls], ["base-tar0", "base-tar1"]) + self.assertTrue(all(c[3] for c in calls)) # temp existed + self.assertTrue(os.path.exists(t1) and os.path.exists(t2)) # originals kept + finally: + P._publish_tar = orig + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hashing.py b/tests/test_hashing.py index dcf19a9..b9852f2 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -163,6 +163,21 @@ def test_adding_url_acknowledgment_source_redistributable_is_invariant(self): "redistributable: false\n") self.assertEqual(self._n(base), self._n(extra)) + def test_adding_preload_block_is_hash_invariant(self): + # The preload: test list (consumed post-publish by `bits preload`) is + # hash-excluded — its indented block is dropped and editing it must not + # change the build hash input. + withpl = self.HEADER.replace( + "requires:\n", + "preload:\n" + " - exe: bin/xrdcp\n" + " args: [--version]\n" + " - exe: bin/xrdfs\n" + "requires:\n") + self.assertNotIn("preload:", self._n(withpl)) + self.assertNotIn("xrdcp", self._n(withpl)) + self.assertEqual(self._n(self.HEADER), self._n(withpl)) + def test_multiline_block_value_dropped_entirely(self): base = self.HEADER.replace("description: A physics tool\n", "") block = self.HEADER.replace( diff --git a/tests/test_preload_bundle.py b/tests/test_preload_bundle.py new file mode 100644 index 0000000..ffa9942 --- /dev/null +++ b/tests/test_preload_bundle.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/preload_bundle — post-publish filebundle spec emitter.""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import preload_bundle as P + +REPO = "/cvmfs/sft.cern.ch" +BASE = REPO + "/lcg/releases/x86_64-el9/xrootd/5.9.1" # deployed package dir + + +class RepoRootTest(unittest.TestCase): + def test_repo_root_of(self): + self.assertEqual(P.repo_root_of("/cvmfs/sft.cern.ch/lcg/releases"), + "/cvmfs/sft.cern.ch") + self.assertEqual(P.repo_root_of("/cvmfs/alice.cern.ch"), "/cvmfs/alice.cern.ch") + self.assertIsNone(P.repo_root_of("/home/user/sw")) + self.assertIsNone(P.repo_root_of("")) + + def test_to_repo_absolute(self): + self.assertEqual(P.to_repo_absolute(BASE + "/lib/libXrdCl.so", REPO), + "/lcg/releases/x86_64-el9/xrootd/5.9.1/lib/libXrdCl.so") + self.assertIsNone(P.to_repo_absolute("/usr/lib64/libc.so.6", REPO)) + self.assertIsNone(P.to_repo_absolute(REPO, REPO)) # the mount itself + + def test_to_repo_absolute_normalizes_dot_and_slashes(self): + # '/./' and '//' from the BITS_ARCH_PREFIX="." env trick are collapsed. + self.assertEqual( + P.to_repo_absolute(REPO + "/lcg/./Packages//Boost/lib/l.so", REPO), + "/lcg/Packages/Boost/lib/l.so") + + +class BuildBundleTest(unittest.TestCase): + def test_build_bundle(self): + trigger = BASE + "/bin/xrdcp" + opened = [ + trigger, # excluded (the trigger) + BASE + "/lib/libXrdCl.so", # own package + REPO + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", # dep + "/usr/lib64/libc.so.6", # system -> dropped + "/proc/self/maps", # dropped + ] + tar_rel, spec = P.build_bundle(trigger, opened, REPO) + self.assertEqual(tar_rel, + "lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp") + self.assertEqual(spec["name"], "CVMFS_BUNDLE") + self.assertEqual(spec["version"], "1.0.0") + self.assertEqual(spec["encoding"], "UTF-8") + self.assertEqual(spec["dependencies"], [ + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", + "/lcg/releases/x86_64-el9/xrootd/5.9.1/lib/libXrdCl.so", + ]) + + def test_trigger_not_under_repo(self): + self.assertEqual(P.build_bundle("/home/x/bin/tool", ["/home/x/lib/a"], REPO), + (None, None)) + + def test_no_in_repo_opens_no_bundle(self): + trigger = BASE + "/bin/xrdcp" + self.assertEqual( + P.build_bundle(trigger, [trigger, "/usr/lib64/libc.so.6"], REPO), + (None, None)) + + +class RenderAndPathTest(unittest.TestCase): + def test_render_spec_exact_envelope(self): + self.assertEqual(P.render_spec(["/a", "/b"]), + {"name": "CVMFS_BUNDLE", "version": "1.0.0", + "encoding": "UTF-8", "dependencies": ["/a", "/b"]}) + + def test_bundle_path_for(self): + self.assertEqual(P.bundle_path_for("/x/bin/root"), "/x/bin/.cvmfsbundle-root") + self.assertEqual(P.bundle_path_for("root"), ".cvmfsbundle-root") + + +class StageBundleTest(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.d, True) + + def test_writes_json_into_staging_tree(self): + trigger = BASE + "/bin/xrdcp" + tar_rel, spec = P.build_bundle(trigger, [trigger, BASE + "/lib/l.so"], REPO) + dest = P.stage_bundle(self.d, tar_rel, spec) + self.assertTrue(dest.endswith( + "lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp")) + with open(dest) as fh: + self.assertEqual(json.load(fh)["name"], "CVMFS_BUNDLE") + + def test_unsafe_relpath_rejected(self): + for bad in ("/abs/x", "../escape/x", "a/\x00/b"): + with self.assertRaises(ValueError): + P.stage_bundle(self.d, bad, P.render_spec(["/a"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py new file mode 100644 index 0000000..ebde1ad --- /dev/null +++ b/tests/test_preload_cmd.py @@ -0,0 +1,263 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/preload_cmd — the pure steps of `bits preload`.""" + +import os +import sys +import tarfile +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import preload_cmd as C + +REPO = "/cvmfs/sft.cern.ch" +PKGDIR = REPO + "/lcg/releases/x86_64-el9/xrootd/5.9.1" + + +class RecipeTestsTest(unittest.TestCase): + def test_mapping_and_string_forms(self): + spec = {"package": "xrootd", "preload": [ + {"exe": "bin/xrdcp", "args": ["--version"]}, + {"exe": "bin/xrdfs", "args": "--help -v"}, # string args -> shlex + "bin/xrdmapc plain", # bare string entry + {"args": ["--x"]}, # no exe -> skipped + ]} + self.assertEqual(C.recipe_tests(spec), [ + ("bin/xrdcp", ["--version"]), + ("bin/xrdfs", ["--help", "-v"]), + ("bin/xrdmapc", ["plain"]), + ]) + + def test_absent(self): + self.assertEqual(C.recipe_tests({"package": "x"}), []) + self.assertEqual(C.recipe_tests(None), []) + + +class LoadConfigTest(unittest.TestCase): + def test_bare_list_and_mapping_override(self): + cfg = C.load_config( + "cvmfs: /cvmfs/sft-nightlies-test.cern.ch/lcg/bits\n" + "arch: x86_64-el9-gcc14-opt\n" + "docker: true\n" + "packages:\n" + " - xrootd\n" # bare -> defer to recipe + " - ROOT:\n" + " versions: ['6.38.*']\n" + " tests:\n" + " - { exe: bin/root, args: [-b, -q] }\n") + self.assertEqual(cfg["cvmfs"], "/cvmfs/sft-nightlies-test.cern.ch/lcg/bits") + self.assertEqual(cfg["arch"], ["x86_64-el9-gcc14-opt"]) # scalar -> list + self.assertTrue(cfg["docker"]) + self.assertFalse(cfg["update"]) + self.assertEqual(cfg["packages"]["xrootd"], {"tests": [], "versions": []}) + self.assertEqual(cfg["packages"]["ROOT"]["tests"], [("bin/root", ["-b", "-q"])]) + self.assertEqual(cfg["packages"]["ROOT"]["versions"], ["6.38.*"]) + + def test_arch_omitted_is_none_for_discovery(self): + self.assertIsNone(C.load_config("docker: false\n")["arch"]) + + +class ResolveAndSkipTest(unittest.TestCase): + def test_config_tests_override_else_recipe(self): + recipe = {"preload": [{"exe": "bin/xrdcp", "args": ["--version"]}]} + self.assertEqual(C.resolve_tests({"tests": [("bin/x", [])]}, recipe), + [("bin/x", [])]) # config wins + self.assertEqual(C.resolve_tests({"tests": []}, recipe), + [("bin/xrdcp", ["--version"])]) # falls back to recipe + self.assertEqual(C.resolve_tests(None, recipe), + [("bin/xrdcp", ["--version"])]) + + def test_bundle_exists(self): + d = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, d, True) + self.assertFalse(C.bundle_exists(d, "bin/xrdcp")) + os.makedirs(os.path.join(d, "bin")) + open(os.path.join(d, "bin", ".cvmfsbundle-xrdcp"), "w").close() + self.assertTrue(C.bundle_exists(d, "bin/xrdcp")) + + +class LocatePackageTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for rel in ("lcg/releases/x86_64-el9/xrootd/5.9.1", + "lcg/releases/x86_64-el9/xrootd/5.9.0-2", + "lcg/releases/x86_64-el9/Boost/1.90.0-1"): + os.makedirs(os.path.join(self.root, rel)) + + def test_newest_version(self): + got = C.locate_package(self.root, "xrootd") + self.assertTrue(got.endswith("xrootd/5.9.1")) + + def test_version_prefix_match(self): + got = C.locate_package(self.root, "xrootd", "5.9.0") + self.assertTrue(got.endswith("xrootd/5.9.0-2")) + + def test_absent(self): + self.assertIsNone(C.locate_package(self.root, "ROOT")) + + +class DiscoveryTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for rel in ("x86_64-el9-gcc14-opt/Packages/xrootd/5.9.1-1", + "x86_64-el9-gcc14-opt/Packages/xrootd/5.9.0-2", + "x86_64-el9-gcc14-opt/Packages/ROOT/6.38.00-1", + "aarch64-el9-gcc14-opt/Packages/xrootd/5.9.1-1"): + os.makedirs(os.path.join(self.root, rel)) + + def test_discover_archs(self): + self.assertEqual(sorted(C.discover_archs(self.root)), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + + def test_package_versions_all_and_filtered(self): + a = "x86_64-el9-gcc14-opt" + self.assertEqual(sorted(C.package_versions(self.root, a, "xrootd")), + ["5.9.0-2", "5.9.1-1"]) + # bare version glob matches its - dir + self.assertEqual(C.package_versions(self.root, a, "xrootd", ["5.9.1"]), + ["5.9.1-1"]) + self.assertEqual(C.package_versions(self.root, a, "ROOT", ["6.38.*"]), + ["6.38.00-1"]) + self.assertEqual(C.package_versions(self.root, a, "xrootd", ["9.9.9"]), []) + + def test_package_dir(self): + self.assertEqual( + C.package_dir(self.root, "x86_64-el9-gcc14-opt", "xrootd", "5.9.1-1"), + os.path.join(self.root, "x86_64-el9-gcc14-opt/Packages/xrootd/5.9.1-1")) + + +class TraceScriptTest(unittest.TestCase): + def test_contains_env_setup_and_strace(self): + s = C.build_trace_script( + "/cvmfs/r/x86_64-el9/Packages/xrootd/5.9.1-1", "bin/xrdcp", + ["--version"], "/tmp/log") + self.assertIn('export WORK_DIR=/cvmfs/r/x86_64-el9/Packages', s) + self.assertIn('BITS_ARCH_PREFIX="."', s) + self.assertIn("etc/profile.d/init.sh", s) + self.assertIn("strace -f -e trace=open,openat", s) + self.assertIn("bin/xrdcp --version", s) + + +class SweepTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for v in ("5.9.1-1", "5.9.0-2"): + os.makedirs(os.path.join(self.root, "x86_64-el9/Packages/xrootd", v)) + self.cfg = {"arch": ["x86_64-el9"], + "packages": {"xrootd": {"tests": [("bin/xrdcp", [])], + "versions": []}}} + + def _tracer(self, pdir, exe, args): + # one in-repo lib + one system lib (dropped) + return [os.path.join(pdir, "lib/libXrdCl.so.3"), "/usr/lib64/libc.so.6"] + + def test_stages_bundle_per_version(self): + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, repo_root=self.root) + self.assertEqual(staged, [ + "x86_64-el9/Packages/xrootd/5.9.0-2/bin/.cvmfsbundle-xrdcp", + "x86_64-el9/Packages/xrootd/5.9.1-1/bin/.cvmfsbundle-xrdcp", + ]) + + def test_skips_existing_unless_update(self): + # pre-create the 5.9.1-1 bundle in the tree -> skipped by default + bdir = os.path.join(self.root, "x86_64-el9/Packages/xrootd/5.9.1-1/bin") + os.makedirs(bdir) + open(os.path.join(bdir, ".cvmfsbundle-xrdcp"), "w").close() + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, repo_root=self.root) + self.assertEqual(staged, + ["x86_64-el9/Packages/xrootd/5.9.0-2/bin/.cvmfsbundle-xrdcp"]) + # with --update both are (re)generated + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, update=True, repo_root=self.root) + self.assertEqual(len(staged), 2) + + def test_recipe_fallback_when_config_has_no_tests(self): + cfg = {"arch": ["x86_64-el9"], "packages": {"xrootd": None}} + reader = lambda p: {"preload": [{"exe": "bin/xrdfs", "args": ["--help"]}]} + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, cfg, reader, stage, self._tracer, + repo_root=self.root) + self.assertTrue(all(s.endswith("/bin/.cvmfsbundle-xrdfs") for s in staged)) + self.assertEqual(len(staged), 2) + + +class MainGuardTest(unittest.TestCase): + def test_docker_requires_image(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main(["--cvmfs", "/cvmfs/r/lcg", "--docker"]) + + def test_missing_config_errors(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main(["--cvmfs", "/cvmfs/r/lcg", "--config", "/no/such.yaml"]) + + def test_no_cvmfs_anywhere_errors(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main([]) # neither --cvmfs nor config cvmfs: + + +class ParseStraceTest(unittest.TestCase): + def test_success_only_abs_dedup(self): + text = ( + 'open("%s/lib/libXrdCl.so", O_RDONLY) = 3\n' + 'openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY) = 4\n' + 'openat(AT_FDCWD, "%s/lib/libXrdCl.so", O_RDONLY) = 5\n' # dup + 'openat(AT_FDCWD, "relative/path", O_RDONLY) = 6\n' # relative -> dropped + # failed loader probes must be dropped, not listed: + 'openat(AT_FDCWD, "%s/lib/tls/x86_64/libXrdCl.so", O_RDONLY) = -1 ENOENT (No such file or directory)\n' + 'openat(AT_FDCWD, "%s/lib/glibc-hwcaps/x86-64-v3/libstdc++.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)\n' + ) % (PKGDIR, PKGDIR, PKGDIR, PKGDIR) + self.assertEqual(C.parse_strace_opens(text), + [PKGDIR + "/lib/libXrdCl.so", "/usr/lib64/libc.so.6"]) + + +class AssembleAndTarTest(unittest.TestCase): + def setUp(self): + self.stage = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.stage, True) + + def test_assemble_then_tar(self): + traces = [(PKGDIR + "/bin/xrdcp", + [PKGDIR + "/bin/xrdcp", # trigger, excluded + PKGDIR + "/lib/libXrdCl.so", + REPO + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", + "/usr/lib64/libc.so.6"])] # system, dropped + staged = C.assemble_bundles(traces, REPO, self.stage) + self.assertEqual( + staged, + ["lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp"]) + out = os.path.join(self.stage, "..", "b.tar") + C.make_tar(self.stage, out) + with tarfile.open(out) as tf: + names = tf.getnames() + self.assertEqual( + names, + ["lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp"]) + os.remove(out) + + def test_assemble_skips_when_no_repo_opens(self): + traces = [(PKGDIR + "/bin/xrdcp", + [PKGDIR + "/bin/xrdcp", "/usr/lib64/libc.so.6"])] + self.assertEqual(C.assemble_bundles(traces, REPO, self.stage), []) + + +if __name__ == "__main__": + unittest.main()