From 200ca02ab8a7df3ce6258fcc25b844220421348c Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 11:33:46 +0200 Subject: [PATCH 01/10] preload: generate CVMFS filebundle specs from package sidecars (core) New bits_helpers/preload_bundle.py turns a package's .bits-preload/*.paths sidecars into spec-conformant .cvmfsbundle- JSON: resolves each opened file to its owning package's CVMFS path, lists repo-root-absolute deps (trigger excluded), removes the sidecar dir. Pure + unit-tested; not yet wired into publish. --- bits_helpers/preload_bundle.py | 210 +++++++++++++++++++++++++++++++++ tests/test_preload_bundle.py | 156 ++++++++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 bits_helpers/preload_bundle.py create mode 100644 tests/test_preload_bundle.py diff --git a/bits_helpers/preload_bundle.py b/bits_helpers/preload_bundle.py new file mode 100644 index 0000000..c52fe90 --- /dev/null +++ b/bits_helpers/preload_bundle.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Generate CVMFS filebundle specs from a package's preload sidecars. + +Background +---------- +`PreloadRecipe` traces an application's startup with strace and drops a sidecar +per traced executable at:: + + /.bits-preload/.paths + +The sidecar is dumb, deliberately: every line is a path the launch opened, +written relative to the architecture install base +(``//`` — or the grouped ``///``), +so it is portable across the build-time and publish-time path prefixes. Line 1 +is the trigger executable itself (same coordinate system), so it can both locate +the bundle and be excluded from its own dependency list. Dependency files appear +as other packages' entries, which is the whole point (prefetch the closure, not +just this package). + +This module turns those sidecars into the CVMFS filebundle spec files: + + /.cvmfsbundle- + +a versioned JSON document listing the dependencies as repository-root-absolute +paths (see https://cvmfs.readthedocs.io/en/stable/cpt-file-bundles/):: + + { "name": "CVMFS_BUNDLE", "version": "1.0.0", "encoding": "UTF-8", + "dependencies": [ "/el9/Packages/Boost/1.90.0/lib/libboost.so", ... ] } + +Because a dependency file belongs to a *different* package that publishes to its +own CVMFS path, each entry is resolved through its owning package (found by +walking up to the directory that holds a ``.meta.json``) and that package's +resolved repo path — a release-aware step, but self-contained: every dependency's +``.meta.json`` is present in the work tree at publish time. +""" + +import json +import os + +# CVMFS filebundle spec envelope (cpt-file-bundles). +SPEC_NAME = "CVMFS_BUNDLE" +SPEC_VERSION = "1.0.0" +SPEC_ENCODING = "UTF-8" + +SIDECAR_DIR = ".bits-preload" + + +def parse_sidecar(path): + """Return ``(trigger_rel, [file_rel, ...])`` from a sidecar file. + + All lines are arch-base-relative; line 1 is the trigger, the rest are the + opened files. ``#`` comments and blank lines are ignored. Order is preserved + and duplicates dropped. Returns ``(None, [])`` on a read error or an empty + sidecar — a bad sidecar must never abort a publish. + """ + trigger, files, seen = None, [], set() + try: + with open(path) as fh: + for raw in fh: + line = raw.strip() + if not line or line.startswith("#"): + continue + if trigger is None: + trigger = line + continue + if line not in seen: + seen.add(line) + files.append(line) + except OSError: + return None, [] + return trigger, files + + +def _owning_pkg_root(rel, meta_exists): + """Longest leading prefix of *rel* that is a package root (holds .meta.json). + + Handles both the 2-level ``/`` and grouped 3-level + ``//`` layouts by asking *meta_exists(prefix)* for + growing prefixes and taking the longest that matches. Returns + ``(pkg_root, rest)`` or ``(None, None)`` when no ancestor is a package. + """ + parts = [p for p in rel.split("/") if p not in ("", ".")] + best = None + for i in range(1, len(parts)): # need at least one trailing component + prefix = "/".join(parts[:i]) + if meta_exists(prefix): + best = i # keep the LONGEST matching prefix + if best is None: + return None, None + return "/".join(parts[:best]), "/".join(parts[best:]) + + +def _is_safe_rel(rel): + """True if *rel* is a plain, in-tree relative path (no abs, no ``..``, no NUL).""" + if not rel or rel.startswith("/") or "\x00" in rel: + return False + return ".." not in [p for p in rel.split("/")] + + +def _warn(fmt, *args): + """Best-effort warning; never raise (this runs on the publish path).""" + try: + from bits_helpers.log import warning + warning(fmt, *args) + except Exception: # pragma: no cover + pass + + +def build_dependencies(files_rel, resolve_repo, meta_exists, skip=None): + """Map arch-base-relative files to repo-root-absolute bundle entries. + + *resolve_repo(pkg_root)* returns the owning package's repo-relative CVMFS + path (e.g. ``"el9/Packages/Boost/1.90.0"``); *meta_exists(prefix)* reports + whether *prefix* is a package root. Entries that are unsafe, whose owner + cannot be found or resolved, or that equal *skip* (the trigger's own file) + are dropped. Result is sorted and de-duplicated. + """ + skip = skip or set() + out = set() + for rel in files_rel: + if rel in skip or not _is_safe_rel(rel): + continue + pkg_root, rest = _owning_pkg_root(rel, meta_exists) + if not pkg_root or not rest: + continue + repo_path = resolve_repo(pkg_root) + if not repo_path: + continue + out.add("/" + repo_path.strip("/") + "/" + rest) + return sorted(out) + + +def render_spec(dependencies): + """The versioned filebundle JSON document for *dependencies* (verbatim keys).""" + return { + "name": SPEC_NAME, + "version": SPEC_VERSION, + "encoding": SPEC_ENCODING, + "dependencies": list(dependencies), + } + + +def bundle_path_for(trigger_rel): + """``/.cvmfsbundle-`` for a trigger's package-relative path.""" + d, base = os.path.split(trigger_rel) + name = ".cvmfsbundle-" + base + return os.path.join(d, name) if d else name + + +def _generate_one(pkgroot, sidecar_path, resolve_repo, meta_exists): + """Write one bundle from one sidecar; return its pkgroot-relative path or None. + + None when the sidecar is unreadable/empty, its trigger owner cannot be + resolved, or it yields no dependencies (no empty bundle is written). + """ + trigger_rel, files_rel = parse_sidecar(sidecar_path) + if not trigger_rel or not _is_safe_rel(trigger_rel): + return None + # The trigger is arch-base-relative too; its in-package location is the part + # after its own / root, which is where the bundle goes. + _pkg_root, trigger_rest = _owning_pkg_root(trigger_rel, meta_exists) + if not trigger_rest: + return None + deps = build_dependencies(files_rel, resolve_repo, meta_exists, + skip={trigger_rel}) + if not deps: + return None + rel = bundle_path_for(trigger_rest) + dest = os.path.join(pkgroot, rel) + os.makedirs(os.path.dirname(dest) or pkgroot, exist_ok=True) + with open(dest, "w", encoding="utf-8") as fh: + json.dump(render_spec(deps), fh, indent=2) + fh.write("\n") + return rel + + +def generate_for_package(pkgroot, resolve_repo, meta_exists): + """Turn every sidecar under ``/.bits-preload/`` into a bundle file. + + Writes ``//.cvmfsbundle-`` next to each trigger, then + removes the ``.bits-preload`` directory so it is not published. *resolve_repo* + and *meta_exists* provide the owning-package resolution. Returns the list of + bundle paths written (pkgroot-relative). + + Fail-safe: one bad sidecar must never abort a publish, and the sidecar dir is + always removed — so a per-sidecar failure is logged and skipped, and the + cleanup runs in a ``finally`` even if something raises. + """ + import shutil + sdir = os.path.join(pkgroot, SIDECAR_DIR) + if not os.path.isdir(sdir): + return [] + written = [] + try: + for name in sorted(os.listdir(sdir)): + if not name.endswith(".paths"): + continue + try: + rel = _generate_one(pkgroot, os.path.join(sdir, name), + resolve_repo, meta_exists) + except Exception as exc: # never let one sidecar abort publish + _warn("preload bundle: skipping %s: %s", name, exc) + continue + if rel: + written.append(rel) + finally: + shutil.rmtree(sdir, ignore_errors=True) # always drop sidecars + return written diff --git a/tests/test_preload_bundle.py b/tests/test_preload_bundle.py new file mode 100644 index 0000000..ef55b9a --- /dev/null +++ b/tests/test_preload_bundle.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/preload_bundle — CVMFS filebundle spec generation.""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import preload_bundle as P + + +# A synthetic release: package root (arch-base-relative) -> repo-relative CVMFS +# path. 2-level (/) and grouped 3-level (//). +RELEASE = { + "xrootd/5.9.1-1": "el9/Packages/xrootd/5.9.1", + "Boost/1.90.0-1": "el9/Packages/Boost/1.90.0", + "gcc/GCC/14.2.0-1": "el9/Packages/GCC/14.2.0", # grouped: // +} +_meta_exists = lambda prefix: prefix in RELEASE +_resolve = RELEASE.get + + +class ParseSidecarTest(unittest.TestCase): + def _write(self, text): + fd, p = tempfile.mkstemp() + os.write(fd, text.encode()); os.close(fd) + self.addCleanup(os.remove, p) + return p + + def test_trigger_then_files_dedup_comments(self): + p = self._write( + "# a preload sidecar\n" + "xrootd/5.9.1-1/bin/xrdcp\n" + "Boost/1.90.0-1/lib/libboost.so\n" + "\n" + "Boost/1.90.0-1/lib/libboost.so\n" # dup dropped + "xrootd/5.9.1-1/lib/libXrdCl.so\n") + trig, files = P.parse_sidecar(p) + self.assertEqual(trig, "xrootd/5.9.1-1/bin/xrdcp") + self.assertEqual(files, ["Boost/1.90.0-1/lib/libboost.so", + "xrootd/5.9.1-1/lib/libXrdCl.so"]) + + def test_missing_file_is_safe(self): + self.assertEqual(P.parse_sidecar("/no/such/sidecar.paths"), (None, [])) + + +class OwningPkgRootTest(unittest.TestCase): + def test_two_level(self): + self.assertEqual( + P._owning_pkg_root("Boost/1.90.0-1/lib/libboost.so", _meta_exists), + ("Boost/1.90.0-1", "lib/libboost.so")) + + def test_grouped_three_level_longest_wins(self): + self.assertEqual( + P._owning_pkg_root("gcc/GCC/14.2.0-1/lib64/libstdc++.so", _meta_exists), + ("gcc/GCC/14.2.0-1", "lib64/libstdc++.so")) + + def test_unknown_owner(self): + self.assertEqual(P._owning_pkg_root("Nope/1.0/lib/x.so", _meta_exists), + (None, None)) + + +class BuildDependenciesTest(unittest.TestCase): + def test_maps_sorts_dedups_and_skips_trigger(self): + files = [ + "xrootd/5.9.1-1/bin/xrdcp", # trigger -> skipped + "Boost/1.90.0-1/lib/libboost.so", + "xrootd/5.9.1-1/lib/libXrdCl.so", + "gcc/GCC/14.2.0-1/lib64/libstdc++.so", + ] + deps = P.build_dependencies(files, _resolve, _meta_exists, + skip={"xrootd/5.9.1-1/bin/xrdcp"}) + self.assertEqual(deps, [ + "/el9/Packages/Boost/1.90.0/lib/libboost.so", + "/el9/Packages/GCC/14.2.0/lib64/libstdc++.so", + "/el9/Packages/xrootd/5.9.1/lib/libXrdCl.so", + ]) + + def test_unsafe_and_unresolvable_dropped(self): + files = ["/abs/evil", "../escape/x", "Unknown/1.0/lib/x.so", + "Boost/1.90.0-1/lib/ok.so"] + deps = P.build_dependencies(files, _resolve, _meta_exists) + self.assertEqual(deps, ["/el9/Packages/Boost/1.90.0/lib/ok.so"]) + + +class RenderAndPathTest(unittest.TestCase): + def test_render_spec_exact_envelope(self): + spec = P.render_spec(["/a", "/b"]) + self.assertEqual(spec, {"name": "CVMFS_BUNDLE", "version": "1.0.0", + "encoding": "UTF-8", "dependencies": ["/a", "/b"]}) + + def test_bundle_path_for(self): + self.assertEqual(P.bundle_path_for("bin/xrdcp"), "bin/.cvmfsbundle-xrdcp") + self.assertEqual(P.bundle_path_for("root"), ".cvmfsbundle-root") + + +class GenerateForPackageTest(unittest.TestCase): + def setUp(self): + self.pkgroot = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.pkgroot, True) + os.makedirs(os.path.join(self.pkgroot, P.SIDECAR_DIR)) + + def _sidecar(self, name, lines): + with open(os.path.join(self.pkgroot, P.SIDECAR_DIR, name), "w") as fh: + fh.write("\n".join(lines) + "\n") + + def test_writes_bundle_next_to_trigger_and_removes_sidecar(self): + self._sidecar("xrdcp.paths", [ + "xrootd/5.9.1-1/bin/xrdcp", # trigger + "Boost/1.90.0-1/lib/libboost.so", + "xrootd/5.9.1-1/lib/libXrdCl.so", + ]) + written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) + self.assertEqual(written, ["bin/.cvmfsbundle-xrdcp"]) + dest = os.path.join(self.pkgroot, "bin", ".cvmfsbundle-xrdcp") + with open(dest) as fh: + doc = json.load(fh) + self.assertEqual(doc["name"], "CVMFS_BUNDLE") + self.assertEqual(doc["dependencies"], [ + "/el9/Packages/Boost/1.90.0/lib/libboost.so", + "/el9/Packages/xrootd/5.9.1/lib/libXrdCl.so", # trigger's own file kept; only the exe excluded + ]) + # sidecar dir removed so it never reaches CVMFS + self.assertFalse(os.path.exists(os.path.join(self.pkgroot, P.SIDECAR_DIR))) + + def test_empty_deps_sidecar_writes_no_bundle(self): + self._sidecar("xrdcp.paths", ["xrootd/5.9.1-1/bin/xrdcp"]) # only the trigger + written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) + self.assertEqual(written, []) + self.assertFalse(os.path.exists(os.path.join(self.pkgroot, "bin"))) + + def test_one_bad_sidecar_does_not_abort_and_dir_always_removed(self): + # Inject a write-time failure: a plain file where one bundle's directory + # needs to be created, so os.makedirs raises for that sidecar only. + self._sidecar("good.paths", ["xrootd/5.9.1-1/bin/xrdcp", + "Boost/1.90.0-1/lib/libboost.so"]) + open(os.path.join(self.pkgroot, "clash"), "w").close() # not a dir + self._sidecar("clash.paths", ["xrootd/5.9.1-1/clash/x", + "Boost/1.90.0-1/lib/libboost.so"]) + written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) + # the good sidecar still produced its bundle; clash was skipped, not fatal + self.assertEqual(written, ["bin/.cvmfsbundle-xrdcp"]) + # cleanup ran despite the failure + self.assertFalse(os.path.exists(os.path.join(self.pkgroot, P.SIDECAR_DIR))) + + def test_nul_in_path_is_rejected(self): + self.assertFalse(P._is_safe_rel("Boost/1.90.0-1/lib/\x00evil")) + + +if __name__ == "__main__": + unittest.main() From dc48a827b610c052cf26ab15a10cd229a4c02060 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 12:34:20 +0200 Subject: [PATCH 02/10] preload: post-publish filebundle core (supersedes sidecar model) Regenerate .cvmfsbundle specs from a trigger straced on an already-deployed CVMFS tree: opens are already /cvmfs-absolute, so repo-root is a mount strip (no relocation, no cross-package map). Pure + unit-tested; drives the coming 'bits preload' tool. --- bits_helpers/preload_bundle.py | 234 ++++++++++----------------------- tests/test_preload_bundle.py | 191 ++++++++++----------------- 2 files changed, 138 insertions(+), 287 deletions(-) diff --git a/bits_helpers/preload_bundle.py b/bits_helpers/preload_bundle.py index c52fe90..e6f5e90 100644 --- a/bits_helpers/preload_bundle.py +++ b/bits_helpers/preload_bundle.py @@ -1,135 +1,61 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -"""Generate CVMFS filebundle specs from a package's preload sidecars. +"""CVMFS filebundle spec emitter for the post-publish `bits preload` tool. -Background ----------- -`PreloadRecipe` traces an application's startup with strace and drops a sidecar -per traced executable at:: +The tool straces a trigger binary that already lives on a deployed CVMFS tree, +so every opened path is already an absolute ``/cvmfs/.cern.ch/…`` path. +Turning that into a filebundle spec (per +https://cvmfs.readthedocs.io/en/stable/cpt-file-bundles/) is therefore just: - /.bits-preload/.paths + * keep the opens under the repo mount (drop system files, /proc, …); + * strip the mount to get repository-root-absolute paths (``/lcg/releases/…``); + * emit ``/.cvmfsbundle-`` next to the trigger, listing those + paths as ``dependencies`` (the trigger itself excluded). -The sidecar is dumb, deliberately: every line is a path the launch opened, -written relative to the architecture install base -(``//`` — or the grouped ``///``), -so it is portable across the build-time and publish-time path prefixes. Line 1 -is the trigger executable itself (same coordinate system), so it can both locate -the bundle and be excluded from its own dependency list. Dependency files appear -as other packages' entries, which is the whole point (prefetch the closure, not -just this package). - -This module turns those sidecars into the CVMFS filebundle spec files: - - /.cvmfsbundle- - -a versioned JSON document listing the dependencies as repository-root-absolute -paths (see https://cvmfs.readthedocs.io/en/stable/cpt-file-bundles/):: - - { "name": "CVMFS_BUNDLE", "version": "1.0.0", "encoding": "UTF-8", - "dependencies": [ "/el9/Packages/Boost/1.90.0/lib/libboost.so", ... ] } - -Because a dependency file belongs to a *different* package that publishes to its -own CVMFS path, each entry is resolved through its owning package (found by -walking up to the directory that holds a ``.meta.json``) and that package's -resolved repo path — a release-aware step, but self-contained: every dependency's -``.meta.json`` is present in the work tree at publish time. +No relocation and no cross-package resolution: the deployed paths are already +final. This module is the pure, unit-testable core; the tool driver +(`preload_cmd`) handles recipe parsing, env setup, strace, tar and publish. """ import json import os -# CVMFS filebundle spec envelope (cpt-file-bundles). + SPEC_NAME = "CVMFS_BUNDLE" SPEC_VERSION = "1.0.0" SPEC_ENCODING = "UTF-8" -SIDECAR_DIR = ".bits-preload" +def repo_root_of(cvmfs_path): + """The repo mount root of a ``/cvmfs//…`` path, i.e. ``/cvmfs/``. -def parse_sidecar(path): - """Return ``(trigger_rel, [file_rel, ...])`` from a sidecar file. - - All lines are arch-base-relative; line 1 is the trigger, the rest are the - opened files. ``#`` comments and blank lines are ignored. Order is preserved - and duplicates dropped. Returns ``(None, [])`` on a read error or an empty - sidecar — a bad sidecar must never abort a publish. + E.g. ``/cvmfs/sft.cern.ch/lcg/releases`` -> ``/cvmfs/sft.cern.ch``. Returns + None when *cvmfs_path* is not under ``/cvmfs//``. """ - trigger, files, seen = None, [], set() - try: - with open(path) as fh: - for raw in fh: - line = raw.strip() - if not line or line.startswith("#"): - continue - if trigger is None: - trigger = line - continue - if line not in seen: - seen.add(line) - files.append(line) - except OSError: - return None, [] - return trigger, files - - -def _owning_pkg_root(rel, meta_exists): - """Longest leading prefix of *rel* that is a package root (holds .meta.json). - - Handles both the 2-level ``/`` and grouped 3-level - ``//`` layouts by asking *meta_exists(prefix)* for - growing prefixes and taking the longest that matches. Returns - ``(pkg_root, rest)`` or ``(None, None)`` when no ancestor is a package. + parts = [p for p in (cvmfs_path or "").split("/") if p] + if len(parts) >= 2 and parts[0] == "cvmfs": + return "/cvmfs/" + parts[1] + return None + + +def to_repo_absolute(abs_path, repo_root): + """``/cvmfs//a/b`` -> ``/a/b``; None if *abs_path* is not under the repo. + + The result is repository-root-absolute (leading slash), exactly the form the + filebundle spec's ``dependencies`` want. """ - parts = [p for p in rel.split("/") if p not in ("", ".")] - best = None - for i in range(1, len(parts)): # need at least one trailing component - prefix = "/".join(parts[:i]) - if meta_exists(prefix): - best = i # keep the LONGEST matching prefix - if best is None: - return None, None - return "/".join(parts[:best]), "/".join(parts[best:]) + root = (repo_root or "").rstrip("/") + if not root or abs_path == root or not abs_path.startswith(root + "/"): + return None + return abs_path[len(root):] # keeps the leading '/' def _is_safe_rel(rel): """True if *rel* is a plain, in-tree relative path (no abs, no ``..``, no NUL).""" if not rel or rel.startswith("/") or "\x00" in rel: return False - return ".." not in [p for p in rel.split("/")] - - -def _warn(fmt, *args): - """Best-effort warning; never raise (this runs on the publish path).""" - try: - from bits_helpers.log import warning - warning(fmt, *args) - except Exception: # pragma: no cover - pass - - -def build_dependencies(files_rel, resolve_repo, meta_exists, skip=None): - """Map arch-base-relative files to repo-root-absolute bundle entries. - - *resolve_repo(pkg_root)* returns the owning package's repo-relative CVMFS - path (e.g. ``"el9/Packages/Boost/1.90.0"``); *meta_exists(prefix)* reports - whether *prefix* is a package root. Entries that are unsafe, whose owner - cannot be found or resolved, or that equal *skip* (the trigger's own file) - are dropped. Result is sorted and de-duplicated. - """ - skip = skip or set() - out = set() - for rel in files_rel: - if rel in skip or not _is_safe_rel(rel): - continue - pkg_root, rest = _owning_pkg_root(rel, meta_exists) - if not pkg_root or not rest: - continue - repo_path = resolve_repo(pkg_root) - if not repo_path: - continue - out.add("/" + repo_path.strip("/") + "/" + rest) - return sorted(out) + return ".." not in rel.split("/") def render_spec(dependencies): @@ -142,69 +68,51 @@ def render_spec(dependencies): } -def bundle_path_for(trigger_rel): - """``/.cvmfsbundle-`` for a trigger's package-relative path.""" - d, base = os.path.split(trigger_rel) +def bundle_path_for(path): + """``/.cvmfsbundle-`` for a trigger path (any form: abs or rel).""" + d, base = os.path.split(path) name = ".cvmfsbundle-" + base - return os.path.join(d, name) if d else name + return (d + "/" + name) if d else name -def _generate_one(pkgroot, sidecar_path, resolve_repo, meta_exists): - """Write one bundle from one sidecar; return its pkgroot-relative path or None. +def build_bundle(trigger_abs, opened_abs, repo_root): + """Build one bundle from a trigger and the files its launch opened. - None when the sidecar is unreadable/empty, its trigger owner cannot be - resolved, or it yields no dependencies (no empty bundle is written). + Returns ``(tar_relpath, spec_dict)`` — *tar_relpath* is the bundle's location + relative to the repo root (no leading slash), for placement in the staging + tar; *spec_dict* is the filebundle JSON. Returns ``(None, None)`` when the + trigger is not under the repo or nothing under the repo was opened (no empty + bundle). Opens outside the repo (system libs, /proc) and the trigger itself + are excluded; the result is sorted and de-duplicated. """ - trigger_rel, files_rel = parse_sidecar(sidecar_path) - if not trigger_rel or not _is_safe_rel(trigger_rel): - return None - # The trigger is arch-base-relative too; its in-package location is the part - # after its own / root, which is where the bundle goes. - _pkg_root, trigger_rest = _owning_pkg_root(trigger_rel, meta_exists) - if not trigger_rest: - return None - deps = build_dependencies(files_rel, resolve_repo, meta_exists, - skip={trigger_rel}) + trig_rel = to_repo_absolute(trigger_abs, repo_root) + if not trig_rel: + return None, None + deps = set() + for p in opened_abs: + if p == trigger_abs: + continue + r = to_repo_absolute(p, repo_root) + if r: + deps.add(r) if not deps: - return None - rel = bundle_path_for(trigger_rest) - dest = os.path.join(pkgroot, rel) - os.makedirs(os.path.dirname(dest) or pkgroot, exist_ok=True) - with open(dest, "w", encoding="utf-8") as fh: - json.dump(render_spec(deps), fh, indent=2) - fh.write("\n") - return rel - + return None, None + bundle_abs = bundle_path_for(trig_rel) # '/…/bin/.cvmfsbundle-root' + return bundle_abs.lstrip("/"), render_spec(sorted(deps)) -def generate_for_package(pkgroot, resolve_repo, meta_exists): - """Turn every sidecar under ``/.bits-preload/`` into a bundle file. - Writes ``//.cvmfsbundle-`` next to each trigger, then - removes the ``.bits-preload`` directory so it is not published. *resolve_repo* - and *meta_exists* provide the owning-package resolution. Returns the list of - bundle paths written (pkgroot-relative). +def stage_bundle(staging_dir, tar_relpath, spec): + """Write *spec* as JSON to ``/`` (dirs created). - Fail-safe: one bad sidecar must never abort a publish, and the sidecar dir is - always removed — so a per-sidecar failure is logged and skipped, and the - cleanup runs in a ``finally`` even if something raises. + *tar_relpath* must be a safe in-tree relative path. Returns the file path + written. The staging tree mirrors the repo layout so a single tar of it + drops each bundle next to its trigger on publish. """ - import shutil - sdir = os.path.join(pkgroot, SIDECAR_DIR) - if not os.path.isdir(sdir): - return [] - written = [] - try: - for name in sorted(os.listdir(sdir)): - if not name.endswith(".paths"): - continue - try: - rel = _generate_one(pkgroot, os.path.join(sdir, name), - resolve_repo, meta_exists) - except Exception as exc: # never let one sidecar abort publish - _warn("preload bundle: skipping %s: %s", name, exc) - continue - if rel: - written.append(rel) - finally: - shutil.rmtree(sdir, ignore_errors=True) # always drop sidecars - return written + if not _is_safe_rel(tar_relpath): + raise ValueError("unsafe bundle path: %r" % (tar_relpath,)) + dest = os.path.join(staging_dir, tar_relpath) + os.makedirs(os.path.dirname(dest) or staging_dir, exist_ok=True) + with open(dest, "w", encoding="utf-8") as fh: + json.dump(spec, fh, indent=2) + fh.write("\n") + return dest diff --git a/tests/test_preload_bundle.py b/tests/test_preload_bundle.py index ef55b9a..c6e005e 100644 --- a/tests/test_preload_bundle.py +++ b/tests/test_preload_bundle.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -"""Tests for bits_helpers/preload_bundle — CVMFS filebundle spec generation.""" +"""Tests for bits_helpers/preload_bundle — post-publish filebundle spec emitter.""" import json import os @@ -13,143 +13,86 @@ from bits_helpers import preload_bundle as P - -# A synthetic release: package root (arch-base-relative) -> repo-relative CVMFS -# path. 2-level (/) and grouped 3-level (//). -RELEASE = { - "xrootd/5.9.1-1": "el9/Packages/xrootd/5.9.1", - "Boost/1.90.0-1": "el9/Packages/Boost/1.90.0", - "gcc/GCC/14.2.0-1": "el9/Packages/GCC/14.2.0", # grouped: // -} -_meta_exists = lambda prefix: prefix in RELEASE -_resolve = RELEASE.get - - -class ParseSidecarTest(unittest.TestCase): - def _write(self, text): - fd, p = tempfile.mkstemp() - os.write(fd, text.encode()); os.close(fd) - self.addCleanup(os.remove, p) - return p - - def test_trigger_then_files_dedup_comments(self): - p = self._write( - "# a preload sidecar\n" - "xrootd/5.9.1-1/bin/xrdcp\n" - "Boost/1.90.0-1/lib/libboost.so\n" - "\n" - "Boost/1.90.0-1/lib/libboost.so\n" # dup dropped - "xrootd/5.9.1-1/lib/libXrdCl.so\n") - trig, files = P.parse_sidecar(p) - self.assertEqual(trig, "xrootd/5.9.1-1/bin/xrdcp") - self.assertEqual(files, ["Boost/1.90.0-1/lib/libboost.so", - "xrootd/5.9.1-1/lib/libXrdCl.so"]) - - def test_missing_file_is_safe(self): - self.assertEqual(P.parse_sidecar("/no/such/sidecar.paths"), (None, [])) - - -class OwningPkgRootTest(unittest.TestCase): - def test_two_level(self): - self.assertEqual( - P._owning_pkg_root("Boost/1.90.0-1/lib/libboost.so", _meta_exists), - ("Boost/1.90.0-1", "lib/libboost.so")) - - def test_grouped_three_level_longest_wins(self): - self.assertEqual( - P._owning_pkg_root("gcc/GCC/14.2.0-1/lib64/libstdc++.so", _meta_exists), - ("gcc/GCC/14.2.0-1", "lib64/libstdc++.so")) - - def test_unknown_owner(self): - self.assertEqual(P._owning_pkg_root("Nope/1.0/lib/x.so", _meta_exists), - (None, None)) - - -class BuildDependenciesTest(unittest.TestCase): - def test_maps_sorts_dedups_and_skips_trigger(self): - files = [ - "xrootd/5.9.1-1/bin/xrdcp", # trigger -> skipped - "Boost/1.90.0-1/lib/libboost.so", - "xrootd/5.9.1-1/lib/libXrdCl.so", - "gcc/GCC/14.2.0-1/lib64/libstdc++.so", +REPO = "/cvmfs/sft.cern.ch" +BASE = REPO + "/lcg/releases/x86_64-el9/xrootd/5.9.1" # deployed package dir + + +class RepoRootTest(unittest.TestCase): + def test_repo_root_of(self): + self.assertEqual(P.repo_root_of("/cvmfs/sft.cern.ch/lcg/releases"), + "/cvmfs/sft.cern.ch") + self.assertEqual(P.repo_root_of("/cvmfs/alice.cern.ch"), "/cvmfs/alice.cern.ch") + self.assertIsNone(P.repo_root_of("/home/user/sw")) + self.assertIsNone(P.repo_root_of("")) + + def test_to_repo_absolute(self): + self.assertEqual(P.to_repo_absolute(BASE + "/lib/libXrdCl.so", REPO), + "/lcg/releases/x86_64-el9/xrootd/5.9.1/lib/libXrdCl.so") + self.assertIsNone(P.to_repo_absolute("/usr/lib64/libc.so.6", REPO)) + self.assertIsNone(P.to_repo_absolute(REPO, REPO)) # the mount itself + + +class BuildBundleTest(unittest.TestCase): + def test_build_bundle(self): + trigger = BASE + "/bin/xrdcp" + opened = [ + trigger, # excluded (the trigger) + BASE + "/lib/libXrdCl.so", # own package + REPO + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", # dep + "/usr/lib64/libc.so.6", # system -> dropped + "/proc/self/maps", # dropped ] - deps = P.build_dependencies(files, _resolve, _meta_exists, - skip={"xrootd/5.9.1-1/bin/xrdcp"}) - self.assertEqual(deps, [ - "/el9/Packages/Boost/1.90.0/lib/libboost.so", - "/el9/Packages/GCC/14.2.0/lib64/libstdc++.so", - "/el9/Packages/xrootd/5.9.1/lib/libXrdCl.so", + tar_rel, spec = P.build_bundle(trigger, opened, REPO) + self.assertEqual(tar_rel, + "lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp") + self.assertEqual(spec["name"], "CVMFS_BUNDLE") + self.assertEqual(spec["version"], "1.0.0") + self.assertEqual(spec["encoding"], "UTF-8") + self.assertEqual(spec["dependencies"], [ + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", + "/lcg/releases/x86_64-el9/xrootd/5.9.1/lib/libXrdCl.so", ]) - def test_unsafe_and_unresolvable_dropped(self): - files = ["/abs/evil", "../escape/x", "Unknown/1.0/lib/x.so", - "Boost/1.90.0-1/lib/ok.so"] - deps = P.build_dependencies(files, _resolve, _meta_exists) - self.assertEqual(deps, ["/el9/Packages/Boost/1.90.0/lib/ok.so"]) + def test_trigger_not_under_repo(self): + self.assertEqual(P.build_bundle("/home/x/bin/tool", ["/home/x/lib/a"], REPO), + (None, None)) + + def test_no_in_repo_opens_no_bundle(self): + trigger = BASE + "/bin/xrdcp" + self.assertEqual( + P.build_bundle(trigger, [trigger, "/usr/lib64/libc.so.6"], REPO), + (None, None)) class RenderAndPathTest(unittest.TestCase): def test_render_spec_exact_envelope(self): - spec = P.render_spec(["/a", "/b"]) - self.assertEqual(spec, {"name": "CVMFS_BUNDLE", "version": "1.0.0", - "encoding": "UTF-8", "dependencies": ["/a", "/b"]}) + self.assertEqual(P.render_spec(["/a", "/b"]), + {"name": "CVMFS_BUNDLE", "version": "1.0.0", + "encoding": "UTF-8", "dependencies": ["/a", "/b"]}) def test_bundle_path_for(self): - self.assertEqual(P.bundle_path_for("bin/xrdcp"), "bin/.cvmfsbundle-xrdcp") + self.assertEqual(P.bundle_path_for("/x/bin/root"), "/x/bin/.cvmfsbundle-root") self.assertEqual(P.bundle_path_for("root"), ".cvmfsbundle-root") -class GenerateForPackageTest(unittest.TestCase): +class StageBundleTest(unittest.TestCase): def setUp(self): - self.pkgroot = tempfile.mkdtemp() - self.addCleanup(__import__("shutil").rmtree, self.pkgroot, True) - os.makedirs(os.path.join(self.pkgroot, P.SIDECAR_DIR)) - - def _sidecar(self, name, lines): - with open(os.path.join(self.pkgroot, P.SIDECAR_DIR, name), "w") as fh: - fh.write("\n".join(lines) + "\n") - - def test_writes_bundle_next_to_trigger_and_removes_sidecar(self): - self._sidecar("xrdcp.paths", [ - "xrootd/5.9.1-1/bin/xrdcp", # trigger - "Boost/1.90.0-1/lib/libboost.so", - "xrootd/5.9.1-1/lib/libXrdCl.so", - ]) - written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) - self.assertEqual(written, ["bin/.cvmfsbundle-xrdcp"]) - dest = os.path.join(self.pkgroot, "bin", ".cvmfsbundle-xrdcp") + self.d = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.d, True) + + def test_writes_json_into_staging_tree(self): + trigger = BASE + "/bin/xrdcp" + tar_rel, spec = P.build_bundle(trigger, [trigger, BASE + "/lib/l.so"], REPO) + dest = P.stage_bundle(self.d, tar_rel, spec) + self.assertTrue(dest.endswith( + "lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp")) with open(dest) as fh: - doc = json.load(fh) - self.assertEqual(doc["name"], "CVMFS_BUNDLE") - self.assertEqual(doc["dependencies"], [ - "/el9/Packages/Boost/1.90.0/lib/libboost.so", - "/el9/Packages/xrootd/5.9.1/lib/libXrdCl.so", # trigger's own file kept; only the exe excluded - ]) - # sidecar dir removed so it never reaches CVMFS - self.assertFalse(os.path.exists(os.path.join(self.pkgroot, P.SIDECAR_DIR))) - - def test_empty_deps_sidecar_writes_no_bundle(self): - self._sidecar("xrdcp.paths", ["xrootd/5.9.1-1/bin/xrdcp"]) # only the trigger - written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) - self.assertEqual(written, []) - self.assertFalse(os.path.exists(os.path.join(self.pkgroot, "bin"))) - - def test_one_bad_sidecar_does_not_abort_and_dir_always_removed(self): - # Inject a write-time failure: a plain file where one bundle's directory - # needs to be created, so os.makedirs raises for that sidecar only. - self._sidecar("good.paths", ["xrootd/5.9.1-1/bin/xrdcp", - "Boost/1.90.0-1/lib/libboost.so"]) - open(os.path.join(self.pkgroot, "clash"), "w").close() # not a dir - self._sidecar("clash.paths", ["xrootd/5.9.1-1/clash/x", - "Boost/1.90.0-1/lib/libboost.so"]) - written = P.generate_for_package(self.pkgroot, _resolve, _meta_exists) - # the good sidecar still produced its bundle; clash was skipped, not fatal - self.assertEqual(written, ["bin/.cvmfsbundle-xrdcp"]) - # cleanup ran despite the failure - self.assertFalse(os.path.exists(os.path.join(self.pkgroot, P.SIDECAR_DIR))) - - def test_nul_in_path_is_rejected(self): - self.assertFalse(P._is_safe_rel("Boost/1.90.0-1/lib/\x00evil")) + self.assertEqual(json.load(fh)["name"], "CVMFS_BUNDLE") + + def test_unsafe_relpath_rejected(self): + for bad in ("/abs/x", "../escape/x", "a/\x00/b"): + with self.assertRaises(ValueError): + P.stage_bundle(self.d, bad, P.render_spec(["/a"])) if __name__ == "__main__": From c2af59aac5825c8cae4735f2e1f2e36b39e25714 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 12:39:43 +0200 Subject: [PATCH 03/10] preload: bits preload driver helpers (recipe triggers, locate, strace parse, tar) Pure, unit-tested steps for the post-publish tool: detect/extract a recipe's Preload() triggers by parsing cvmfs_preload calls (no recipe sourcing), locate a package under a deployed CVMFS path, parse strace open/openat output, and assemble staged .cvmfsbundle-* files into one tar. Orchestration + wiring next. --- bits_helpers/preload_cmd.py | 146 ++++++++++++++++++++++++++++++++++++ tests/test_preload_cmd.py | 115 ++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 bits_helpers/preload_cmd.py create mode 100644 tests/test_preload_cmd.py diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py new file mode 100644 index 0000000..cc2c9ad --- /dev/null +++ b/bits_helpers/preload_cmd.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""`bits preload` — post-publish CVMFS filebundle generator. + +Given a list of packages and a deployed CVMFS path, for each package that +declares a ``Preload()`` in its recipe: locate it under the tree, set up its +deployed environment, strace the recipe's trigger (via a tool-provided +``cvmfs_preload`` so ``bits-recipe-tools`` is untouched), turn the captured +``/cvmfs`` opens into ``.cvmfsbundle-`` spec files, tar them, and +publish that one tar into CVMFS. + +This module keeps the pure, unit-testable steps (recipe `Preload` detection, +locating a package on the tree, parsing the tracer output, assembling bundles +and the tar). The strace, environment setup and publish are shelled out and are +validated on a host with a real mounted repo — they cannot run in CI. +""" + +import os +import re +import shlex +import tarfile + +from bits_helpers import preload_bundle as B + +_PRELOAD_DEF = re.compile(r'(?m)^[ \t]*(?:function[ \t]+)?Preload[ \t]*\([ \t]*\)') +# open("/path", ...) and openat(AT_FDCWD, "/path", ...) — capture the path arg. +_OPEN_RE = re.compile(r'(?:\bopen\(|\bopenat\(AT_FDCWD,\s*)"([^"]+)"') + + +def has_preload(recipe_body): + """True if a recipe's bash body defines a ``Preload()`` function.""" + return bool(recipe_body and _PRELOAD_DEF.search(recipe_body)) + + +def preload_triggers(recipe_body): + """Extract ``(exe, [args])`` for each ``cvmfs_preload`` call in ``Preload()``. + + We parse the calls rather than source the recipe body: sourcing arbitrary + build-time bash (with its ``bits-include`` and top-level side effects) is + unsafe and unnecessary — a ``Preload()`` in practice just lists + ``cvmfs_preload [args...]`` invocations. The ``Preload()`` block is + isolated by brace matching; each ``cvmfs_preload`` line is shlex-split. + Lines that will not shlex-parse are skipped. + """ + if not has_preload(recipe_body): + return [] + m = _PRELOAD_DEF.search(recipe_body) + body = recipe_body[m.end():] + open_brace = body.find("{") + if open_brace < 0: + return [] + depth, i, n = 0, open_brace, len(body) + while i < n: # find the matching close brace + if body[i] == "{": + depth += 1 + elif body[i] == "}": + depth -= 1 + if depth == 0: + break + i += 1 + block = body[open_brace + 1:i] + triggers = [] + for line in block.replace(";", "\n").splitlines(): + line = line.strip() + if not line.startswith("cvmfs_preload"): + continue + try: + toks = shlex.split(line) + except ValueError: + continue + if len(toks) >= 2: # cvmfs_preload [args...] + triggers.append((toks[1], toks[2:])) + return triggers + + +def parse_strace_opens(strace_text): + """Absolute paths opened in ``strace -e trace=open,openat`` output. + + Returns them in first-seen order, de-duplicated; relative paths (rare, from + an already-chdir'd process) are dropped since they cannot be mapped to a + repo-absolute location. + """ + out, seen = [], set() + for p in _OPEN_RE.findall(strace_text or ""): + if p.startswith("/") and p not in seen: + seen.add(p) + out.append(p) + return out + + +def locate_package(cvmfs_path, pkg, ver=None): + """Deployed package directory ``…//`` under *cvmfs_path*. + + Finds a directory named *pkg* that itself contains a version directory; when + *ver* is given, the version dir must equal it or start with ``ver + '-'`` + (so ``6.24.06`` matches ``6.24.06-4``). Returns the newest matching verrev's + absolute path, or None. Does not follow symlinks out of the tree. + """ + hits = [] + for dirpath, dirnames, _ in os.walk(cvmfs_path): + if os.path.basename(dirpath) != pkg: + continue + for v in sorted(dirnames): + if ver and not (v == ver or v.startswith(ver + "-")): + continue + hits.append(os.path.join(dirpath, v)) + dirnames[:] = [] # a dir's children are versions, stop + if not hits: + return None + return sorted(hits)[-1] + + +def assemble_bundles(traces, repo_root, staging_dir): + """Stage a ``.cvmfsbundle-*`` file for every trace block that yields deps. + + *traces* is the parsed tracer output; each ``(trigger, opens)`` becomes a + bundle via :func:`preload_bundle.build_bundle`, written into *staging_dir* + at its repo-relative path. Returns the sorted list of staged tar-relative + paths (empty when nothing under the repo was opened). + """ + staged = [] + for trigger_abs, opened_abs in traces: + tar_rel, spec = B.build_bundle(trigger_abs, opened_abs, repo_root) + if not tar_rel: + continue + B.stage_bundle(staging_dir, tar_rel, spec) + staged.append(tar_rel) + return sorted(staged) + + +def make_tar(staging_dir, out_tar): + """Tar the staged bundle tree (paths relative to *staging_dir*) into *out_tar*. + + Deterministic ordering. Returns *out_tar*. An empty staging tree yields an + empty tar, which the caller should not publish. + """ + entries = [] + for dp, _dn, fns in os.walk(staging_dir): + for f in fns: + full = os.path.join(dp, f) + entries.append((full, os.path.relpath(full, staging_dir))) + with tarfile.open(out_tar, "w") as tf: + for full, arc in sorted(entries, key=lambda e: e[1]): + tf.add(full, arcname=arc, recursive=False) + return out_tar diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py new file mode 100644 index 0000000..4401e65 --- /dev/null +++ b/tests/test_preload_cmd.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/preload_cmd — the pure steps of `bits preload`.""" + +import os +import sys +import tarfile +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import preload_cmd as C + +REPO = "/cvmfs/sft.cern.ch" +PKGDIR = REPO + "/lcg/releases/x86_64-el9/xrootd/5.9.1" + + +class HasPreloadTest(unittest.TestCase): + def test_detects_forms(self): + self.assertTrue(C.has_preload("function Preload() {\n cvmfs_preload bin/x\n}")) + self.assertTrue(C.has_preload("Preload () {\n :\n}")) + self.assertTrue(C.has_preload(" Preload(){ cvmfs_preload bin/root -b -q; }")) + + def test_absent(self): + self.assertFalse(C.has_preload("function Build() { true; }")) + self.assertFalse(C.has_preload("")) + self.assertFalse(C.has_preload("# mentions Preload() in a comment only? no def")) + + +class LocatePackageTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for rel in ("lcg/releases/x86_64-el9/xrootd/5.9.1", + "lcg/releases/x86_64-el9/xrootd/5.9.0-2", + "lcg/releases/x86_64-el9/Boost/1.90.0-1"): + os.makedirs(os.path.join(self.root, rel)) + + def test_newest_version(self): + got = C.locate_package(self.root, "xrootd") + self.assertTrue(got.endswith("xrootd/5.9.1")) + + def test_version_prefix_match(self): + got = C.locate_package(self.root, "xrootd", "5.9.0") + self.assertTrue(got.endswith("xrootd/5.9.0-2")) + + def test_absent(self): + self.assertIsNone(C.locate_package(self.root, "ROOT")) + + +class PreloadTriggersTest(unittest.TestCase): + def test_extracts_exe_and_args(self): + body = ("MODULE_OPTIONS=x\n" + "function Preload() {\n" + " cvmfs_preload bin/root -b -q\n" + " cvmfs_preload bin/hadd\n" + "}\n" + "function Build(){ cvmfs_preload NOT_A_TRIGGER; }\n") + self.assertEqual(C.preload_triggers(body), + [("bin/root", ["-b", "-q"]), ("bin/hadd", [])]) + + def test_semicolon_separated_and_quotes(self): + body = 'Preload() { cvmfs_preload bin/app "a b" -x; }' + self.assertEqual(C.preload_triggers(body), [("bin/app", ["a b", "-x"])]) + + def test_none_when_no_preload(self): + self.assertEqual(C.preload_triggers("Build(){ true; }"), []) + + +class ParseStraceTest(unittest.TestCase): + def test_open_and_openat_abs_only_dedup(self): + text = ( + 'open("%s/lib/libXrdCl.so", O_RDONLY) = 3\n' + 'openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY) = 4\n' + 'openat(AT_FDCWD, "%s/lib/libXrdCl.so", O_RDONLY) = 5\n' # dup + 'openat(AT_FDCWD, "relative/path", O_RDONLY) = 6\n' # dropped + ) % (PKGDIR, PKGDIR) + self.assertEqual(C.parse_strace_opens(text), + [PKGDIR + "/lib/libXrdCl.so", "/usr/lib64/libc.so.6"]) + + +class AssembleAndTarTest(unittest.TestCase): + def setUp(self): + self.stage = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.stage, True) + + def test_assemble_then_tar(self): + traces = [(PKGDIR + "/bin/xrdcp", + [PKGDIR + "/bin/xrdcp", # trigger, excluded + PKGDIR + "/lib/libXrdCl.so", + REPO + "/lcg/releases/x86_64-el9/Boost/1.90.0/lib/libboost.so", + "/usr/lib64/libc.so.6"])] # system, dropped + staged = C.assemble_bundles(traces, REPO, self.stage) + self.assertEqual( + staged, + ["lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp"]) + out = os.path.join(self.stage, "..", "b.tar") + C.make_tar(self.stage, out) + with tarfile.open(out) as tf: + names = tf.getnames() + self.assertEqual( + names, + ["lcg/releases/x86_64-el9/xrootd/5.9.1/bin/.cvmfsbundle-xrdcp"]) + os.remove(out) + + def test_assemble_skips_when_no_repo_opens(self): + traces = [(PKGDIR + "/bin/xrdcp", + [PKGDIR + "/bin/xrdcp", "/usr/lib64/libc.so.6"])] + self.assertEqual(C.assemble_bundles(traces, REPO, self.stage), []) + + +if __name__ == "__main__": + unittest.main() From 379e46844df07c79f516cdd415310feb47492e8f Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 16:38:07 +0200 Subject: [PATCH 04/10] preload: drop failed-open probes and normalize paths in bundles parse_strace_opens keeps only successful opens (ret >= 0), dropping the loader's ENOENT probes in tls/, glibc-hwcaps/ and arch subdirs; to_repo_absolute normpath's the path so the BITS_ARCH_PREFIX="." '/./' does not leak into deps. --- bits_helpers/preload_bundle.py | 3 +++ bits_helpers/preload_cmd.py | 26 ++++++++++++++++---------- tests/test_preload_bundle.py | 6 ++++++ tests/test_preload_cmd.py | 9 ++++++--- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/bits_helpers/preload_bundle.py b/bits_helpers/preload_bundle.py index e6f5e90..1fe2a3b 100644 --- a/bits_helpers/preload_bundle.py +++ b/bits_helpers/preload_bundle.py @@ -46,6 +46,9 @@ def to_repo_absolute(abs_path, repo_root): filebundle spec's ``dependencies`` want. """ root = (repo_root or "").rstrip("/") + # Canonicalise: env setup can leave '/./' or '//' in captured paths (e.g. the + # BITS_ARCH_PREFIX="." trick used to source a deployed init.sh). + abs_path = os.path.normpath(abs_path) if not root or abs_path == root or not abs_path.startswith(root + "/"): return None return abs_path[len(root):] # keeps the leading '/' diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index cc2c9ad..f694d63 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -24,8 +24,10 @@ from bits_helpers import preload_bundle as B _PRELOAD_DEF = re.compile(r'(?m)^[ \t]*(?:function[ \t]+)?Preload[ \t]*\([ \t]*\)') -# open("/path", ...) and openat(AT_FDCWD, "/path", ...) — capture the path arg. -_OPEN_RE = re.compile(r'(?:\bopen\(|\bopenat\(AT_FDCWD,\s*)"([^"]+)"') +# open("/path", ...) = / openat(AT_FDCWD, "/path", ...) = — capture +# the path AND the syscall return so failed probes (= -1 ENOENT) are dropped. +_OPEN_RE = re.compile( + r'(?:\bopen\(|\bopenat\(AT_FDCWD,\s*)"([^"]+)"[^=\n]*=\s*(-?\d+)') def has_preload(recipe_body): @@ -75,17 +77,21 @@ def preload_triggers(recipe_body): def parse_strace_opens(strace_text): - """Absolute paths opened in ``strace -e trace=open,openat`` output. + """Absolute paths SUCCESSFULLY opened in ``strace -e open,openat`` output. - Returns them in first-seen order, de-duplicated; relative paths (rare, from - an already-chdir'd process) are dropped since they cannot be mapped to a - repo-absolute location. + Keeps only opens whose syscall returned a valid fd (``= N`` with N >= 0), so + the dynamic loader's failed probes (``= -1 ENOENT`` in ``glibc-hwcaps/``, + ``tls/`` and arch subdirs) are excluded — otherwise the bundle lists files + that do not exist. First-seen order, de-duplicated; relative paths (from an + already-chdir'd process) are dropped as they cannot be mapped to the repo. """ out, seen = [], set() - for p in _OPEN_RE.findall(strace_text or ""): - if p.startswith("/") and p not in seen: - seen.add(p) - out.append(p) + for path, ret in _OPEN_RE.findall(strace_text or ""): + if ret.startswith("-"): # failed syscall (-1 ENOENT, …) + continue + if path.startswith("/") and path not in seen: + seen.add(path) + out.append(path) return out diff --git a/tests/test_preload_bundle.py b/tests/test_preload_bundle.py index c6e005e..ffa9942 100644 --- a/tests/test_preload_bundle.py +++ b/tests/test_preload_bundle.py @@ -31,6 +31,12 @@ def test_to_repo_absolute(self): self.assertIsNone(P.to_repo_absolute("/usr/lib64/libc.so.6", REPO)) self.assertIsNone(P.to_repo_absolute(REPO, REPO)) # the mount itself + def test_to_repo_absolute_normalizes_dot_and_slashes(self): + # '/./' and '//' from the BITS_ARCH_PREFIX="." env trick are collapsed. + self.assertEqual( + P.to_repo_absolute(REPO + "/lcg/./Packages//Boost/lib/l.so", REPO), + "/lcg/Packages/Boost/lib/l.so") + class BuildBundleTest(unittest.TestCase): def test_build_bundle(self): diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py index 4401e65..a5dca4c 100644 --- a/tests/test_preload_cmd.py +++ b/tests/test_preload_cmd.py @@ -70,13 +70,16 @@ def test_none_when_no_preload(self): class ParseStraceTest(unittest.TestCase): - def test_open_and_openat_abs_only_dedup(self): + def test_success_only_abs_dedup(self): text = ( 'open("%s/lib/libXrdCl.so", O_RDONLY) = 3\n' 'openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY) = 4\n' 'openat(AT_FDCWD, "%s/lib/libXrdCl.so", O_RDONLY) = 5\n' # dup - 'openat(AT_FDCWD, "relative/path", O_RDONLY) = 6\n' # dropped - ) % (PKGDIR, PKGDIR) + 'openat(AT_FDCWD, "relative/path", O_RDONLY) = 6\n' # relative -> dropped + # failed loader probes must be dropped, not listed: + 'openat(AT_FDCWD, "%s/lib/tls/x86_64/libXrdCl.so", O_RDONLY) = -1 ENOENT (No such file or directory)\n' + 'openat(AT_FDCWD, "%s/lib/glibc-hwcaps/x86-64-v3/libstdc++.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)\n' + ) % (PKGDIR, PKGDIR, PKGDIR, PKGDIR) self.assertEqual(C.parse_strace_opens(text), [PKGDIR + "/lib/libXrdCl.so", "/usr/lib64/libc.so.6"]) From c06e48e26fb33c0d8f7d8be4c5af4a2c9e2ce4db Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 17:30:55 +0200 Subject: [PATCH 05/10] preload: exclude recipe preload: section from the build hash Add 'preload' to _HASH_EXCLUDED_META_KEYS so a recipe can carry its CVMFS filebundle test list (consumed post-publish by 'bits preload') without editing it forcing a rebuild. --- bits_helpers/build.py | 3 +++ tests/test_hashing.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 0413a44..ed64f87 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -723,6 +723,9 @@ def storeHook(package, specs, defaults) -> bool: _HASH_EXCLUDED_META_KEYS = frozenset({ "license", "description", "url", "homepage", "acknowledgment", "acknowledgement", "source_url", "redistributable", + # preload: CVMFS filebundle test list, consumed post-publish by `bits preload`; + # it never affects the build, so editing it must not force a rebuild. + "preload", }) # Source-selection keys are ALSO dropped from the recipe TEXT hash — not because diff --git a/tests/test_hashing.py b/tests/test_hashing.py index dcf19a9..b9852f2 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -163,6 +163,21 @@ def test_adding_url_acknowledgment_source_redistributable_is_invariant(self): "redistributable: false\n") self.assertEqual(self._n(base), self._n(extra)) + def test_adding_preload_block_is_hash_invariant(self): + # The preload: test list (consumed post-publish by `bits preload`) is + # hash-excluded — its indented block is dropped and editing it must not + # change the build hash input. + withpl = self.HEADER.replace( + "requires:\n", + "preload:\n" + " - exe: bin/xrdcp\n" + " args: [--version]\n" + " - exe: bin/xrdfs\n" + "requires:\n") + self.assertNotIn("preload:", self._n(withpl)) + self.assertNotIn("xrdcp", self._n(withpl)) + self.assertEqual(self._n(self.HEADER), self._n(withpl)) + def test_multiline_block_value_dropped_entirely(self): base = self.HEADER.replace("description: A physics tool\n", "") block = self.HEADER.replace( From ad659f60fe5593e56739e3bd69352c2d15b0dd33 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 17:33:26 +0200 Subject: [PATCH 06/10] preload: recipe preload: reader, config loader, idempotency + discovery helpers Read tests from a recipe preload: section (replaces the Preload() bash parser); load config/preload.yaml (arch scalar/list or discover; packages bare/mapping with test overrides); resolve config-over-recipe; bundle_exists skip check; discover_archs / package_versions / package_dir via cvmfs_inspect. --- bits_helpers/preload_cmd.py | 152 ++++++++++++++++++++++++++---------- tests/test_preload_cmd.py | 108 +++++++++++++++++++------ 2 files changed, 196 insertions(+), 64 deletions(-) diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index f694d63..8b3b9e1 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -23,57 +23,129 @@ from bits_helpers import preload_bundle as B -_PRELOAD_DEF = re.compile(r'(?m)^[ \t]*(?:function[ \t]+)?Preload[ \t]*\([ \t]*\)') # open("/path", ...) = / openat(AT_FDCWD, "/path", ...) = — capture # the path AND the syscall return so failed probes (= -1 ENOENT) are dropped. _OPEN_RE = re.compile( r'(?:\bopen\(|\bopenat\(AT_FDCWD,\s*)"([^"]+)"[^=\n]*=\s*(-?\d+)') -def has_preload(recipe_body): - """True if a recipe's bash body defines a ``Preload()`` function.""" - return bool(recipe_body and _PRELOAD_DEF.search(recipe_body)) +def _norm_tests(raw): + """Normalise a ``preload:`` / config ``tests:`` list to ``[(exe, [args]), …]``. + + Accepts either mapping entries (``{exe: bin/x, args: [--v]}``) or bare + strings (``"bin/x --v"``, shlex-split). Entries without an exe are skipped. + """ + out = [] + for item in raw or []: + if isinstance(item, dict): + exe = item.get("exe") + args = item.get("args") or [] + if isinstance(args, str): + args = shlex.split(args) + args = [str(a) for a in args] + elif isinstance(item, str): + try: + toks = shlex.split(item) + except ValueError: + continue + exe, args = (toks[0] if toks else None), toks[1:] + else: + continue + if exe: + out.append((str(exe), args)) + return out -def preload_triggers(recipe_body): - """Extract ``(exe, [args])`` for each ``cvmfs_preload`` call in ``Preload()``. +def recipe_tests(spec): + """Tests from a recipe's hash-excluded ``preload:`` section (via ``parseRecipe``). - We parse the calls rather than source the recipe body: sourcing arbitrary - build-time bash (with its ``bits-include`` and top-level side effects) is - unsafe and unnecessary — a ``Preload()`` in practice just lists - ``cvmfs_preload [args...]`` invocations. The ``Preload()`` block is - isolated by brace matching; each ``cvmfs_preload`` line is shlex-split. - Lines that will not shlex-parse are skipped. + *spec* is the parsed YAML front-matter; ``spec['preload']`` is the test list. + Returns ``[(exe, [args]), …]`` (empty when the recipe declares none). """ - if not has_preload(recipe_body): - return [] - m = _PRELOAD_DEF.search(recipe_body) - body = recipe_body[m.end():] - open_brace = body.find("{") - if open_brace < 0: + if not isinstance(spec, dict): return [] - depth, i, n = 0, open_brace, len(body) - while i < n: # find the matching close brace - if body[i] == "{": - depth += 1 - elif body[i] == "}": - depth -= 1 - if depth == 0: - break - i += 1 - block = body[open_brace + 1:i] - triggers = [] - for line in block.replace(";", "\n").splitlines(): - line = line.strip() - if not line.startswith("cvmfs_preload"): - continue - try: - toks = shlex.split(line) - except ValueError: - continue - if len(toks) >= 2: # cvmfs_preload [args...] - triggers.append((toks[1], toks[2:])) - return triggers + return _norm_tests(spec.get("preload")) + + +def load_config(yaml_text): + """Parse ``config/preload.yaml`` into a normalised sweep config. + + Returns ``{arch: [..]|None, docker: bool, update: bool, packages: {name: + {tests: [(exe,[args])], versions: [..]}}}``. ``packages`` accepts a bare list + (names defer to the recipe), a mapping, or a list mixing names and single-key + mappings; ``arch`` accepts a scalar or list (None ⇒ discover). An empty/{} + ``packages`` means "all packages that carry a recipe preload:". + """ + from bits_helpers.utilities import yamlLoad + data = yamlLoad(yaml_text) or {} + arch = data.get("arch") + if isinstance(arch, str): + arch = [arch] + pkgs = {} + + def _add(name, spec): + spec = spec or {} + pkgs[name] = {"tests": _norm_tests(spec.get("tests")), + "versions": list(spec.get("versions") or [])} + + raw = data.get("packages") + if isinstance(raw, dict): + for name, spec in raw.items(): + _add(name, spec) + elif isinstance(raw, list): + for item in raw: + if isinstance(item, str): + _add(item, {}) + elif isinstance(item, dict) and len(item) == 1: + (name, spec), = item.items() + _add(name, spec) + return {"arch": arch or None, + "docker": bool(data.get("docker", False)), + "update": bool(data.get("update", False)), + "packages": pkgs} + + +def resolve_tests(cfg_pkg, recipe_spec): + """Tests for a package: the config's ``tests`` if any, else the recipe's. + + *cfg_pkg* is a ``load_config`` package entry (or None); *recipe_spec* the + package's parsed recipe front-matter (or None). + """ + if cfg_pkg and cfg_pkg.get("tests"): + return list(cfg_pkg["tests"]) + return recipe_tests(recipe_spec) + + +def bundle_exists(pkg_dir, exe): + """True if ``//.cvmfsbundle-`` already exists.""" + return os.path.exists(os.path.join(pkg_dir, B.bundle_path_for(exe))) + + +def discover_archs(cvmfs_root): + """Deployed platforms under *cvmfs_root* (reuses cvmfs_inspect).""" + from bits_helpers import cvmfs_inspect as I + return I.list_platforms(cvmfs_root) + + +def package_dir(cvmfs_root, arch, pkg, verrev): + """``//Packages//`` — a deployed package dir.""" + return os.path.join(cvmfs_root, arch, "Packages", pkg, verrev) + + +def package_versions(cvmfs_root, arch, pkg, want=None): + """Deployed ```` dirs for *pkg* (reuses cvmfs_inspect), newest last. + + *want* is an optional list of version globs (from config ``versions:``); a + bare version matches its ``-`` dir too (``5.9.1`` ⇒ ``5.9.1-1``). + """ + import fnmatch + from bits_helpers import cvmfs_inspect as I + vers = I.list_packages(cvmfs_root, arch).get(pkg, []) + if want: + vers = [v for v in vers + if any(fnmatch.fnmatch(v, w) or fnmatch.fnmatch(v, w + "-*") + for w in want)] + return vers def parse_strace_opens(strace_text): diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py index a5dca4c..f24e23a 100644 --- a/tests/test_preload_cmd.py +++ b/tests/test_preload_cmd.py @@ -17,16 +17,64 @@ PKGDIR = REPO + "/lcg/releases/x86_64-el9/xrootd/5.9.1" -class HasPreloadTest(unittest.TestCase): - def test_detects_forms(self): - self.assertTrue(C.has_preload("function Preload() {\n cvmfs_preload bin/x\n}")) - self.assertTrue(C.has_preload("Preload () {\n :\n}")) - self.assertTrue(C.has_preload(" Preload(){ cvmfs_preload bin/root -b -q; }")) +class RecipeTestsTest(unittest.TestCase): + def test_mapping_and_string_forms(self): + spec = {"package": "xrootd", "preload": [ + {"exe": "bin/xrdcp", "args": ["--version"]}, + {"exe": "bin/xrdfs", "args": "--help -v"}, # string args -> shlex + "bin/xrdmapc plain", # bare string entry + {"args": ["--x"]}, # no exe -> skipped + ]} + self.assertEqual(C.recipe_tests(spec), [ + ("bin/xrdcp", ["--version"]), + ("bin/xrdfs", ["--help", "-v"]), + ("bin/xrdmapc", ["plain"]), + ]) def test_absent(self): - self.assertFalse(C.has_preload("function Build() { true; }")) - self.assertFalse(C.has_preload("")) - self.assertFalse(C.has_preload("# mentions Preload() in a comment only? no def")) + self.assertEqual(C.recipe_tests({"package": "x"}), []) + self.assertEqual(C.recipe_tests(None), []) + + +class LoadConfigTest(unittest.TestCase): + def test_bare_list_and_mapping_override(self): + cfg = C.load_config( + "arch: x86_64-el9-gcc14-opt\n" + "docker: true\n" + "packages:\n" + " - xrootd\n" # bare -> defer to recipe + " - ROOT:\n" + " versions: ['6.38.*']\n" + " tests:\n" + " - { exe: bin/root, args: [-b, -q] }\n") + self.assertEqual(cfg["arch"], ["x86_64-el9-gcc14-opt"]) # scalar -> list + self.assertTrue(cfg["docker"]) + self.assertFalse(cfg["update"]) + self.assertEqual(cfg["packages"]["xrootd"], {"tests": [], "versions": []}) + self.assertEqual(cfg["packages"]["ROOT"]["tests"], [("bin/root", ["-b", "-q"])]) + self.assertEqual(cfg["packages"]["ROOT"]["versions"], ["6.38.*"]) + + def test_arch_omitted_is_none_for_discovery(self): + self.assertIsNone(C.load_config("docker: false\n")["arch"]) + + +class ResolveAndSkipTest(unittest.TestCase): + def test_config_tests_override_else_recipe(self): + recipe = {"preload": [{"exe": "bin/xrdcp", "args": ["--version"]}]} + self.assertEqual(C.resolve_tests({"tests": [("bin/x", [])]}, recipe), + [("bin/x", [])]) # config wins + self.assertEqual(C.resolve_tests({"tests": []}, recipe), + [("bin/xrdcp", ["--version"])]) # falls back to recipe + self.assertEqual(C.resolve_tests(None, recipe), + [("bin/xrdcp", ["--version"])]) + + def test_bundle_exists(self): + d = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, d, True) + self.assertFalse(C.bundle_exists(d, "bin/xrdcp")) + os.makedirs(os.path.join(d, "bin")) + open(os.path.join(d, "bin", ".cvmfsbundle-xrdcp"), "w").close() + self.assertTrue(C.bundle_exists(d, "bin/xrdcp")) class LocatePackageTest(unittest.TestCase): @@ -50,23 +98,35 @@ def test_absent(self): self.assertIsNone(C.locate_package(self.root, "ROOT")) -class PreloadTriggersTest(unittest.TestCase): - def test_extracts_exe_and_args(self): - body = ("MODULE_OPTIONS=x\n" - "function Preload() {\n" - " cvmfs_preload bin/root -b -q\n" - " cvmfs_preload bin/hadd\n" - "}\n" - "function Build(){ cvmfs_preload NOT_A_TRIGGER; }\n") - self.assertEqual(C.preload_triggers(body), - [("bin/root", ["-b", "-q"]), ("bin/hadd", [])]) - - def test_semicolon_separated_and_quotes(self): - body = 'Preload() { cvmfs_preload bin/app "a b" -x; }' - self.assertEqual(C.preload_triggers(body), [("bin/app", ["a b", "-x"])]) +class DiscoveryTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for rel in ("x86_64-el9-gcc14-opt/Packages/xrootd/5.9.1-1", + "x86_64-el9-gcc14-opt/Packages/xrootd/5.9.0-2", + "x86_64-el9-gcc14-opt/Packages/ROOT/6.38.00-1", + "aarch64-el9-gcc14-opt/Packages/xrootd/5.9.1-1"): + os.makedirs(os.path.join(self.root, rel)) - def test_none_when_no_preload(self): - self.assertEqual(C.preload_triggers("Build(){ true; }"), []) + def test_discover_archs(self): + self.assertEqual(sorted(C.discover_archs(self.root)), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + + def test_package_versions_all_and_filtered(self): + a = "x86_64-el9-gcc14-opt" + self.assertEqual(sorted(C.package_versions(self.root, a, "xrootd")), + ["5.9.0-2", "5.9.1-1"]) + # bare version glob matches its - dir + self.assertEqual(C.package_versions(self.root, a, "xrootd", ["5.9.1"]), + ["5.9.1-1"]) + self.assertEqual(C.package_versions(self.root, a, "ROOT", ["6.38.*"]), + ["6.38.00-1"]) + self.assertEqual(C.package_versions(self.root, a, "xrootd", ["9.9.9"]), []) + + def test_package_dir(self): + self.assertEqual( + C.package_dir(self.root, "x86_64-el9-gcc14-opt", "xrootd", "5.9.1-1"), + os.path.join(self.root, "x86_64-el9-gcc14-opt/Packages/xrootd/5.9.1-1")) class ParseStraceTest(unittest.TestCase): From 83027b6f04464917ebf50025d3ff47df469c45d7 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 17:49:08 +0200 Subject: [PATCH 07/10] preload: bits preload sweep + run/main + wrapper wiring Config-driven sweep over arch x package x version x test: strace each recipe test (native or --docker), build .cvmfsbundle-* specs, one tar. Idempotent (skip existing unless --update), arch discovery, injectable tracer for tests. Wire 'bits preload' in the wrapper. --publish deferred to the prepub --tar path. --- bits | 9 ++ bits_helpers/preload_cmd.py | 192 ++++++++++++++++++++++++++++++++++++ tests/test_preload_cmd.py | 77 +++++++++++++++ 3 files changed, 278 insertions(+) diff --git a/bits b/bits index 8d96ec4..7fc48e0 100755 --- a/bits +++ b/bits @@ -405,6 +405,15 @@ if [[ "${1:-}" == "cvmfs" ]]; then BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.cvmfs_inspect import main; sys.exit(main())' "$@" fi +# `bits preload …` → post-publish CVMFS filebundle generator. Traces a deployed +# package's recipe-declared tests on the tree (strace, optionally --docker) and +# writes .cvmfsbundle-* prefetch files into one tar. Reads the tree (and runs +# binaries); no work dir, so handled here like `cvmfs`. +if [[ "${1:-}" == "preload" ]]; then + shift + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.preload_cmd import main; sys.exit(main())' "$@" +fi + # `bits cvmfs-stage …` → producer-side CVMFS staging (ADR-0011). # Prepares a package into a staging S3 prefix with the canonical publisher and # names the catalog prepub must graft. Handled here, before work-dir/module diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index 8b3b9e1..15fb4ef 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -207,6 +207,34 @@ def assemble_bundles(traces, repo_root, staging_dir): return sorted(staged) +def packages_base(pkg_dir): + """The ``…//Packages`` base for a ``…/Packages//`` dir.""" + return os.path.dirname(os.path.dirname(pkg_dir)) + + +def build_trace_script(pkg_dir, exe, args, log_path): + """Bash that sets up the deployed env and straces one trigger to *log_path*. + + Sources the deployed ``init.sh`` with ``WORK_DIR=`` and + ``BITS_ARCH_PREFIX="."`` (so its ``$WORK_DIR/$BITS_ARCH_PREFIX/`` deps + resolve without doubling the arch — validated on the host), then runs + ``strace -f -e trace=open,openat``. No ``set -u``: init.sh references unset + vars by design. Returns the script text. + """ + import shlex + exe_abs = os.path.join(pkg_dir, exe) + quoted = " ".join(shlex.quote(a) for a in (args or [])) + return "\n".join([ + "set -o pipefail", + "export WORK_DIR=%s" % shlex.quote(packages_base(pkg_dir)), + 'export BITS_ARCH_PREFIX="."', + "INIT=%s/etc/profile.d/init.sh" % shlex.quote(pkg_dir), + '[ -r "$INIT" ] && . "$INIT" 2>/dev/null || true', + "strace -f -e trace=open,openat -o %s %s %s >/dev/null 2>&1 || true" + % (shlex.quote(log_path), shlex.quote(exe_abs), quoted), + ]) + "\n" + + def make_tar(staging_dir, out_tar): """Tar the staged bundle tree (paths relative to *staging_dir*) into *out_tar*. @@ -222,3 +250,167 @@ def make_tar(staging_dir, out_tar): for full, arc in sorted(entries, key=lambda e: e[1]): tf.add(full, arcname=arc, recursive=False) return out_tar + + +# ── host-only trace executors (proven by run-preload-xrootd.sh) ─────────────── + +# A trigger that blocks (waits on stdin/network) must not stall the sweep. +_TRACE_TIMEOUT = 300 + + +def _trace_native(pkg_dir, exe, args, log_dir): + """Run the trace script on the host; return the strace log text ("" on error).""" + import subprocess + log = os.path.join(log_dir, "strace.log") + script = build_trace_script(pkg_dir, exe, args, log) + try: + subprocess.run(["bash", "-c", script], check=False, timeout=_TRACE_TIMEOUT) + with open(log) as fh: + return fh.read() + except Exception: # never let one trace abort the sweep + return "" + + +def _trace_docker(pkg_dir, exe, args, log_dir, image, cvmfs_mount="/cvmfs"): + """Run the trace script inside *image* with /cvmfs bind-mounted + SYS_PTRACE.""" + import subprocess + script = build_trace_script(pkg_dir, exe, args, "/out/strace.log") + cmd = ["docker", "run", "--rm", "-i", + "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", + "-v", "%s:%s:ro,rslave" % (cvmfs_mount, cvmfs_mount), + "-v", "%s:/out" % log_dir, image, "bash", "-s"] + try: + subprocess.run(cmd, input=script, text=True, check=False, + timeout=_TRACE_TIMEOUT) + with open(os.path.join(log_dir, "strace.log")) as fh: + return fh.read() + except Exception: # docker missing, timeout, bad image, … + return "" + + +def _default_tracer(docker=False, image=None): + """Return a tracer(pkg_dir, exe, args) -> [opened_abs] for the sweep.""" + import tempfile + + def tracer(pkg_dir, exe, args): + d = tempfile.mkdtemp(prefix="preload-") + try: + text = (_trace_docker(pkg_dir, exe, args, d, image) if docker + else _trace_native(pkg_dir, exe, args, d)) + finally: + __import__("shutil").rmtree(d, ignore_errors=True) + return parse_strace_opens(text) + return tracer + + +# ── the sweep ───────────────────────────────────────────────────────────────── + +def sweep(cvmfs_root, config, recipe_reader, staging_dir, + tracer, update=False, log=None, repo_root=None): + """Run the config-driven sweep, staging bundles into *staging_dir*. + + *recipe_reader(pkg)* returns a package's parsed recipe front-matter (or None); + *tracer(pkg_dir, exe, args)* returns the opened absolute paths (injected so + the loop is testable without strace). Skips a test whose bundle already exists + unless *update*. *repo_root* defaults to the mount of *cvmfs_root* (override + only in tests). Returns the sorted list of staged bundle tar-paths. + """ + log = log or (lambda *a: None) + repo_root = repo_root or B.repo_root_of(cvmfs_root) + if not repo_root: + raise ValueError("not a /cvmfs//… path: %s" % cvmfs_root) + archs = config.get("arch") or discover_archs(cvmfs_root) + staged = [] + for arch in archs: + for pkg, cfg_pkg in _sweep_packages(cvmfs_root, arch, config, recipe_reader): + tests = resolve_tests(cfg_pkg, recipe_reader(pkg)) + if not tests: + continue + for verrev in package_versions(cvmfs_root, arch, pkg, + (cfg_pkg or {}).get("versions")): + pdir = package_dir(cvmfs_root, arch, pkg, verrev) + for exe, args in tests: + if bundle_exists(pdir, exe) and not update: + log("skip %s/%s %s (bundle exists)", arch, verrev, exe) + continue + exe_abs = os.path.join(pdir, exe) + tar_rel, spec = B.build_bundle(exe_abs, tracer(pdir, exe, args), + repo_root) + if not tar_rel: + log("empty %s/%s %s (no in-repo opens)", arch, verrev, exe) + continue + B.stage_bundle(staging_dir, tar_rel, spec) + staged.append(tar_rel) + log("staged %s", tar_rel) + return sorted(staged) + + +def _sweep_packages(cvmfs_root, arch, config, recipe_reader): + """Yield ``(pkg, cfg_pkg)`` for the sweep: the config's packages, or — when + none are listed — every deployed package that carries a recipe ``preload:``.""" + pkgs = config.get("packages") or {} + if pkgs: + for name, cfg_pkg in pkgs.items(): + yield name, cfg_pkg + return + from bits_helpers import cvmfs_inspect as I + for name in sorted(I.list_packages(cvmfs_root, arch)): + if recipe_tests(recipe_reader(name)): + yield name, None + + +def main(argv=None): + import argparse + ap = argparse.ArgumentParser(prog="bits preload", + description="Generate CVMFS filebundle prefetch " + "files for deployed packages.") + ap.add_argument("--cvmfs", required=True, metavar="ROOT", + help="deployed tree root, e.g. /cvmfs//lcg/bits") + ap.add_argument("--config", metavar="YAML", help="config/preload.yaml") + ap.add_argument("--config-dir", default=".", metavar="DIR", + help="recipe directory (for preload: sections). Default: .") + ap.add_argument("--arch", action="append", metavar="ARCH", + help="platform (repeatable); default: discover from the tree") + ap.add_argument("--update", action="store_true", + help="regenerate even if a .cvmfsbundle-* already exists") + ap.add_argument("--docker", action="store_true", help="trace inside a container") + ap.add_argument("--docker-image", metavar="IMAGE", help="image for --docker") + ap.add_argument("--output", metavar="TAR", default="preload-bundles.tar", + help="write the bundle tar here. Default: %(default)s") + a = ap.parse_args(argv) + + if a.config and not os.path.isfile(a.config): + ap.error("config file not found: %s" % a.config) + config = load_config(open(a.config).read()) if a.config else { + "arch": None, "docker": False, "update": False, "packages": {}} + if a.arch: + config["arch"] = a.arch + update = a.update or config.get("update") + docker = a.docker or config.get("docker") + if docker and not a.docker_image: + ap.error("--docker requires --docker-image IMAGE") + + def recipe_reader(pkg): + from bits_helpers.utilities import parseRecipe, FileReader + path = os.path.join(a.config_dir, pkg + ".sh") + if not os.path.isfile(path): + return None + try: + _err, spec, _body = parseRecipe(FileReader(path)) + return spec + except Exception: + return None + + import tempfile + staging = tempfile.mkdtemp(prefix="preload-stage-") + try: + staged = sweep(a.cvmfs, config, recipe_reader, staging, + _default_tracer(docker, a.docker_image), update=update, + log=lambda f, *ar: print(">> " + (f % ar))) + if not staged: + print("no bundles generated"); return 0 + make_tar(staging, a.output) + print("wrote %d bundle(s) to %s" % (len(staged), a.output)) + finally: + __import__("shutil").rmtree(staging, ignore_errors=True) + return 0 diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py index f24e23a..8e07a68 100644 --- a/tests/test_preload_cmd.py +++ b/tests/test_preload_cmd.py @@ -129,6 +129,83 @@ def test_package_dir(self): os.path.join(self.root, "x86_64-el9-gcc14-opt/Packages/xrootd/5.9.1-1")) +class TraceScriptTest(unittest.TestCase): + def test_contains_env_setup_and_strace(self): + s = C.build_trace_script( + "/cvmfs/r/x86_64-el9/Packages/xrootd/5.9.1-1", "bin/xrdcp", + ["--version"], "/tmp/log") + self.assertIn('export WORK_DIR=/cvmfs/r/x86_64-el9/Packages', s) + self.assertIn('BITS_ARCH_PREFIX="."', s) + self.assertIn("etc/profile.d/init.sh", s) + self.assertIn("strace -f -e trace=open,openat", s) + self.assertIn("bin/xrdcp --version", s) + + +class SweepTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.root, True) + for v in ("5.9.1-1", "5.9.0-2"): + os.makedirs(os.path.join(self.root, "x86_64-el9/Packages/xrootd", v)) + self.cfg = {"arch": ["x86_64-el9"], + "packages": {"xrootd": {"tests": [("bin/xrdcp", [])], + "versions": []}}} + + def _tracer(self, pdir, exe, args): + # one in-repo lib + one system lib (dropped) + return [os.path.join(pdir, "lib/libXrdCl.so.3"), "/usr/lib64/libc.so.6"] + + def test_stages_bundle_per_version(self): + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, repo_root=self.root) + self.assertEqual(staged, [ + "x86_64-el9/Packages/xrootd/5.9.0-2/bin/.cvmfsbundle-xrdcp", + "x86_64-el9/Packages/xrootd/5.9.1-1/bin/.cvmfsbundle-xrdcp", + ]) + + def test_skips_existing_unless_update(self): + # pre-create the 5.9.1-1 bundle in the tree -> skipped by default + bdir = os.path.join(self.root, "x86_64-el9/Packages/xrootd/5.9.1-1/bin") + os.makedirs(bdir) + open(os.path.join(bdir, ".cvmfsbundle-xrdcp"), "w").close() + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, repo_root=self.root) + self.assertEqual(staged, + ["x86_64-el9/Packages/xrootd/5.9.0-2/bin/.cvmfsbundle-xrdcp"]) + # with --update both are (re)generated + staged = C.sweep(self.root, self.cfg, lambda p: None, stage, + self._tracer, update=True, repo_root=self.root) + self.assertEqual(len(staged), 2) + + def test_recipe_fallback_when_config_has_no_tests(self): + cfg = {"arch": ["x86_64-el9"], "packages": {"xrootd": None}} + reader = lambda p: {"preload": [{"exe": "bin/xrdfs", "args": ["--help"]}]} + stage = tempfile.mkdtemp(); self.addCleanup( + __import__("shutil").rmtree, stage, True) + staged = C.sweep(self.root, cfg, reader, stage, self._tracer, + repo_root=self.root) + self.assertTrue(all(s.endswith("/bin/.cvmfsbundle-xrdfs") for s in staged)) + self.assertEqual(len(staged), 2) + + +class MainGuardTest(unittest.TestCase): + def test_docker_requires_image(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main(["--cvmfs", "/cvmfs/r/lcg", "--docker"]) + + def test_missing_config_errors(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main(["--cvmfs", "/cvmfs/r/lcg", "--config", "/no/such.yaml"]) + + class ParseStraceTest(unittest.TestCase): def test_success_only_abs_dedup(self): text = ( From 840be45cb20191f8edac4ce75cb1055b5be59134 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 17:53:43 +0200 Subject: [PATCH 08/10] cvmfs-publish: --tar to publish overlay tar(s) at the repo root via prepub publish_overlay_tars stages each tar at path='' (files land at their repo- relative locations, e.g. bits preload .cvmfsbundle-*) through the existing stage_tar/submit_staged path; distinct job_id_base per tar; caller's tar kept. 'bits preload' prints the ready-to-run 'bits cvmfs-publish --tar' command. --- bits_helpers/cvmfs_publish.py | 46 ++++++++++++++++++++++++++++++++++- bits_helpers/preload_cmd.py | 2 ++ tests/test_cvmfs_publish.py | 37 ++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/bits_helpers/cvmfs_publish.py b/bits_helpers/cvmfs_publish.py index 4bf101a..7bc8daa 100644 --- a/bits_helpers/cvmfs_publish.py +++ b/bits_helpers/cvmfs_publish.py @@ -363,6 +363,29 @@ def _publish_tar(ctx, path, tar, label, fp=None): return jid +def publish_overlay_tars(ctx, tars): + """Publish each overlay tar at the repo ROOT (path="") via the staged/ingest + path; return the job ids. + + An overlay tar carries files at their repo-relative locations (e.g. the + ``.cvmfsbundle-*`` files `bits preload` writes next to their triggers), so a + single root lease drops each file into place — unlike a package tar staged at + one subtree. The caller's tar is preserved: a temp copy is published + (``_publish_tar`` removes what it publishes). Each tar gets a distinct + ``job_id_base`` so their leases do not collide. + """ + import shutil + base = ctx.get("job_id_base", "local") + jids = [] + for i, tar in enumerate(tars): + fd, tmp = tempfile.mkstemp(suffix=".tar", dir=ctx.get("tmp_dir") or None) + os.close(fd) + shutil.copyfile(tar, tmp) + jids.append(_publish_tar(dict(ctx, job_id_base="%s-tar%d" % (base, i)), + "", tmp, os.path.basename(tar))) + return jids + + def publish_one(spec, ctx): """Full producer pipeline for ONE package, staged OR ingest path. Mirrors the CI loop body: locate tar -> untar -> resolve path -> relocate -> relativise -> tar -> @@ -537,7 +560,11 @@ def main(argv=None): help="print the content fingerprint of a directory tree and " "exit (the CI uses this to fingerprint its own relocated " "tree with the identical algorithm); other args ignored") - ap.add_argument("--manifest") # required unless --fingerprint (checked below) + ap.add_argument("--manifest") # required unless --fingerprint/--tar (checked below) + ap.add_argument("--tar", action="append", default=[], metavar="FILE", + help="publish overlay tar(s) at the repo root (e.g. the bundle " + "tar from 'bits preload') via the staged/ingest path, " + "instead of a build manifest. Repeatable.") ap.add_argument("--repo") ap.add_argument("--one", help="publish only this package (increment-1 test)") ap.add_argument("--tars-root", default=os.path.join( @@ -594,6 +621,23 @@ def main(argv=None): if a.fingerprint: print(tree_fingerprint(a.fingerprint)) return 0 + if a.tar: + if not a.repo: + ap.error("--tar requires --repo") + ctx = {"repo": a.repo, "stratum0_url": a.stratum0_url, + "prepub_url": a.prepub_url, "token": a.token, + "job_id_base": a.job_id_base, "build_id": a.build_id, + "swissknife": a.swissknife or None, "base_root": a.base_root or None, + "bearer_auth": a.bearer_auth, "no_stats_db": a.no_stats_db, + "no_prepare_lock": a.no_prepare_lock, + "replace_on_conflict": a.replace_on_conflict, + "publish_path": a.publish_path, "direct_s3": a.direct_s3, + "submit": not a.dry_run, + "tmp_dir": os.path.join(os.environ.get("BITS_WORK_DIR", "/tmp"), "tmp")} + os.makedirs(ctx["tmp_dir"], exist_ok=True) + for tar, jid in zip(a.tar, publish_overlay_tars(ctx, a.tar)): + print("published overlay %s -> %s" % (tar, jid)) + return 0 if not a.manifest or not a.repo: ap.error("--manifest and --repo are required") if a.dry_run: diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index 15fb4ef..03843b4 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -411,6 +411,8 @@ def recipe_reader(pkg): print("no bundles generated"); return 0 make_tar(staging, a.output) print("wrote %d bundle(s) to %s" % (len(staged), a.output)) + print("publish with: bits cvmfs-publish --tar %s --repo " + "--prepub-url [--stratum0-url ]" % a.output) finally: __import__("shutil").rmtree(staging, ignore_errors=True) return 0 diff --git a/tests/test_cvmfs_publish.py b/tests/test_cvmfs_publish.py index 0063c7b..e39fa9c 100644 --- a/tests/test_cvmfs_publish.py +++ b/tests/test_cvmfs_publish.py @@ -622,3 +622,40 @@ def test_ingest_workers_gt1_does_not_require_stage_flags(self): if __name__ == "__main__": unittest.main() + + +class TestPublishOverlayTars(unittest.TestCase): + """publish_overlay_tars stages each tar at the repo root (path=''), gives each + a distinct job_id_base, and preserves the caller's tar (a temp copy is + published). The actual stage/submit is proven on a build host separately.""" + + def test_overlay_publish(self): + import bits_helpers.cvmfs_publish as P + calls = [] + + def fake_publish_tar(ctx, path, tar, label, fp=None): + # mimic _publish_tar: it removes the (temp) tar it publishes + calls.append((ctx["job_id_base"], path, label, os.path.exists(tar))) + os.remove(tar) + return "job-" + label + + orig = P._publish_tar + P._publish_tar = fake_publish_tar + try: + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + t1 = os.path.join(d, "a.tar"); open(t1, "w").write("x") + t2 = os.path.join(d, "b.tar"); open(t2, "w").write("y") + jids = P.publish_overlay_tars( + {"job_id_base": "base", "tmp_dir": d}, [t1, t2]) + self.assertEqual(jids, ["job-a.tar", "job-b.tar"]) + self.assertEqual([c[1] for c in calls], ["", ""]) # root path + self.assertEqual([c[0] for c in calls], ["base-tar0", "base-tar1"]) + self.assertTrue(all(c[3] for c in calls)) # temp existed + self.assertTrue(os.path.exists(t1) and os.path.exists(t2)) # originals kept + finally: + P._publish_tar = orig + + +if __name__ == "__main__": + unittest.main() From a69f96837100540b2860cc5182f5f79bf62977f9 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 17:57:41 +0200 Subject: [PATCH 09/10] preload: config may fully specify cvmfs/arch/packages/tests (recipe-independent) Add a cvmfs: key to preload.yaml and make --cvmfs optional (CLI overrides config, error if neither). With per-package tests: in the config no recipe is consulted, so 'bits preload --config preload.yaml' works with no bits recipes. --- bits_helpers/preload_cmd.py | 20 ++++++++++++++------ tests/test_preload_cmd.py | 8 ++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index 03843b4..923cfba 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -99,7 +99,8 @@ def _add(name, spec): elif isinstance(item, dict) and len(item) == 1: (name, spec), = item.items() _add(name, spec) - return {"arch": arch or None, + return {"cvmfs": data.get("cvmfs") or None, + "arch": arch or None, "docker": bool(data.get("docker", False)), "update": bool(data.get("update", False)), "packages": pkgs} @@ -364,9 +365,13 @@ def main(argv=None): ap = argparse.ArgumentParser(prog="bits preload", description="Generate CVMFS filebundle prefetch " "files for deployed packages.") - ap.add_argument("--cvmfs", required=True, metavar="ROOT", - help="deployed tree root, e.g. /cvmfs//lcg/bits") - ap.add_argument("--config", metavar="YAML", help="config/preload.yaml") + ap.add_argument("--cvmfs", metavar="ROOT", + help="deployed tree root, e.g. /cvmfs//lcg/bits " + "(overrides the config's cvmfs: key; required if the " + "config has none)") + ap.add_argument("--config", metavar="YAML", + help="preload.yaml: may fully specify cvmfs/arch/packages/tests " + "(recipe-independent), or just select scope/policy") ap.add_argument("--config-dir", default=".", metavar="DIR", help="recipe directory (for preload: sections). Default: .") ap.add_argument("--arch", action="append", metavar="ARCH", @@ -382,9 +387,12 @@ def main(argv=None): if a.config and not os.path.isfile(a.config): ap.error("config file not found: %s" % a.config) config = load_config(open(a.config).read()) if a.config else { - "arch": None, "docker": False, "update": False, "packages": {}} + "cvmfs": None, "arch": None, "docker": False, "update": False, "packages": {}} if a.arch: config["arch"] = a.arch + cvmfs = a.cvmfs or config.get("cvmfs") + if not cvmfs: + ap.error("no CVMFS tree given: pass --cvmfs or set cvmfs: in the config") update = a.update or config.get("update") docker = a.docker or config.get("docker") if docker and not a.docker_image: @@ -404,7 +412,7 @@ def recipe_reader(pkg): import tempfile staging = tempfile.mkdtemp(prefix="preload-stage-") try: - staged = sweep(a.cvmfs, config, recipe_reader, staging, + staged = sweep(cvmfs, config, recipe_reader, staging, _default_tracer(docker, a.docker_image), update=update, log=lambda f, *ar: print(">> " + (f % ar))) if not staged: diff --git a/tests/test_preload_cmd.py b/tests/test_preload_cmd.py index 8e07a68..ebde1ad 100644 --- a/tests/test_preload_cmd.py +++ b/tests/test_preload_cmd.py @@ -39,6 +39,7 @@ def test_absent(self): class LoadConfigTest(unittest.TestCase): def test_bare_list_and_mapping_override(self): cfg = C.load_config( + "cvmfs: /cvmfs/sft-nightlies-test.cern.ch/lcg/bits\n" "arch: x86_64-el9-gcc14-opt\n" "docker: true\n" "packages:\n" @@ -47,6 +48,7 @@ def test_bare_list_and_mapping_override(self): " versions: ['6.38.*']\n" " tests:\n" " - { exe: bin/root, args: [-b, -q] }\n") + self.assertEqual(cfg["cvmfs"], "/cvmfs/sft-nightlies-test.cern.ch/lcg/bits") self.assertEqual(cfg["arch"], ["x86_64-el9-gcc14-opt"]) # scalar -> list self.assertTrue(cfg["docker"]) self.assertFalse(cfg["update"]) @@ -205,6 +207,12 @@ def test_missing_config_errors(self): with self.assertRaises(SystemExit): C.main(["--cvmfs", "/cvmfs/r/lcg", "--config", "/no/such.yaml"]) + def test_no_cvmfs_anywhere_errors(self): + import io, contextlib + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + C.main([]) # neither --cvmfs nor config cvmfs: + class ParseStraceTest(unittest.TestCase): def test_success_only_abs_dedup(self): From a0c9aed32cde6c2f06a5c7ab73ae96441f2f7e9a Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Wed, 26 Aug 2026 18:12:07 +0200 Subject: [PATCH 10/10] preload: dispatch before the global --config pre-parse The wrapper pre-parses a global --config (bits.rc) that swallowed 'bits preload --config ', leaving the value as a stray positional. Handle 'preload' from raw ARGV before that loop so its own --config reaches preload_cmd. --- bits | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bits b/bits index 7fc48e0..c28654b 100755 --- a/bits +++ b/bits @@ -20,6 +20,15 @@ fi ARGV=("$@"); ARGC=$# # ARGC must be a plain integer, not an array +# `bits preload …` MUST be handled before the global `--config` pre-parse below, +# which would otherwise swallow preload's own `--config ` (a name clash with +# the bits.rc `--config`). Post-publish CVMFS filebundle generator: traces the +# deployed tests (strace, optionally --docker) and writes .cvmfsbundle-* into one +# tar. Self-contained; needs no work dir. Uses the raw ARGV, unmangled. +if [[ "${ARGV[0]:-}" == "preload" ]]; then + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.preload_cmd import main; sys.exit(main())' "${ARGV[@]:1}" +fi + # Optional phase timing for diagnosing slow load/enter: set BITS_TIMING=1 to # print the wall time of each phase to stderr. Uses bash EPOCHREALTIME (>=5), # else GNU date; no overhead when unset. @@ -405,15 +414,6 @@ if [[ "${1:-}" == "cvmfs" ]]; then BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.cvmfs_inspect import main; sys.exit(main())' "$@" fi -# `bits preload …` → post-publish CVMFS filebundle generator. Traces a deployed -# package's recipe-declared tests on the tree (strace, optionally --docker) and -# writes .cvmfsbundle-* prefetch files into one tar. Reads the tree (and runs -# binaries); no work dir, so handled here like `cvmfs`. -if [[ "${1:-}" == "preload" ]]; then - shift - BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.preload_cmd import main; sys.exit(main())' "$@" -fi - # `bits cvmfs-stage …` → producer-side CVMFS staging (ADR-0011). # Prepares a package into a staging S3 prefix with the canonical publisher and # names the catalog prepub must graft. Handled here, before work-dir/module