generate CVMFS filebundle prefetch files

bits

@@ -20,6 +20,15 @@ fi
 
 ARGV=("$@"); ARGC=$#   # ARGC must be a plain integer, not an array
 
+# `bits preload …` MUST be handled before the global `--config` pre-parse below,
+# which would otherwise swallow preload's own `--config <yaml>` (a name clash with
+# the bits.rc `--config`). Post-publish CVMFS filebundle generator: traces the
+# deployed tests (strace, optionally --docker) and writes .cvmfsbundle-* into one
+# tar. Self-contained; needs no work dir. Uses the raw ARGV, unmangled.
+if [[ "${ARGV[0]:-}" == "preload" ]]; then
+  BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.preload_cmd import main; sys.exit(main())' "${ARGV[@]:1}"
+fi
+
 # Optional phase timing for diagnosing slow load/enter: set BITS_TIMING=1 to
 # print the wall time of each phase to stderr. Uses bash EPOCHREALTIME (>=5),
 # else GNU date; no overhead when unset.

bits_helpers/build.py

@@ -723,6 +723,9 @@ def storeHook(package, specs, defaults) -> bool:
 _HASH_EXCLUDED_META_KEYS = frozenset({
     "license", "description", "url", "homepage",
     "acknowledgment", "acknowledgement", "source_url", "redistributable",
+    # preload: CVMFS filebundle test list, consumed post-publish by `bits preload`;
+    # it never affects the build, so editing it must not force a rebuild.
+    "preload",
 })
 
 # Source-selection keys are ALSO dropped from the recipe TEXT hash — not because

bits_helpers/cvmfs_publish.py

@@ -363,6 +363,29 @@ def _publish_tar(ctx, path, tar, label, fp=None):
     return jid
 
 
+def publish_overlay_tars(ctx, tars):
+    """Publish each overlay tar at the repo ROOT (path="") via the staged/ingest
+    path; return the job ids.
+
+    An overlay tar carries files at their repo-relative locations (e.g. the
+    ``.cvmfsbundle-*`` files `bits preload` writes next to their triggers), so a
+    single root lease drops each file into place — unlike a package tar staged at
+    one subtree. The caller's tar is preserved: a temp copy is published
+    (``_publish_tar`` removes what it publishes). Each tar gets a distinct
+    ``job_id_base`` so their leases do not collide.
+    """
+    import shutil
+    base = ctx.get("job_id_base", "local")
+    jids = []
+    for i, tar in enumerate(tars):
+        fd, tmp = tempfile.mkstemp(suffix=".tar", dir=ctx.get("tmp_dir") or None)
+        os.close(fd)
+        shutil.copyfile(tar, tmp)
+        jids.append(_publish_tar(dict(ctx, job_id_base="%s-tar%d" % (base, i)),
+                                 "", tmp, os.path.basename(tar)))
+    return jids
+
+
 def publish_one(spec, ctx):
     """Full producer pipeline for ONE package, staged OR ingest path. Mirrors the CI loop
     body: locate tar -> untar -> resolve path -> relocate -> relativise -> tar ->
@@ -537,7 +560,11 @@ def main(argv=None):
                     help="print the content fingerprint of a directory tree and "
                          "exit (the CI uses this to fingerprint its own relocated "
                          "tree with the identical algorithm); other args ignored")
-    ap.add_argument("--manifest")   # required unless --fingerprint (checked below)
+    ap.add_argument("--manifest")   # required unless --fingerprint/--tar (checked below)
+    ap.add_argument("--tar", action="append", default=[], metavar="FILE",
+                    help="publish overlay tar(s) at the repo root (e.g. the bundle "
+                         "tar from 'bits preload') via the staged/ingest path, "
+                         "instead of a build manifest. Repeatable.")
     ap.add_argument("--repo")
     ap.add_argument("--one", help="publish only this package (increment-1 test)")
     ap.add_argument("--tars-root", default=os.path.join(
@@ -594,6 +621,23 @@ def main(argv=None):
     if a.fingerprint:
         print(tree_fingerprint(a.fingerprint))
         return 0
+    if a.tar:
+        if not a.repo:
+            ap.error("--tar requires --repo")
+        ctx = {"repo": a.repo, "stratum0_url": a.stratum0_url,
+               "prepub_url": a.prepub_url, "token": a.token,
+               "job_id_base": a.job_id_base, "build_id": a.build_id,
+               "swissknife": a.swissknife or None, "base_root": a.base_root or None,
+               "bearer_auth": a.bearer_auth, "no_stats_db": a.no_stats_db,
+               "no_prepare_lock": a.no_prepare_lock,
+               "replace_on_conflict": a.replace_on_conflict,
+               "publish_path": a.publish_path, "direct_s3": a.direct_s3,
+               "submit": not a.dry_run,
+               "tmp_dir": os.path.join(os.environ.get("BITS_WORK_DIR", "/tmp"), "tmp")}
+        os.makedirs(ctx["tmp_dir"], exist_ok=True)
+        for tar, jid in zip(a.tar, publish_overlay_tars(ctx, a.tar)):
+            print("published overlay %s -> %s" % (tar, jid))
+        return 0
     if not a.manifest or not a.repo:
         ap.error("--manifest and --repo are required")
     if a.dry_run:

bits_helpers/preload_bundle.py

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: 2015-2026 CERN
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""CVMFS filebundle spec emitter for the post-publish `bits preload` tool.
+
+The tool straces a trigger binary that already lives on a deployed CVMFS tree,
+so every opened path is already an absolute ``/cvmfs/<repo>.cern.ch/…`` path.
+Turning that into a filebundle spec (per
+https://cvmfs.readthedocs.io/en/stable/cpt-file-bundles/) is therefore just:
+
+  * keep the opens under the repo mount (drop system files, /proc, …);
+  * strip the mount to get repository-root-absolute paths (``/lcg/releases/…``);
+  * emit ``<dir>/.cvmfsbundle-<trigger>`` next to the trigger, listing those
+    paths as ``dependencies`` (the trigger itself excluded).
+
+No relocation and no cross-package resolution: the deployed paths are already
+final. This module is the pure, unit-testable core; the tool driver
+(`preload_cmd`) handles recipe parsing, env setup, strace, tar and publish.
+"""
+
+import json
+import os
+
+
+SPEC_NAME = "CVMFS_BUNDLE"
+SPEC_VERSION = "1.0.0"
+SPEC_ENCODING = "UTF-8"
+
+
+def repo_root_of(cvmfs_path):
+    """The repo mount root of a ``/cvmfs/<repo>/…`` path, i.e. ``/cvmfs/<repo>``.
+
+    E.g. ``/cvmfs/sft.cern.ch/lcg/releases`` -> ``/cvmfs/sft.cern.ch``. Returns
+    None when *cvmfs_path* is not under ``/cvmfs/<repo>/``.
+    """
+    parts = [p for p in (cvmfs_path or "").split("/") if p]
+    if len(parts) >= 2 and parts[0] == "cvmfs":
+        return "/cvmfs/" + parts[1]
+    return None
+
+
+def to_repo_absolute(abs_path, repo_root):
+    """``/cvmfs/<repo>/a/b`` -> ``/a/b``; None if *abs_path* is not under the repo.
+
+    The result is repository-root-absolute (leading slash), exactly the form the
+    filebundle spec's ``dependencies`` want.
+    """
+    root = (repo_root or "").rstrip("/")
+    # Canonicalise: env setup can leave '/./' or '//' in captured paths (e.g. the
+    # BITS_ARCH_PREFIX="." trick used to source a deployed init.sh).
+    abs_path = os.path.normpath(abs_path)
+    if not root or abs_path == root or not abs_path.startswith(root + "/"):
+        return None
+    return abs_path[len(root):]            # keeps the leading '/'
+
+
+def _is_safe_rel(rel):
+    """True if *rel* is a plain, in-tree relative path (no abs, no ``..``, no NUL)."""
+    if not rel or rel.startswith("/") or "\x00" in rel:
+        return False
+    return ".." not in rel.split("/")
+
+
+def render_spec(dependencies):
+    """The versioned filebundle JSON document for *dependencies* (verbatim keys)."""
+    return {
+        "name": SPEC_NAME,
+        "version": SPEC_VERSION,
+        "encoding": SPEC_ENCODING,
+        "dependencies": list(dependencies),
+    }
+
+
+def bundle_path_for(path):
+    """``<dir>/.cvmfsbundle-<base>`` for a trigger path (any form: abs or rel)."""
+    d, base = os.path.split(path)
+    name = ".cvmfsbundle-" + base
+    return (d + "/" + name) if d else name
+
+
+def build_bundle(trigger_abs, opened_abs, repo_root):
+    """Build one bundle from a trigger and the files its launch opened.
+
+    Returns ``(tar_relpath, spec_dict)`` — *tar_relpath* is the bundle's location
+    relative to the repo root (no leading slash), for placement in the staging
+    tar; *spec_dict* is the filebundle JSON. Returns ``(None, None)`` when the
+    trigger is not under the repo or nothing under the repo was opened (no empty
+    bundle). Opens outside the repo (system libs, /proc) and the trigger itself
+    are excluded; the result is sorted and de-duplicated.
+    """
+    trig_rel = to_repo_absolute(trigger_abs, repo_root)
+    if not trig_rel:
+        return None, None
+    deps = set()
+    for p in opened_abs:
+        if p == trigger_abs:
+            continue
+        r = to_repo_absolute(p, repo_root)
+        if r:
+            deps.add(r)
+    if not deps:
+        return None, None
+    bundle_abs = bundle_path_for(trig_rel)     # '/…/bin/.cvmfsbundle-root'
+    return bundle_abs.lstrip("/"), render_spec(sorted(deps))
+
+
+def stage_bundle(staging_dir, tar_relpath, spec):
+    """Write *spec* as JSON to ``<staging_dir>/<tar_relpath>`` (dirs created).
+
+    *tar_relpath* must be a safe in-tree relative path. Returns the file path
+    written. The staging tree mirrors the repo layout so a single tar of it
+    drops each bundle next to its trigger on publish.
+    """
+    if not _is_safe_rel(tar_relpath):
+        raise ValueError("unsafe bundle path: %r" % (tar_relpath,))
+    dest = os.path.join(staging_dir, tar_relpath)
+    os.makedirs(os.path.dirname(dest) or staging_dir, exist_ok=True)
+    with open(dest, "w", encoding="utf-8") as fh:
+        json.dump(spec, fh, indent=2)
+        fh.write("\n")
+    return dest