From 38aaa67e8d2c4c67044a3c6be4252311d99f16ad Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 3 Nov 2026 12:05:12 -0500 Subject: [PATCH] fix(agents): checkpoint large repositories against their origin Save repositories against a verified HTTPS origin when possible, preserving unpushed history, local refs, edits and deletions within the checkpoint budget. Keep full bundle snapshots for repositories that cannot use origin. Restore origin snapshots with prerequisite fetching and safe path replacement. Bound origin discovery to one deadline, preserve version 1 compatibility and validate source types and origin records. Signed-off-by: Tee8z --- voltage/images/buzz-agent/README.md | 17 +- voltage/images/buzz-agent/checkpoint.py | 371 ++++++++++++++++-- .../buzz-agent/test-checkpoint-shallow.py | 27 ++ voltage/images/buzz-agent/test-checkpoint.py | 298 +++++++++++++- 4 files changed, 676 insertions(+), 37 deletions(-) diff --git a/voltage/images/buzz-agent/README.md b/voltage/images/buzz-agent/README.md index aaed3208307..8947ba6bed2 100644 --- a/voltage/images/buzz-agent/README.md +++ b/voltage/images/buzz-agent/README.md @@ -60,13 +60,22 @@ If these UIDs differ, the entrypoint refuses startup. A replacement Pod requires New sessions load the developer's current account profile. Checkpoints use the developer's S3 prefix and service account from Terraform. The harness stops adapter processes before invoking `buzz-agent-checkpoint save`, including cancellation during startup. -Each save records shallow Git boundaries and validates every bundle in an independent repository before uploading the checkpoint. -Restore preserves these boundaries, local branches, tags, and remote-tracking refs without contacting the forge. -For older archives without boundary metadata, restore reconstructs missing parent boundaries and then runs `git fsck`. +Each full snapshot records shallow Git boundaries and validates its bundle in an independent repository before upload. +Full-snapshot restore preserves these boundaries, local branches, tags, and remote-tracking refs without contacting the forge. +For older full snapshots without boundary metadata, restore reconstructs missing parent boundaries and then runs `git fsck`. Missing trees, missing blobs, and other integrity failures stop recovery and remove the staged destination. -The archive format remains version 1. Older images still restore complete histories, but cannot restore shallow bundles. +Full snapshots retain manifest version 1. Older images restore complete histories, but cannot restore shallow bundles. Publish and promote the updated agent image before recovering a session with shallow repositories. A failed save retains the Sandbox for recovery; do not delete its workspace. +A checkpoint holds at most 256 MiB of source and 50,000 files. +A repository whose HTTPS `origin` shares a branch tip with it, as `git ls-remote` reports at save time, is saved against origin: only commits beyond origin's branches, changed and untracked files, deletions, local branches, and tags origin lacks (annotated tag objects by content). +The save asks every origin at once, using the normalized URL and global Git configuration that restore uses, with one 20-second deadline for all of them. +Its restore clones `origin` with the Sandbox's GitHub credential, so pushed content of any size does not count against the budget. +The bundle holds only what the agent owns: HEAD, local branches, the stash, and tags origin lacks, beyond origin's branch tips. +If a branch the checkpoint builds on was deleted or force-pushed after the save, restore fetches its commit from origin by name, which GitHub still serves; restore fails only if origin no longer has the commit. +Any other repository (no remote, an SSH origin, origin unreachable in time, or nothing pushed) is saved whole: full history and every file. +A checkpoint without an origin-based repository keeps manifest version 1, which older images restore. +Origin-based snapshots do not save Git notes (`refs/notes`) or stash entries other than the latest. Explicit checkpoint recovery sets `BUZZ_CHECKPOINT_RESTORE_KEY` in the launch Secret; it is empty otherwise. After the developer profile bootstrap and before the harness starts, the entrypoint runs `buzz-agent-checkpoint restore-once`. diff --git a/voltage/images/buzz-agent/checkpoint.py b/voltage/images/buzz-agent/checkpoint.py index 388eb4ec3b6..a358a572fad 100644 --- a/voltage/images/buzz-agent/checkpoint.py +++ b/voltage/images/buzz-agent/checkpoint.py @@ -2,6 +2,7 @@ """Save and explicitly restore private development workspaces.""" import argparse import base64 +import concurrent.futures import hashlib import json import os @@ -9,6 +10,7 @@ import re import selectors import shutil +import signal import stat import subprocess import sys @@ -20,6 +22,8 @@ MAX_BYTES = 256 * 1024 * 1024 MAX_FILES = 50000 +# All origins are asked at once; this bounds the whole save's network wait. +ORIGIN_SECONDS = 20 EXCLUDED = {'.git', '.aws', '.ssh', '.gnupg', '.config', '.codex', '.claude', '.cache', '.local', '.npm', '.cargo', '.rustup', '.venv', 'venv', 'node_modules', 'target', '__pycache__', 'dist', 'build', '.terraform'} @@ -185,11 +189,13 @@ def public_origin(repo): return urlunsplit((parsed.scheme, host, parsed.path, '', '')) -def scan(directory, *, history=False): - """Refuse a credential the agent added: commits not on any remote, or files. +def scan(directory, *, history=False, revisions=()): + """Refuse a credential the agent added: commits not on a remote, or files. History and files already on a remote are on the forge, where the agent's - token can read them, so the scan leaves them out.""" + token can read them, so the scan leaves them out. `revisions` select an + origin-based save's bundled commits in a few arguments, so the scan covers + exactly what it uploads.""" # Immutable image configuration prevents workspace allowlists from bypassing # the scan. Scanner output can contain secrets even on errors, so suppress it. previous = STEP[0] @@ -200,7 +206,8 @@ def scan(directory, *, history=False): '--gitleaks-ignore-path', '/dev/null', '--ignore-gitleaks-allow', '--no-banner', '--redact', '--timeout', '60'] if history: - command += ['--log-opts=--all HEAD --not --remotes'] + command += ['--log-opts=' + (' '.join(revisions) if revisions + else '--all HEAD --not --remotes')] run(*command, timeout=65, env=env) step(previous) @@ -231,6 +238,86 @@ def write(path, content): write(path, b'\n'.join(added) + b'\n') +def origin_listings(origins): + """`git ls-remote --heads --tags` of each origin, or None where unusable. + + Asks the normalized URL restore clones, with the global Git configuration + restore uses, so a save never relies on a credential restore lacks. Only + HTTPS origins qualify: the Sandbox's GitHub credential serves HTTPS. All + origins are asked at once under one ORIGIN_SECONDS deadline; an origin + that cannot answer in time is saved as a bundle instead.""" + deadline = time.monotonic() + ORIGIN_SECONDS + env = dict(os.environ, GIT_TERMINAL_PROMPT='0') + + def ask(origin): + remaining = deadline - time.monotonic() + if urlsplit(origin).scheme != 'https' or remaining <= 0: + return None + # Its own process group: git's HTTPS helper would otherwise outlive a + # timed-out git and hold the output pipe open. + process = subprocess.Popen(['git', '-c', 'core.hooksPath=/dev/null', 'ls-remote', '--heads', + '--tags', origin], cwd='/', env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=True) + try: + listing, _ = process.communicate(timeout=remaining) + except subprocess.TimeoutExpired: + listing = None + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + return listing if listing is not None and process.returncode == 0 else None + + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + return list(pool.map(ask, origins)) + + +def origin_base(repo, listing): + """What a clone of origin would give this repository. + + Returns `^` rev-list lines for origin's branch tips that this clone + has, and origin's (ref, object) pairs. A clone restores everything + reachable from those tips, so the checkpoint needs only what lies beyond + them. Asks origin, not remote-tracking refs: those can be stale, and after + a bundle restore they name commits origin never had. No tips means restore + cannot build on origin.""" + if listing is None: + return b'', set() + refs = set() + for line in listing.decode().splitlines(): + if '\t' in line: + name, ref = line.split('\t', 1) + refs.add((ref, name)) + heads = sorted({value for ref, value in refs if ref.startswith('refs/heads/')}) + found = git(repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)', + input=''.join(head + '\n' for head in heads).encode()) + tips = b''.join(b'^' + line.split(b' ', 1)[0] + b'\n' + for line in found.splitlines() if line.endswith(b' commit')) + return tips, refs + + +def local_refs(repo, origin_refs): + """Branches, and tags origin lacks, for the bundle and for restore to + recreate exactly. + + Annotated tag objects travel by content: a bundle omits a ref whose target + origin already has, and a tag object is not on origin.""" + refs, tags = [], {} + listing = git(repo, 'for-each-ref', '--format=%(objectname) %(objecttype) %(refname)', + 'refs/heads/', 'refs/tags/') + for line in listing.decode().splitlines(): + value, kind, ref = line.split(' ', 2) + if ref.startswith('refs/tags/') and (ref, value) in origin_refs: + continue + refs.append([ref, value]) + if kind == 'tag': + tags[value] = base64.b64encode(git(repo, 'cat-file', 'tag', value)).decode() + return refs, tags + + def pack(workspace, output): workspace = workspace.resolve(strict=True) manifest = {'version': 1, 'repositories': []} @@ -241,37 +328,89 @@ def pack(workspace, output): stage = Path(temp) # What the agent added to each repository, for the secret scan. review = Path(review_temp) - for index, repo in enumerate(repositories(workspace)): + repos = repositories(workspace) + if any(repo == workspace for repo in repos): + raise ValueError('Use a workspace directory containing repositories') + origins = [public_origin(repo) for repo in repos] + listings = origin_listings(origins) + for index, (repo, origin, listing) in enumerate(zip(repos, origins, listings)): relative = repo.relative_to(workspace).as_posix() - if relative == '.': - raise ValueError('Use a workspace directory containing repositories') - record = {'path': relative, 'origin': public_origin(repo), + tips, origin_refs = origin_base(repo, listing) + base = 'origin' if tips else 'bundle' + record = {'path': relative, 'origin': origin, 'base': base, 'head': git(repo, 'rev-parse', 'HEAD').decode().strip(), 'branch': git(repo, 'branch', '--show-current').decode().strip(), 'shallow': shallow_commits(repo), - 'excluded': [], 'files': []} + 'excluded': [], 'files': [], 'deleted': []} + if base == 'origin': + # Older images refuse version 2, so it marks only what needs it. + manifest['version'] = 2 + record['refs'], record['tags'] = local_refs(repo, origin_refs) + total += sum(len(content) for content in record['tags'].values()) destination = stage / 'repositories' / str(index) (destination / 'files').mkdir(parents=True) - scan(repo, history=True) + beyond = [] + if base == 'origin': + # Only what the agent owns, beyond origin's branches: never + # origin's remote-tracking refs or tags, whose boundaries would + # sit on branches origin may delete before the restore. A ref at + # or behind an origin tip adds no commit. + owned = ['HEAD'] + [ref for ref, _ in record['refs']] + if git(repo, 'for-each-ref', '--format=%(refname)', 'refs/stash').strip(): + owned.append('refs/stash') + beyond = [name for name in owned if git( + repo, 'rev-list', '--max-count=1', '--stdin', input=(name + '\n').encode() + tips).strip()] + revisions = ''.join(name + '\n' for name in beyond).encode() + tips + if beyond: + # The same commits as the bundle, bounded by their boundary + # rather than by every origin tip, which can exceed one + # argument's size limit. + boundary = [line[1:] for line in git(repo, 'rev-list', '--boundary', '--stdin', + input=revisions).decode().splitlines() + if line.startswith('-')] + scan(repo, history=True, revisions=beyond + ['^' + value for value in boundary]) + else: + scan(repo, history=True) review_changes(repo, review / str(index)) - git(repo, 'bundle', 'create', str(destination / 'history.bundle'), '--all', 'HEAD') - total += (destination / 'history.bundle').stat().st_size + bundle = destination / 'history.bundle' + if base == 'bundle': + # The whole history, and every file: restore needs nothing else. + git(repo, 'bundle', 'create', str(bundle), '--all', 'HEAD') + names = git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + else: + # Only commits origin lacks, and only files that differ from HEAD. + # git refuses an empty bundle, so a fully pushed repository has none. + if beyond: + git(repo, 'bundle', 'create', str(bundle), '--stdin', input=revisions) + names = (git(repo, 'ls-files', '-z', '--others', '--exclude-standard') + b'\0' + + git(repo, 'diff', '--name-only', '--no-renames', '-z', 'HEAD')) + if bundle.exists(): + total += bundle.stat().st_size if total > MAX_BYTES: raise ValueError('Checkpoint history exceeds the 256 MiB source budget') - # Verify in an independent object store, just as a future restore - # will, before advertising or uploading this checkpoint. - with tempfile.TemporaryDirectory(prefix='agent-checkpoint-history-') as history_temp: - restore_history(destination / 'history.bundle', record, Path(history_temp) / 'repo') - names = git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + if base == 'bundle': + # Preserve independent validation of self-contained bundles. + # Origin deltas depend on prerequisites supplied by the forge. + with tempfile.TemporaryDirectory(prefix='agent-checkpoint-history-') as history_temp: + restore_history(bundle, record, Path(history_temp) / 'repo') for raw in sorted(set(names.split(b'\0')) - {b''}): path = safe_path(os.fsdecode(raw)) if excluded(path): record['excluded'].append(str(path)) continue source = repo / path + if any((repo / PurePosixPath(*path.parts[:depth])).is_symlink() + for depth in range(1, len(path.parts))): + # A directory replaced by a symlink: the symlink is its own + # entry, and nothing is read through it. + if base == 'origin': + record['deleted'].append(str(path)) + continue if not source.exists() and not source.is_symlink(): - continue # A tracked deletion is represented by absence. - if source.is_dir(): + if base == 'origin': + record['deleted'].append(str(path)) + continue # A bundle snapshot represents a deletion by absence. + if source.is_dir() and not source.is_symlink(): continue # Submodule repositories are discovered separately. info = source.lstat() target = destination / 'files' / path @@ -329,7 +468,7 @@ def restore(archive_path, destination): archive.extractall(temp, filter='data') stage = Path(temp) / 'snapshot' manifest = json.loads((stage / 'manifest.json').read_text()) - if manifest.get('version') != 1: + if manifest.get('version') not in (1, 2): raise ValueError('Unsupported checkpoint version') destination.mkdir(mode=0o700, parents=True) try: @@ -352,25 +491,193 @@ def restore_repositories(stage, manifest, destination): raise ValueError('Checkpoint repository path is reserved') source = stage / 'repositories' / str(index) repo = destination / relative - restore_history(source / 'history.bundle', record, repo) - git(repo, 'checkout', '--detach', record['head']) + if not re.fullmatch('[0-9a-f]{40,64}', record['head']): + raise ValueError('Invalid checkpoint revision') + # Version 1 checkpoints are all bundle snapshots. + base = record.get('base', 'bundle') + if base == 'bundle': + restore_history(source / 'history.bundle', record, repo) + elif base == 'origin': + validate_origin_record(record) + clone_origin(record['origin'], repo) + if (source / 'history.bundle').exists(): + fetch_bundle(repo, source / 'history.bundle') + # HEAD or a saved ref can sit on a branch origin deleted after + # the save, such as a pushed branch removed by a squash merge. + fetch_missing(repo, [record['head']] + [value for _, value in record['refs'] + if value not in record['tags']] + + [tag_target(content) for content in record['tags'].values()]) + restore_refs(repo, record) + else: + raise ValueError('Unsupported checkpoint repository base') + # Checking out a large tree can take minutes. + git(repo, 'checkout', '--detach', record['head'], timeout=600) if record['branch']: git(repo, 'check-ref-format', '--branch', record['branch']) - git(repo, 'checkout', '-B', record['branch']) + git(repo, 'checkout', '-B', record['branch'], timeout=600) git(repo, 'remote', 'set-url', 'origin', record['origin']) - # Absence from the snapshot means deletion, except for excluded paths: - # they were never saved, so they keep their committed content. - for raw in git(repo, 'ls-files', '-z').split(b'\0'): - if raw: - path = safe_path(os.fsdecode(raw)) - target = repo / path - if not excluded(path) and (target.is_file() or target.is_symlink()): - target.unlink() - shutil.copytree(source / 'files', repo, dirs_exist_ok=True, symlinks=True) + if base == 'bundle': + # Absence from the snapshot means deletion, except for excluded + # paths: they were never saved, so they keep their committed content. + deleted = [os.fsdecode(raw) for raw in git(repo, 'ls-files', '-z').split(b'\0') if raw] + else: + deleted = record.get('deleted', []) + for value in deleted: + path = safe_path(value) + if not excluded(path): + remove(repo, path) + for value in record['files']: + path = safe_path(value) + if excluded(path): + raise ValueError('Checkpoint file path is reserved') + place(repo, source / 'files', path) review_changes(repo, review / str(index)) scan(review) +def clone_origin(origin, repo): + """Clone a checkpoint's pushed base with the Sandbox's Git credentials. + + A large repository can clone slowly but steadily, so there is no overall + deadline; Git fails the clone only when the transfer stalls.""" + if urlsplit(origin).scheme != 'https': + raise ValueError('Checkpoint origin base requires an HTTPS origin') + run('git', '-c', 'core.hooksPath=/dev/null', '-c', 'http.lowSpeedLimit=1000', + '-c', 'http.lowSpeedTime=60', 'clone', '--no-checkout', '--', origin, str(repo), + timeout=None, env=dict(os.environ, GIT_TERMINAL_PROMPT='0')) + + +def remove(repo, path): + """Delete the file or symlink at path, never through a symlink or file. + + Git tracks nothing beyond a symlink, so a parent that is not a real + directory means the path is already absent.""" + target = repo + for part in path.parts[:-1]: + target = target / part + if target.is_symlink() or not target.is_dir(): + return + target = target / path.parts[-1] + if target.is_symlink() or target.is_file(): + target.unlink() + + +def place(repo, files, path): + """Put the saved file or symlink at path, replacing whatever is there. + + The checkout can hold another type at the path or a parent: a symlink the + agent retargeted or replaced, a file it turned into a directory, or the + reverse. Replacing, not writing through, keeps every write inside repo.""" + target = repo + for part in path.parts[:-1]: + target = target / part + if target.is_symlink() or (target.exists() and not target.is_dir()): + target.unlink() + target.mkdir(exist_ok=True) + target = target / path.parts[-1] + if target.is_symlink() or target.is_file(): + target.unlink() + elif target.is_dir(): + shutil.rmtree(target) + saved = files / path + if saved.is_symlink(): + target.symlink_to(os.readlink(saved)) + elif saved.is_file(): + shutil.copy2(saved, target, follow_symlinks=False) + else: + raise ValueError('Checkpoint file is missing from the archive') + + +def validate_origin_record(record): + """Refuse a malformed origin record before acting on it.""" + if not (isinstance(record.get('origin'), str) and isinstance(record.get('files'), list) + and isinstance(record.get('deleted', []), list) and isinstance(record.get('refs', []), list) + and isinstance(record.get('tags', {}), dict) + and all(isinstance(value, str) for value in record['files'] + record.get('deleted', [])) + and all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(v, str) for v in pair) + for pair in record.get('refs', [])) + and all(isinstance(key, str) and isinstance(value, str) + for key, value in record.get('tags', {}).items())): + raise ValueError('Invalid checkpoint repository record') + record.setdefault('refs', []) + record.setdefault('tags', {}) + + +def tag_target(content): + """The object an annotated tag names, from its saved content.""" + first = base64.b64decode(content, validate=True).split(b'\n', 1)[0].decode() + if not re.fullmatch('object [0-9a-f]{40,64}', first): + raise ValueError('Invalid checkpoint tag object') + return first.split(' ', 1)[1] + + +def fetch_missing(repo, values): + """Fetch the named commits the clone lacks from origin. + + GitHub still serves a commit by name after its branch is deleted or + force-pushed; a commit origin no longer has fails the restore here.""" + for value in values: + if not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint revision') + found = git(repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)', + input=''.join(value + '\n' for value in sorted(set(values))).encode()) + missing = [line.split(' ', 1)[0] for line in found.decode().splitlines() + if line.endswith(' missing')] + if missing: + git(repo, '-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=60', 'fetch', + '--no-tags', '--no-write-fetch-head', 'origin', *missing, timeout=None, + env=dict(os.environ, GIT_TERMINAL_PROMPT='0')) + + +def restore_refs(repo, record): + """Recreate the saved branches and tags, annotated tag objects first.""" + for name, content in record.get('tags', {}).items(): + made = git(repo, 'hash-object', '-t', 'tag', '-w', '--stdin', + input=base64.b64decode(content, validate=True)).decode().strip() + if made != name: + raise ValueError('Checkpoint tag object does not match its name') + for ref, value in record.get('refs', []): + if not ref.startswith(('refs/heads/', 'refs/tags/')) or not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint ref') + git(repo, 'check-ref-format', ref) + git(repo, 'update-ref', ref, value) + + +def prerequisites(bundle): + """The commits a bundle builds on, from its header.""" + with open(bundle, 'rb') as stream: + header = stream.read(1024 * 1024) + if b'\n\n' not in header: + raise ValueError('Checkpoint bundle header is too large') + lines = header.split(b'\n\n', 1)[0].split(b'\n') + if lines[0] not in (b'# v2 git bundle', b'# v3 git bundle'): + raise ValueError('Unsupported checkpoint bundle') + found = [] + for line in lines[1:]: + if line.startswith(b'-'): + value = line[1:].split(b' ', 1)[0].decode() + if not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint bundle prerequisite') + found.append(value) + return found + + +def fetch_bundle(repo, bundle): + """Add a checkpoint's unpushed commits, and their refs, to an origin clone.""" + # A prerequisite can leave origin's branches after the save. + fetch_missing(repo, prerequisites(bundle)) + specs = [] + for line in git(repo, 'bundle', 'list-heads', str(bundle)).decode().splitlines(): + name = line.split(' ', 1)[1] if ' ' in line else '' + if name == 'HEAD': + specs.append('HEAD') # A detached HEAD commit is not on any branch. + else: + git(repo, 'check-ref-format', name) + specs.append('+' + name + ':' + name) + # The bundle's prerequisites are origin commits; a missing one fails here. + git(repo, 'fetch', '--update-head-ok', '--no-tags', str(bundle), *specs) + + def storage(): bucket = os.environ.get('BUZZ_CHECKPOINT_BUCKET', '') prefix = os.environ.get('BUZZ_CHECKPOINT_PREFIX', '') diff --git a/voltage/images/buzz-agent/test-checkpoint-shallow.py b/voltage/images/buzz-agent/test-checkpoint-shallow.py index 7855deac994..5a738255bdd 100644 --- a/voltage/images/buzz-agent/test-checkpoint-shallow.py +++ b/voltage/images/buzz-agent/test-checkpoint-shallow.py @@ -135,6 +135,33 @@ def damaged_bundle(repo, *args, **kwargs): checkpoint.git = original_git print('PASS: save validates its bundle in an independent repository before producing an archive') + # Exercise the integration with origin-based snapshots too. The URL rewrite + # supplies a local forge; the container still has no network access. + git(root, 'config', '--global', 'url.'+upstream.as_uri()+'.insteadOf', + 'https://github.com/example/fixture.git') + origin_archive = root/'origin.tar.gz' + origin_manifest = checkpoint.pack(workspace, origin_archive) + assert origin_manifest['version'] == 2 + assert origin_manifest['repositories'][0]['base'] == 'origin' + origin_destination = root/'restored-origin' + checkpoint.restore(origin_archive, origin_destination) + origin_repo = origin_destination/'example' + assert git(origin_repo, 'rev-parse', 'HEAD').decode().strip() == head + assert git(origin_repo, 'status', '--porcelain') == original_status + assert (origin_repo/'source.txt').read_text() == 'working edit\n' + assert git(origin_repo, 'rev-parse', '--is-shallow-repository').strip() == b'false' + git(origin_repo, 'fsck', '--full') + # If the origin cannot be read, the same shallow checkout must still use + # the independent bundle validation and recovery from #99. + git(root, 'config', '--global', '--unset-all', + 'url.'+upstream.as_uri()+'.insteadOf') + fallback_archive = root/'fallback.tar.gz' + fallback_manifest = checkpoint.pack(workspace, fallback_archive) + assert fallback_manifest['version'] == 1 + assert fallback_manifest['repositories'][0]['base'] == 'bundle' + verify(fallback_archive, 'restored-fallback') + print('PASS: shallow origin snapshots restore full history; an unavailable origin falls back to a verified shallow bundle') + # A damaged archive can produce large fsck diagnostics. Exercise the real # child process boundary and verify it is reaped when the output is capped. tools = root/'tools' diff --git a/voltage/images/buzz-agent/test-checkpoint.py b/voltage/images/buzz-agent/test-checkpoint.py index c3e18dd4deb..2dabb553e6a 100644 --- a/voltage/images/buzz-agent/test-checkpoint.py +++ b/voltage/images/buzz-agent/test-checkpoint.py @@ -5,7 +5,7 @@ SCRIPT = r''' import base64, hashlib, importlib.machinery, importlib.util, io, json, os from pathlib import Path -import subprocess, tarfile, tempfile +import shutil, socket, subprocess, tarfile, tempfile, threading, time loader = importlib.machinery.SourceFileLoader('checkpoint', '/usr/local/bin/buzz-agent-checkpoint') spec = importlib.util.spec_from_loader(loader.name, loader) @@ -72,6 +72,22 @@ def git(repo, *args): raise AssertionError('restore overwrote an existing workspace') except ValueError: pass + # Without an origin base the manifest stays version 1, which older images + # restore; archives written before origin bases still restore too. + assert manifest['version'] == 1 and manifest['repositories'][0]['base'] == 'bundle' + legacy = root/'legacy' + with tarfile.open(artifact) as archive: + archive.extractall(legacy, filter='data') + old_manifest = json.loads((legacy/'snapshot/manifest.json').read_text()) + for record in old_manifest['repositories']: + del record['base'], record['deleted'] + (legacy/'snapshot/manifest.json').write_text(json.dumps(old_manifest)) + with tarfile.open(root/'legacy.tar.gz','w:gz') as archive: + archive.add(legacy/'snapshot',arcname='snapshot') + checkpoint.restore(root/'legacy.tar.gz',root/'restored-legacy') + assert git(root/'restored-legacy/example','rev-parse','HEAD') == head + assert (root/'restored-legacy/example/new.rs').read_bytes() == (repo/'new.rs').read_bytes() + assert not (root/'restored-legacy/example/deleted.txt').exists() print('PASS: SSH origin identity/port, local commit, edited/untracked source, deletion, and symlink survive restore; account files and build outputs excluded; excluded tracked files keep committed content') (repo/'leak.txt').write_text('github_token = "ghp_'+'aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6'+'"\n') @@ -98,11 +114,16 @@ def git(repo, *args): git(upstream,'config','user.name','Checkpoint Fixture') git(upstream,'config','user.email','fixture@example.invalid') (upstream/'values.yaml').write_text(token) + (upstream/'large.bin').write_bytes(bytes(1024*1024)) + (upstream/'remove.txt').write_text('pushed, then deleted locally\n') git(upstream,'add','.') git(upstream,'commit','-m','chore: commit a credential upstream') clone = workspace/'clone' subprocess.check_call(['git','clone','-q',str(upstream),str(clone)],stderr=subprocess.DEVNULL) git(clone,'remote','set-url','origin','https://github.com/example/upstream.git') + # Origin-based restore clones origin; offline, serve it from the fixture. + os.environ['GIT_CONFIG_GLOBAL'] = str(root/'gitconfig') + git(root,'config','--global','url.'+str(upstream)+'.insteadOf','https://github.com/example/upstream.git') git(clone,'config','user.name','Checkpoint Fixture') git(clone,'config','user.email','fixture@example.invalid') checkpoint.pack(workspace,root/'upstream.tar.gz') @@ -148,6 +169,281 @@ def git(repo, *args): checkpoint.pack(workspace,root/'reset.tar.gz') print('PASS: credentials already on the remote are allowed and restore; one added in a file, an unpushed commit or an archive is refused with the scan step') + # With an HTTPS origin, a checkpoint holds only what origin lacks, so + # pushed content larger than the budget does not count against it. + git(clone,'checkout','-q','-b','feature') + (clone/'local.txt').write_text('unpushed commit\n') + git(clone,'add','local.txt') + git(clone,'commit','-m','feat: add an unpushed commit') + # A remote-tracking ref origin does not have, as a bundle restore leaves: + # the save asks origin, so the commit stays in the checkpoint. + git(clone,'update-ref','refs/remotes/origin/feature','HEAD') + (clone/'remove.txt').unlink() + (clone/'untracked.txt').write_text('untracked\n') + budget = checkpoint.MAX_BYTES + checkpoint.MAX_BYTES = 512*1024 + try: + thin = checkpoint.pack(workspace,root/'thin.tar.gz') + # With an SSH origin the same workspace is a bundle snapshot, which + # counts the pushed tree and exceeds the budget. + git(clone,'remote','set-url','origin','ssh://git@example.com/upstream.git') + try: + checkpoint.pack(workspace,root/'ssh.tar.gz') + raise AssertionError('an SSH origin repository skipped its bundle snapshot') + except ValueError as error: + assert 'budget' in str(error), error + git(clone,'remote','set-url','origin','https://github.com/example/upstream.git') + finally: + checkpoint.MAX_BYTES = budget + record = next(r for r in thin['repositories'] if r['path'] == 'clone') + assert record['base'] == 'origin' and record['deleted'] == ['remove.txt'], record + # The unpushed commit travels in the bundle, not as a changed file. + assert record['files'] == ['untracked.txt'], record['files'] + with tarfile.open(root/'thin.tar.gz') as archive: + assert not any(n.endswith('/large.bin') for n in archive.getnames()) + checkpoint.restore(root/'thin.tar.gz',root/'restored-thin') + restored = root/'restored-thin/clone' + assert git(restored,'rev-parse','HEAD') == git(clone,'rev-parse','HEAD') + assert git(restored,'branch','--show-current') == 'feature' + assert git(restored,'config','--get','remote.origin.url') == 'https://github.com/example/upstream.git' + assert (restored/'large.bin').stat().st_size == 1024*1024 + assert (restored/'local.txt').read_text() == 'unpushed commit\n' + assert (restored/'untracked.txt').read_text() == 'untracked\n' + assert not (restored/'remove.txt').exists() + print('PASS: an HTTPS origin repository saves only unpushed commits and changed files, within budget, and restores from origin; an SSH origin keeps the full bundle') + + # Origin-based restore replaces each saved path over the origin checkout, + # whatever type the agent changed it from, and recreates refs origin lacks. + types_upstream = root/'types-upstream' + types_upstream.mkdir() + git(types_upstream,'init','-b','main') + git(types_upstream,'config','user.name','Checkpoint Fixture') + git(types_upstream,'config','user.email','fixture@example.invalid') + for name in ('a.txt','b.txt','c.txt'): + (types_upstream/name).write_text(name+'\n') + (types_upstream/'d').mkdir() + (types_upstream/'d/x.txt').write_text('in a directory\n') + for name in ('linked','other'): + (types_upstream/name).mkdir() + (types_upstream/name/'f.txt').write_text(name+'\n') + (types_upstream/'retarget').symlink_to('a.txt') + (types_upstream/'to-file').symlink_to('a.txt') + git(types_upstream,'add','.') + git(types_upstream,'commit','-m','test: create typed paths') + types = workspace/'types' + subprocess.check_call(['git','clone','-q',str(types_upstream),str(types)],stderr=subprocess.DEVNULL) + git(types,'remote','set-url','origin','https://github.com/example/types.git') + git(root,'config','--global','url.'+str(types_upstream)+'.insteadOf','https://github.com/example/types.git') + git(types,'config','user.name','Checkpoint Fixture') + git(types,'config','user.email','fixture@example.invalid') + (types/'retarget').unlink() + (types/'retarget').symlink_to('b.txt') + (types/'to-file').unlink() + (types/'to-file').write_text('now a regular file\n') + (types/'c.txt').unlink() + (types/'c.txt').mkdir() + (types/'c.txt/inner.txt').write_text('file became a directory\n') + (types/'d/x.txt').unlink() + (types/'d').rmdir() + (types/'d').write_text('directory became a file\n') + (types/'linked/f.txt').unlink() + (types/'linked').rmdir() + (types/'linked').symlink_to('other') + git(types,'tag','light') + git(types,'-c','tag.gpgsign=false','tag','-a','-m','annotated, unpushed','annotated') + git(types,'branch','wip') + typed = checkpoint.pack(workspace,root/'types.tar.gz') + assert typed['version'] == 2 + assert next(r for r in typed['repositories'] if r['path'] == 'types')['base'] == 'origin' + checkpoint.restore(root/'types.tar.gz',root/'restored-types') + restored = root/'restored-types/types' + assert os.readlink(restored/'retarget') == 'b.txt' + assert not (restored/'to-file').is_symlink() + assert (restored/'to-file').read_text() == 'now a regular file\n' + assert (restored/'a.txt').read_text() == 'a.txt\n', 'a write went through the old symlink' + assert (restored/'c.txt/inner.txt').read_text() == 'file became a directory\n' + assert (restored/'d').read_text() == 'directory became a file\n' + assert os.readlink(restored/'linked') == 'other' + assert (restored/'other/f.txt').read_text() == 'other\n' + types_record = next(r for r in typed['repositories'] if r['path'] == 'types') + assert 'linked/f.txt' in types_record['deleted'] and 'linked/f.txt' not in types_record['files'] + assert git(restored,'status','--porcelain') == git(types,'status','--porcelain') + for ref in ('refs/tags/light','refs/tags/annotated','refs/heads/wip'): + assert git(restored,'rev-parse',ref) == git(types,'rev-parse',ref), ref + assert git(restored,'cat-file','-t','annotated') == 'tag' + assert git(restored,'cat-file','tag','annotated') == git(types,'cat-file','tag','annotated') + # An annotated tag message the agent adds is secret-scanned: the manifest + # carries it base64-encoded, and the manifest scan decodes base64. + git(types,'-c','tag.gpgsign=false','tag','-a','-m','github_token = "ghp_'+'aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6'+'"','leaky') + try: + checkpoint.pack(workspace,root/'leaky.tar.gz') + raise AssertionError('secret scanner admitted a credential in an annotated tag message') + except ValueError: + assert checkpoint.STEP[0] == 'scanning for secrets', checkpoint.STEP[0] + git(types,'tag','-d','leaky') + # A crafted deletion cannot reach through a symlinked parent. + forged = root/'forged-types' + with tarfile.open(root/'types.tar.gz') as archive: + archive.extractall(forged, filter='data') + forged_manifest = json.loads((forged/'snapshot/manifest.json').read_text()) + (root/'outside.txt').write_text('outside\n') + git(types_upstream,'rm','-rq','d') + (types_upstream/'escape').symlink_to(root) + git(types_upstream,'add','escape') + git(types_upstream,'commit','-m','test: add a symlink out of the repository') + record = next(r for r in forged_manifest['repositories'] if r['path'] == 'types') + record['head'] = git(types_upstream,'rev-parse','HEAD') + record['branch'] = '' + record['files'], record['refs'], record['tags'] = [], [], {} + record['deleted'] = ['escape/outside.txt'] + (forged/'snapshot/manifest.json').write_text(json.dumps(forged_manifest)) + with tarfile.open(root/'forged-types.tar.gz','w:gz') as archive: + archive.add(forged/'snapshot',arcname='snapshot') + checkpoint.restore(root/'forged-types.tar.gz',root/'restored-forged-types') + assert (root/'outside.txt').read_text() == 'outside\n' + # A malformed record is refused as a checkpoint error, not a crash. + record['refs'] = [['refs/heads/x', 2]] + (forged/'snapshot/manifest.json').write_text(json.dumps(forged_manifest)) + with tarfile.open(root/'malformed.tar.gz','w:gz') as archive: + archive.add(forged/'snapshot',arcname='snapshot') + try: + checkpoint.restore(root/'malformed.tar.gz',root/'restored-malformed') + raise AssertionError('a malformed origin record was restored') + except ValueError: + pass + assert not (root/'restored-malformed').exists() + print('PASS: origin restore replaces retargeted symlinks, symlinks turned files, and file/directory swaps in place; unpushed tags, annotated tag objects and branches survive; deletions never follow a symlink') + + # The bundle holds only what the agent owns, never origin's tags or + # remote-tracking refs. Restore fetches a prerequisite origin stopped + # advertising: here the branch the agent built on is deleted after the save. + topic_upstream = root/'topic-upstream' + topic_upstream.mkdir() + git(topic_upstream,'init','-b','main') + git(topic_upstream,'config','user.name','Checkpoint Fixture') + git(topic_upstream,'config','user.email','fixture@example.invalid') + # GitHub serves any commit in the repository by name. + git(topic_upstream,'config','uploadpack.allowAnySHA1InWant','true') + (topic_upstream/'base.txt').write_text('base\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: base') + git(topic_upstream,'tag','v1') + # A release tag on a commit no branch reaches, as bifrost's releases are. + git(topic_upstream,'checkout','-q','-b','release') + (topic_upstream/'release.txt').write_text('release\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: release') + git(topic_upstream,'tag','v2') + git(topic_upstream,'checkout','-q','main') + git(topic_upstream,'branch','-D','release') + git(topic_upstream,'checkout','-q','-b','topic') + (topic_upstream/'topic.txt').write_text('topic\n') + git(topic_upstream,'add','.') + # A prerequisite header line longer than one 256-byte read, with '-' at + # the split point. + git(topic_upstream,'commit','-m','test: '+'a'*208+'-topic') + git(topic_upstream,'checkout','-q','-b','pushed') + (topic_upstream/'pushed.txt').write_text('pushed\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: pushed') + git(topic_upstream,'checkout','-q','topic') + git(topic_upstream,'checkout','-q','main') + topic = workspace/'topic' + subprocess.check_call(['git','clone','-q',str(topic_upstream),str(topic)],stderr=subprocess.DEVNULL) + git(topic,'remote','set-url','origin','https://github.com/example/topic.git') + # file:// clones over the fetch protocol, so a restore clone lacks commits + # no ref reaches; a plain path would copy the whole object store. + git(root,'config','--global','url.file://'+str(topic_upstream)+'.insteadOf','https://github.com/example/topic.git') + git(topic,'config','user.name','Checkpoint Fixture') + git(topic,'config','user.email','fixture@example.invalid') + git(topic,'checkout','-q','-b','agent','origin/topic') + (topic/'agent.txt').write_text('built on topic\n') + git(topic,'add','agent.txt') + git(topic,'commit','-m','test: build on topic') + # A pushed branch with nothing unpushed, and a local tag on it, whose + # branch origin deletes after the save (as a squash merge does). + pushed = workspace/'pushed' + subprocess.check_call(['git','clone','-q','file://'+str(topic_upstream),str(pushed)],stderr=subprocess.DEVNULL) + git(pushed,'remote','set-url','origin','https://github.com/example/topic.git') + git(pushed,'checkout','-q','-b','pushed','origin/pushed') + git(pushed,'tag','mine') + saved = checkpoint.pack(workspace,root/'topic.tar.gz') + pushed_index = [r['path'] for r in saved['repositories']].index('pushed') + index = [r['path'] for r in saved['repositories']].index('topic') + unpacked = root/'topic-unpacked' + with tarfile.open(root/'topic.tar.gz') as archive: + archive.extractall(unpacked, filter='data') + heads = git(topic,'bundle','list-heads',str(unpacked/'snapshot/repositories'/str(index)/'history.bundle')) + assert git(topic,'rev-parse','-q','--verify','refs/tags/v2'), 'fixture clone lacks the release tag' + assert 'refs/remotes/' not in heads and 'refs/tags/' not in heads, heads + git(topic_upstream,'branch','-D','topic') + git(topic_upstream,'branch','-D','pushed') + assert not (unpacked/'snapshot/repositories'/str(pushed_index)/'history.bundle').exists() + lost = git(topic,'rev-parse','origin/topic') + probe = root/'topic-probe' + git(root,'clone','-q','--no-checkout','https://github.com/example/topic.git',str(probe)) + assert subprocess.run(['git','-C',str(probe),'cat-file','-e',lost]).returncode != 0, 'the clone still has the deleted branch' + checkpoint.restore(root/'topic.tar.gz',root/'restored-topic') + restored = root/'restored-topic/topic' + assert git(restored,'rev-parse','HEAD') == git(topic,'rev-parse','HEAD') + assert (restored/'topic.txt').read_text() == 'topic\n' + assert (restored/'agent.txt').read_text() == 'built on topic\n' + restored_pushed = root/'restored-topic/pushed' + assert git(restored_pushed,'rev-parse','HEAD') == git(pushed,'rev-parse','HEAD') + assert git(restored_pushed,'branch','--show-current') == 'pushed' + assert git(restored_pushed,'rev-parse','mine') == git(pushed,'rev-parse','mine') + assert (restored_pushed/'pushed.txt').read_text() == 'pushed\n' + print('PASS: the bundle excludes origin tags and remote-tracking refs; restore fetches a prerequisite, HEAD and a tag whose branches origin deleted after the save') + + # Thousands of origin branches do not grow any single scan argument. + many_upstream = root/'many-upstream' + many_upstream.mkdir() + git(many_upstream,'init','-b','main') + git(many_upstream,'config','user.name','Checkpoint Fixture') + git(many_upstream,'config','user.email','fixture@example.invalid') + (many_upstream/'base.txt').write_text('base\n') + git(many_upstream,'add','.') + git(many_upstream,'commit','-m','test: base') + stream = [] + for number in range(4000): + message = 'branch %d' % number + stream.append('commit refs/heads/b%d\ncommitter F 0 +0000\ndata %d\n%s\nfrom refs/heads/main\n\n' + % (number, len(message), message)) + subprocess.run(['git','-C',str(many_upstream),'fast-import','--quiet'],input=''.join(stream).encode(),check=True) + many = workspace/'many' + subprocess.check_call(['git','clone','-q',str(many_upstream),str(many)],stderr=subprocess.DEVNULL) + git(many,'remote','set-url','origin','https://github.com/example/many.git') + git(root,'config','--global','url.'+str(many_upstream)+'.insteadOf','https://github.com/example/many.git') + git(many,'config','user.name','Checkpoint Fixture') + git(many,'config','user.email','fixture@example.invalid') + (many/'agent.txt').write_text('unpushed\n') + git(many,'add','agent.txt') + git(many,'commit','-m','test: unpushed') + assert len(set(git(many,'for-each-ref','--format=%(objectname)','refs/remotes/').split())) > 3200 + crowded = checkpoint.pack(workspace,root/'many.tar.gz') + assert next(r for r in crowded['repositories'] if r['path'] == 'many')['base'] == 'origin' + shutil.rmtree(many) + print('PASS: an origin with 4,000 distinct branch tips packs; the history scan stays within argument limits') + + # Unreachable origins cost one shared deadline, not one per repository. + server = socket.socket() + server.bind(('127.0.0.1', 0)) + server.listen() + held = [] + threading.Thread(target=lambda: [held.append(server.accept()) for _ in range(8)], daemon=True).start() + hanging = 'https://127.0.0.1:%d/example.git' % server.getsockname()[1] + deadline = checkpoint.ORIGIN_SECONDS + checkpoint.ORIGIN_SECONDS = 2 + try: + started = time.monotonic() + listings = checkpoint.origin_listings([hanging]*4 + ['ssh://git@example.com/x.git']) + elapsed = time.monotonic() - started + finally: + checkpoint.ORIGIN_SECONDS = deadline + assert listings == [None]*5, listings + assert elapsed < 5, elapsed + print('PASS: four hanging origins and an SSH origin resolve to bundle saves within one shared deadline') + (root/'private').write_text('outside workspace') (repo/'escape').symlink_to(root/'private') try: