diff --git a/voltage/images/buzz-agent/README.md b/voltage/images/buzz-agent/README.md index aaed3208307..8947ba6bed2 100644 --- a/voltage/images/buzz-agent/README.md +++ b/voltage/images/buzz-agent/README.md @@ -60,13 +60,22 @@ If these UIDs differ, the entrypoint refuses startup. A replacement Pod requires New sessions load the developer's current account profile. Checkpoints use the developer's S3 prefix and service account from Terraform. The harness stops adapter processes before invoking `buzz-agent-checkpoint save`, including cancellation during startup. -Each save records shallow Git boundaries and validates every bundle in an independent repository before uploading the checkpoint. -Restore preserves these boundaries, local branches, tags, and remote-tracking refs without contacting the forge. -For older archives without boundary metadata, restore reconstructs missing parent boundaries and then runs `git fsck`. +Each full snapshot records shallow Git boundaries and validates its bundle in an independent repository before upload. +Full-snapshot restore preserves these boundaries, local branches, tags, and remote-tracking refs without contacting the forge. +For older full snapshots without boundary metadata, restore reconstructs missing parent boundaries and then runs `git fsck`. Missing trees, missing blobs, and other integrity failures stop recovery and remove the staged destination. -The archive format remains version 1. Older images still restore complete histories, but cannot restore shallow bundles. +Full snapshots retain manifest version 1. Older images restore complete histories, but cannot restore shallow bundles. Publish and promote the updated agent image before recovering a session with shallow repositories. A failed save retains the Sandbox for recovery; do not delete its workspace. +A checkpoint holds at most 256 MiB of source and 50,000 files. +A repository whose HTTPS `origin` shares a branch tip with it, as `git ls-remote` reports at save time, is saved against origin: only commits beyond origin's branches, changed and untracked files, deletions, local branches, and tags origin lacks (annotated tag objects by content). +The save asks every origin at once, using the normalized URL and global Git configuration that restore uses, with one 20-second deadline for all of them. +Its restore clones `origin` with the Sandbox's GitHub credential, so pushed content of any size does not count against the budget. +The bundle holds only what the agent owns: HEAD, local branches, the stash, and tags origin lacks, beyond origin's branch tips. +If a branch the checkpoint builds on was deleted or force-pushed after the save, restore fetches its commit from origin by name, which GitHub still serves; restore fails only if origin no longer has the commit. +Any other repository (no remote, an SSH origin, origin unreachable in time, or nothing pushed) is saved whole: full history and every file. +A checkpoint without an origin-based repository keeps manifest version 1, which older images restore. +Origin-based snapshots do not save Git notes (`refs/notes`) or stash entries other than the latest. Explicit checkpoint recovery sets `BUZZ_CHECKPOINT_RESTORE_KEY` in the launch Secret; it is empty otherwise. After the developer profile bootstrap and before the harness starts, the entrypoint runs `buzz-agent-checkpoint restore-once`. diff --git a/voltage/images/buzz-agent/checkpoint.py b/voltage/images/buzz-agent/checkpoint.py index 388eb4ec3b6..a358a572fad 100644 --- a/voltage/images/buzz-agent/checkpoint.py +++ b/voltage/images/buzz-agent/checkpoint.py @@ -2,6 +2,7 @@ """Save and explicitly restore private development workspaces.""" import argparse import base64 +import concurrent.futures import hashlib import json import os @@ -9,6 +10,7 @@ import re import selectors import shutil +import signal import stat import subprocess import sys @@ -20,6 +22,8 @@ MAX_BYTES = 256 * 1024 * 1024 MAX_FILES = 50000 +# All origins are asked at once; this bounds the whole save's network wait. +ORIGIN_SECONDS = 20 EXCLUDED = {'.git', '.aws', '.ssh', '.gnupg', '.config', '.codex', '.claude', '.cache', '.local', '.npm', '.cargo', '.rustup', '.venv', 'venv', 'node_modules', 'target', '__pycache__', 'dist', 'build', '.terraform'} @@ -185,11 +189,13 @@ def public_origin(repo): return urlunsplit((parsed.scheme, host, parsed.path, '', '')) -def scan(directory, *, history=False): - """Refuse a credential the agent added: commits not on any remote, or files. +def scan(directory, *, history=False, revisions=()): + """Refuse a credential the agent added: commits not on a remote, or files. History and files already on a remote are on the forge, where the agent's - token can read them, so the scan leaves them out.""" + token can read them, so the scan leaves them out. `revisions` select an + origin-based save's bundled commits in a few arguments, so the scan covers + exactly what it uploads.""" # Immutable image configuration prevents workspace allowlists from bypassing # the scan. Scanner output can contain secrets even on errors, so suppress it. previous = STEP[0] @@ -200,7 +206,8 @@ def scan(directory, *, history=False): '--gitleaks-ignore-path', '/dev/null', '--ignore-gitleaks-allow', '--no-banner', '--redact', '--timeout', '60'] if history: - command += ['--log-opts=--all HEAD --not --remotes'] + command += ['--log-opts=' + (' '.join(revisions) if revisions + else '--all HEAD --not --remotes')] run(*command, timeout=65, env=env) step(previous) @@ -231,6 +238,86 @@ def write(path, content): write(path, b'\n'.join(added) + b'\n') +def origin_listings(origins): + """`git ls-remote --heads --tags` of each origin, or None where unusable. + + Asks the normalized URL restore clones, with the global Git configuration + restore uses, so a save never relies on a credential restore lacks. Only + HTTPS origins qualify: the Sandbox's GitHub credential serves HTTPS. All + origins are asked at once under one ORIGIN_SECONDS deadline; an origin + that cannot answer in time is saved as a bundle instead.""" + deadline = time.monotonic() + ORIGIN_SECONDS + env = dict(os.environ, GIT_TERMINAL_PROMPT='0') + + def ask(origin): + remaining = deadline - time.monotonic() + if urlsplit(origin).scheme != 'https' or remaining <= 0: + return None + # Its own process group: git's HTTPS helper would otherwise outlive a + # timed-out git and hold the output pipe open. + process = subprocess.Popen(['git', '-c', 'core.hooksPath=/dev/null', 'ls-remote', '--heads', + '--tags', origin], cwd='/', env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=True) + try: + listing, _ = process.communicate(timeout=remaining) + except subprocess.TimeoutExpired: + listing = None + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + return listing if listing is not None and process.returncode == 0 else None + + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + return list(pool.map(ask, origins)) + + +def origin_base(repo, listing): + """What a clone of origin would give this repository. + + Returns `^` rev-list lines for origin's branch tips that this clone + has, and origin's (ref, object) pairs. A clone restores everything + reachable from those tips, so the checkpoint needs only what lies beyond + them. Asks origin, not remote-tracking refs: those can be stale, and after + a bundle restore they name commits origin never had. No tips means restore + cannot build on origin.""" + if listing is None: + return b'', set() + refs = set() + for line in listing.decode().splitlines(): + if '\t' in line: + name, ref = line.split('\t', 1) + refs.add((ref, name)) + heads = sorted({value for ref, value in refs if ref.startswith('refs/heads/')}) + found = git(repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)', + input=''.join(head + '\n' for head in heads).encode()) + tips = b''.join(b'^' + line.split(b' ', 1)[0] + b'\n' + for line in found.splitlines() if line.endswith(b' commit')) + return tips, refs + + +def local_refs(repo, origin_refs): + """Branches, and tags origin lacks, for the bundle and for restore to + recreate exactly. + + Annotated tag objects travel by content: a bundle omits a ref whose target + origin already has, and a tag object is not on origin.""" + refs, tags = [], {} + listing = git(repo, 'for-each-ref', '--format=%(objectname) %(objecttype) %(refname)', + 'refs/heads/', 'refs/tags/') + for line in listing.decode().splitlines(): + value, kind, ref = line.split(' ', 2) + if ref.startswith('refs/tags/') and (ref, value) in origin_refs: + continue + refs.append([ref, value]) + if kind == 'tag': + tags[value] = base64.b64encode(git(repo, 'cat-file', 'tag', value)).decode() + return refs, tags + + def pack(workspace, output): workspace = workspace.resolve(strict=True) manifest = {'version': 1, 'repositories': []} @@ -241,37 +328,89 @@ def pack(workspace, output): stage = Path(temp) # What the agent added to each repository, for the secret scan. review = Path(review_temp) - for index, repo in enumerate(repositories(workspace)): + repos = repositories(workspace) + if any(repo == workspace for repo in repos): + raise ValueError('Use a workspace directory containing repositories') + origins = [public_origin(repo) for repo in repos] + listings = origin_listings(origins) + for index, (repo, origin, listing) in enumerate(zip(repos, origins, listings)): relative = repo.relative_to(workspace).as_posix() - if relative == '.': - raise ValueError('Use a workspace directory containing repositories') - record = {'path': relative, 'origin': public_origin(repo), + tips, origin_refs = origin_base(repo, listing) + base = 'origin' if tips else 'bundle' + record = {'path': relative, 'origin': origin, 'base': base, 'head': git(repo, 'rev-parse', 'HEAD').decode().strip(), 'branch': git(repo, 'branch', '--show-current').decode().strip(), 'shallow': shallow_commits(repo), - 'excluded': [], 'files': []} + 'excluded': [], 'files': [], 'deleted': []} + if base == 'origin': + # Older images refuse version 2, so it marks only what needs it. + manifest['version'] = 2 + record['refs'], record['tags'] = local_refs(repo, origin_refs) + total += sum(len(content) for content in record['tags'].values()) destination = stage / 'repositories' / str(index) (destination / 'files').mkdir(parents=True) - scan(repo, history=True) + beyond = [] + if base == 'origin': + # Only what the agent owns, beyond origin's branches: never + # origin's remote-tracking refs or tags, whose boundaries would + # sit on branches origin may delete before the restore. A ref at + # or behind an origin tip adds no commit. + owned = ['HEAD'] + [ref for ref, _ in record['refs']] + if git(repo, 'for-each-ref', '--format=%(refname)', 'refs/stash').strip(): + owned.append('refs/stash') + beyond = [name for name in owned if git( + repo, 'rev-list', '--max-count=1', '--stdin', input=(name + '\n').encode() + tips).strip()] + revisions = ''.join(name + '\n' for name in beyond).encode() + tips + if beyond: + # The same commits as the bundle, bounded by their boundary + # rather than by every origin tip, which can exceed one + # argument's size limit. + boundary = [line[1:] for line in git(repo, 'rev-list', '--boundary', '--stdin', + input=revisions).decode().splitlines() + if line.startswith('-')] + scan(repo, history=True, revisions=beyond + ['^' + value for value in boundary]) + else: + scan(repo, history=True) review_changes(repo, review / str(index)) - git(repo, 'bundle', 'create', str(destination / 'history.bundle'), '--all', 'HEAD') - total += (destination / 'history.bundle').stat().st_size + bundle = destination / 'history.bundle' + if base == 'bundle': + # The whole history, and every file: restore needs nothing else. + git(repo, 'bundle', 'create', str(bundle), '--all', 'HEAD') + names = git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + else: + # Only commits origin lacks, and only files that differ from HEAD. + # git refuses an empty bundle, so a fully pushed repository has none. + if beyond: + git(repo, 'bundle', 'create', str(bundle), '--stdin', input=revisions) + names = (git(repo, 'ls-files', '-z', '--others', '--exclude-standard') + b'\0' + + git(repo, 'diff', '--name-only', '--no-renames', '-z', 'HEAD')) + if bundle.exists(): + total += bundle.stat().st_size if total > MAX_BYTES: raise ValueError('Checkpoint history exceeds the 256 MiB source budget') - # Verify in an independent object store, just as a future restore - # will, before advertising or uploading this checkpoint. - with tempfile.TemporaryDirectory(prefix='agent-checkpoint-history-') as history_temp: - restore_history(destination / 'history.bundle', record, Path(history_temp) / 'repo') - names = git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + if base == 'bundle': + # Preserve independent validation of self-contained bundles. + # Origin deltas depend on prerequisites supplied by the forge. + with tempfile.TemporaryDirectory(prefix='agent-checkpoint-history-') as history_temp: + restore_history(bundle, record, Path(history_temp) / 'repo') for raw in sorted(set(names.split(b'\0')) - {b''}): path = safe_path(os.fsdecode(raw)) if excluded(path): record['excluded'].append(str(path)) continue source = repo / path + if any((repo / PurePosixPath(*path.parts[:depth])).is_symlink() + for depth in range(1, len(path.parts))): + # A directory replaced by a symlink: the symlink is its own + # entry, and nothing is read through it. + if base == 'origin': + record['deleted'].append(str(path)) + continue if not source.exists() and not source.is_symlink(): - continue # A tracked deletion is represented by absence. - if source.is_dir(): + if base == 'origin': + record['deleted'].append(str(path)) + continue # A bundle snapshot represents a deletion by absence. + if source.is_dir() and not source.is_symlink(): continue # Submodule repositories are discovered separately. info = source.lstat() target = destination / 'files' / path @@ -329,7 +468,7 @@ def restore(archive_path, destination): archive.extractall(temp, filter='data') stage = Path(temp) / 'snapshot' manifest = json.loads((stage / 'manifest.json').read_text()) - if manifest.get('version') != 1: + if manifest.get('version') not in (1, 2): raise ValueError('Unsupported checkpoint version') destination.mkdir(mode=0o700, parents=True) try: @@ -352,25 +491,193 @@ def restore_repositories(stage, manifest, destination): raise ValueError('Checkpoint repository path is reserved') source = stage / 'repositories' / str(index) repo = destination / relative - restore_history(source / 'history.bundle', record, repo) - git(repo, 'checkout', '--detach', record['head']) + if not re.fullmatch('[0-9a-f]{40,64}', record['head']): + raise ValueError('Invalid checkpoint revision') + # Version 1 checkpoints are all bundle snapshots. + base = record.get('base', 'bundle') + if base == 'bundle': + restore_history(source / 'history.bundle', record, repo) + elif base == 'origin': + validate_origin_record(record) + clone_origin(record['origin'], repo) + if (source / 'history.bundle').exists(): + fetch_bundle(repo, source / 'history.bundle') + # HEAD or a saved ref can sit on a branch origin deleted after + # the save, such as a pushed branch removed by a squash merge. + fetch_missing(repo, [record['head']] + [value for _, value in record['refs'] + if value not in record['tags']] + + [tag_target(content) for content in record['tags'].values()]) + restore_refs(repo, record) + else: + raise ValueError('Unsupported checkpoint repository base') + # Checking out a large tree can take minutes. + git(repo, 'checkout', '--detach', record['head'], timeout=600) if record['branch']: git(repo, 'check-ref-format', '--branch', record['branch']) - git(repo, 'checkout', '-B', record['branch']) + git(repo, 'checkout', '-B', record['branch'], timeout=600) git(repo, 'remote', 'set-url', 'origin', record['origin']) - # Absence from the snapshot means deletion, except for excluded paths: - # they were never saved, so they keep their committed content. - for raw in git(repo, 'ls-files', '-z').split(b'\0'): - if raw: - path = safe_path(os.fsdecode(raw)) - target = repo / path - if not excluded(path) and (target.is_file() or target.is_symlink()): - target.unlink() - shutil.copytree(source / 'files', repo, dirs_exist_ok=True, symlinks=True) + if base == 'bundle': + # Absence from the snapshot means deletion, except for excluded + # paths: they were never saved, so they keep their committed content. + deleted = [os.fsdecode(raw) for raw in git(repo, 'ls-files', '-z').split(b'\0') if raw] + else: + deleted = record.get('deleted', []) + for value in deleted: + path = safe_path(value) + if not excluded(path): + remove(repo, path) + for value in record['files']: + path = safe_path(value) + if excluded(path): + raise ValueError('Checkpoint file path is reserved') + place(repo, source / 'files', path) review_changes(repo, review / str(index)) scan(review) +def clone_origin(origin, repo): + """Clone a checkpoint's pushed base with the Sandbox's Git credentials. + + A large repository can clone slowly but steadily, so there is no overall + deadline; Git fails the clone only when the transfer stalls.""" + if urlsplit(origin).scheme != 'https': + raise ValueError('Checkpoint origin base requires an HTTPS origin') + run('git', '-c', 'core.hooksPath=/dev/null', '-c', 'http.lowSpeedLimit=1000', + '-c', 'http.lowSpeedTime=60', 'clone', '--no-checkout', '--', origin, str(repo), + timeout=None, env=dict(os.environ, GIT_TERMINAL_PROMPT='0')) + + +def remove(repo, path): + """Delete the file or symlink at path, never through a symlink or file. + + Git tracks nothing beyond a symlink, so a parent that is not a real + directory means the path is already absent.""" + target = repo + for part in path.parts[:-1]: + target = target / part + if target.is_symlink() or not target.is_dir(): + return + target = target / path.parts[-1] + if target.is_symlink() or target.is_file(): + target.unlink() + + +def place(repo, files, path): + """Put the saved file or symlink at path, replacing whatever is there. + + The checkout can hold another type at the path or a parent: a symlink the + agent retargeted or replaced, a file it turned into a directory, or the + reverse. Replacing, not writing through, keeps every write inside repo.""" + target = repo + for part in path.parts[:-1]: + target = target / part + if target.is_symlink() or (target.exists() and not target.is_dir()): + target.unlink() + target.mkdir(exist_ok=True) + target = target / path.parts[-1] + if target.is_symlink() or target.is_file(): + target.unlink() + elif target.is_dir(): + shutil.rmtree(target) + saved = files / path + if saved.is_symlink(): + target.symlink_to(os.readlink(saved)) + elif saved.is_file(): + shutil.copy2(saved, target, follow_symlinks=False) + else: + raise ValueError('Checkpoint file is missing from the archive') + + +def validate_origin_record(record): + """Refuse a malformed origin record before acting on it.""" + if not (isinstance(record.get('origin'), str) and isinstance(record.get('files'), list) + and isinstance(record.get('deleted', []), list) and isinstance(record.get('refs', []), list) + and isinstance(record.get('tags', {}), dict) + and all(isinstance(value, str) for value in record['files'] + record.get('deleted', [])) + and all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(v, str) for v in pair) + for pair in record.get('refs', [])) + and all(isinstance(key, str) and isinstance(value, str) + for key, value in record.get('tags', {}).items())): + raise ValueError('Invalid checkpoint repository record') + record.setdefault('refs', []) + record.setdefault('tags', {}) + + +def tag_target(content): + """The object an annotated tag names, from its saved content.""" + first = base64.b64decode(content, validate=True).split(b'\n', 1)[0].decode() + if not re.fullmatch('object [0-9a-f]{40,64}', first): + raise ValueError('Invalid checkpoint tag object') + return first.split(' ', 1)[1] + + +def fetch_missing(repo, values): + """Fetch the named commits the clone lacks from origin. + + GitHub still serves a commit by name after its branch is deleted or + force-pushed; a commit origin no longer has fails the restore here.""" + for value in values: + if not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint revision') + found = git(repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)', + input=''.join(value + '\n' for value in sorted(set(values))).encode()) + missing = [line.split(' ', 1)[0] for line in found.decode().splitlines() + if line.endswith(' missing')] + if missing: + git(repo, '-c', 'http.lowSpeedLimit=1000', '-c', 'http.lowSpeedTime=60', 'fetch', + '--no-tags', '--no-write-fetch-head', 'origin', *missing, timeout=None, + env=dict(os.environ, GIT_TERMINAL_PROMPT='0')) + + +def restore_refs(repo, record): + """Recreate the saved branches and tags, annotated tag objects first.""" + for name, content in record.get('tags', {}).items(): + made = git(repo, 'hash-object', '-t', 'tag', '-w', '--stdin', + input=base64.b64decode(content, validate=True)).decode().strip() + if made != name: + raise ValueError('Checkpoint tag object does not match its name') + for ref, value in record.get('refs', []): + if not ref.startswith(('refs/heads/', 'refs/tags/')) or not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint ref') + git(repo, 'check-ref-format', ref) + git(repo, 'update-ref', ref, value) + + +def prerequisites(bundle): + """The commits a bundle builds on, from its header.""" + with open(bundle, 'rb') as stream: + header = stream.read(1024 * 1024) + if b'\n\n' not in header: + raise ValueError('Checkpoint bundle header is too large') + lines = header.split(b'\n\n', 1)[0].split(b'\n') + if lines[0] not in (b'# v2 git bundle', b'# v3 git bundle'): + raise ValueError('Unsupported checkpoint bundle') + found = [] + for line in lines[1:]: + if line.startswith(b'-'): + value = line[1:].split(b' ', 1)[0].decode() + if not re.fullmatch('[0-9a-f]{40,64}', value): + raise ValueError('Invalid checkpoint bundle prerequisite') + found.append(value) + return found + + +def fetch_bundle(repo, bundle): + """Add a checkpoint's unpushed commits, and their refs, to an origin clone.""" + # A prerequisite can leave origin's branches after the save. + fetch_missing(repo, prerequisites(bundle)) + specs = [] + for line in git(repo, 'bundle', 'list-heads', str(bundle)).decode().splitlines(): + name = line.split(' ', 1)[1] if ' ' in line else '' + if name == 'HEAD': + specs.append('HEAD') # A detached HEAD commit is not on any branch. + else: + git(repo, 'check-ref-format', name) + specs.append('+' + name + ':' + name) + # The bundle's prerequisites are origin commits; a missing one fails here. + git(repo, 'fetch', '--update-head-ok', '--no-tags', str(bundle), *specs) + + def storage(): bucket = os.environ.get('BUZZ_CHECKPOINT_BUCKET', '') prefix = os.environ.get('BUZZ_CHECKPOINT_PREFIX', '') diff --git a/voltage/images/buzz-agent/test-checkpoint-shallow.py b/voltage/images/buzz-agent/test-checkpoint-shallow.py index 7855deac994..5a738255bdd 100644 --- a/voltage/images/buzz-agent/test-checkpoint-shallow.py +++ b/voltage/images/buzz-agent/test-checkpoint-shallow.py @@ -135,6 +135,33 @@ def damaged_bundle(repo, *args, **kwargs): checkpoint.git = original_git print('PASS: save validates its bundle in an independent repository before producing an archive') + # Exercise the integration with origin-based snapshots too. The URL rewrite + # supplies a local forge; the container still has no network access. + git(root, 'config', '--global', 'url.'+upstream.as_uri()+'.insteadOf', + 'https://github.com/example/fixture.git') + origin_archive = root/'origin.tar.gz' + origin_manifest = checkpoint.pack(workspace, origin_archive) + assert origin_manifest['version'] == 2 + assert origin_manifest['repositories'][0]['base'] == 'origin' + origin_destination = root/'restored-origin' + checkpoint.restore(origin_archive, origin_destination) + origin_repo = origin_destination/'example' + assert git(origin_repo, 'rev-parse', 'HEAD').decode().strip() == head + assert git(origin_repo, 'status', '--porcelain') == original_status + assert (origin_repo/'source.txt').read_text() == 'working edit\n' + assert git(origin_repo, 'rev-parse', '--is-shallow-repository').strip() == b'false' + git(origin_repo, 'fsck', '--full') + # If the origin cannot be read, the same shallow checkout must still use + # the independent bundle validation and recovery from #99. + git(root, 'config', '--global', '--unset-all', + 'url.'+upstream.as_uri()+'.insteadOf') + fallback_archive = root/'fallback.tar.gz' + fallback_manifest = checkpoint.pack(workspace, fallback_archive) + assert fallback_manifest['version'] == 1 + assert fallback_manifest['repositories'][0]['base'] == 'bundle' + verify(fallback_archive, 'restored-fallback') + print('PASS: shallow origin snapshots restore full history; an unavailable origin falls back to a verified shallow bundle') + # A damaged archive can produce large fsck diagnostics. Exercise the real # child process boundary and verify it is reaped when the output is capped. tools = root/'tools' diff --git a/voltage/images/buzz-agent/test-checkpoint.py b/voltage/images/buzz-agent/test-checkpoint.py index c3e18dd4deb..2dabb553e6a 100644 --- a/voltage/images/buzz-agent/test-checkpoint.py +++ b/voltage/images/buzz-agent/test-checkpoint.py @@ -5,7 +5,7 @@ SCRIPT = r''' import base64, hashlib, importlib.machinery, importlib.util, io, json, os from pathlib import Path -import subprocess, tarfile, tempfile +import shutil, socket, subprocess, tarfile, tempfile, threading, time loader = importlib.machinery.SourceFileLoader('checkpoint', '/usr/local/bin/buzz-agent-checkpoint') spec = importlib.util.spec_from_loader(loader.name, loader) @@ -72,6 +72,22 @@ def git(repo, *args): raise AssertionError('restore overwrote an existing workspace') except ValueError: pass + # Without an origin base the manifest stays version 1, which older images + # restore; archives written before origin bases still restore too. + assert manifest['version'] == 1 and manifest['repositories'][0]['base'] == 'bundle' + legacy = root/'legacy' + with tarfile.open(artifact) as archive: + archive.extractall(legacy, filter='data') + old_manifest = json.loads((legacy/'snapshot/manifest.json').read_text()) + for record in old_manifest['repositories']: + del record['base'], record['deleted'] + (legacy/'snapshot/manifest.json').write_text(json.dumps(old_manifest)) + with tarfile.open(root/'legacy.tar.gz','w:gz') as archive: + archive.add(legacy/'snapshot',arcname='snapshot') + checkpoint.restore(root/'legacy.tar.gz',root/'restored-legacy') + assert git(root/'restored-legacy/example','rev-parse','HEAD') == head + assert (root/'restored-legacy/example/new.rs').read_bytes() == (repo/'new.rs').read_bytes() + assert not (root/'restored-legacy/example/deleted.txt').exists() print('PASS: SSH origin identity/port, local commit, edited/untracked source, deletion, and symlink survive restore; account files and build outputs excluded; excluded tracked files keep committed content') (repo/'leak.txt').write_text('github_token = "ghp_'+'aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6'+'"\n') @@ -98,11 +114,16 @@ def git(repo, *args): git(upstream,'config','user.name','Checkpoint Fixture') git(upstream,'config','user.email','fixture@example.invalid') (upstream/'values.yaml').write_text(token) + (upstream/'large.bin').write_bytes(bytes(1024*1024)) + (upstream/'remove.txt').write_text('pushed, then deleted locally\n') git(upstream,'add','.') git(upstream,'commit','-m','chore: commit a credential upstream') clone = workspace/'clone' subprocess.check_call(['git','clone','-q',str(upstream),str(clone)],stderr=subprocess.DEVNULL) git(clone,'remote','set-url','origin','https://github.com/example/upstream.git') + # Origin-based restore clones origin; offline, serve it from the fixture. + os.environ['GIT_CONFIG_GLOBAL'] = str(root/'gitconfig') + git(root,'config','--global','url.'+str(upstream)+'.insteadOf','https://github.com/example/upstream.git') git(clone,'config','user.name','Checkpoint Fixture') git(clone,'config','user.email','fixture@example.invalid') checkpoint.pack(workspace,root/'upstream.tar.gz') @@ -148,6 +169,281 @@ def git(repo, *args): checkpoint.pack(workspace,root/'reset.tar.gz') print('PASS: credentials already on the remote are allowed and restore; one added in a file, an unpushed commit or an archive is refused with the scan step') + # With an HTTPS origin, a checkpoint holds only what origin lacks, so + # pushed content larger than the budget does not count against it. + git(clone,'checkout','-q','-b','feature') + (clone/'local.txt').write_text('unpushed commit\n') + git(clone,'add','local.txt') + git(clone,'commit','-m','feat: add an unpushed commit') + # A remote-tracking ref origin does not have, as a bundle restore leaves: + # the save asks origin, so the commit stays in the checkpoint. + git(clone,'update-ref','refs/remotes/origin/feature','HEAD') + (clone/'remove.txt').unlink() + (clone/'untracked.txt').write_text('untracked\n') + budget = checkpoint.MAX_BYTES + checkpoint.MAX_BYTES = 512*1024 + try: + thin = checkpoint.pack(workspace,root/'thin.tar.gz') + # With an SSH origin the same workspace is a bundle snapshot, which + # counts the pushed tree and exceeds the budget. + git(clone,'remote','set-url','origin','ssh://git@example.com/upstream.git') + try: + checkpoint.pack(workspace,root/'ssh.tar.gz') + raise AssertionError('an SSH origin repository skipped its bundle snapshot') + except ValueError as error: + assert 'budget' in str(error), error + git(clone,'remote','set-url','origin','https://github.com/example/upstream.git') + finally: + checkpoint.MAX_BYTES = budget + record = next(r for r in thin['repositories'] if r['path'] == 'clone') + assert record['base'] == 'origin' and record['deleted'] == ['remove.txt'], record + # The unpushed commit travels in the bundle, not as a changed file. + assert record['files'] == ['untracked.txt'], record['files'] + with tarfile.open(root/'thin.tar.gz') as archive: + assert not any(n.endswith('/large.bin') for n in archive.getnames()) + checkpoint.restore(root/'thin.tar.gz',root/'restored-thin') + restored = root/'restored-thin/clone' + assert git(restored,'rev-parse','HEAD') == git(clone,'rev-parse','HEAD') + assert git(restored,'branch','--show-current') == 'feature' + assert git(restored,'config','--get','remote.origin.url') == 'https://github.com/example/upstream.git' + assert (restored/'large.bin').stat().st_size == 1024*1024 + assert (restored/'local.txt').read_text() == 'unpushed commit\n' + assert (restored/'untracked.txt').read_text() == 'untracked\n' + assert not (restored/'remove.txt').exists() + print('PASS: an HTTPS origin repository saves only unpushed commits and changed files, within budget, and restores from origin; an SSH origin keeps the full bundle') + + # Origin-based restore replaces each saved path over the origin checkout, + # whatever type the agent changed it from, and recreates refs origin lacks. + types_upstream = root/'types-upstream' + types_upstream.mkdir() + git(types_upstream,'init','-b','main') + git(types_upstream,'config','user.name','Checkpoint Fixture') + git(types_upstream,'config','user.email','fixture@example.invalid') + for name in ('a.txt','b.txt','c.txt'): + (types_upstream/name).write_text(name+'\n') + (types_upstream/'d').mkdir() + (types_upstream/'d/x.txt').write_text('in a directory\n') + for name in ('linked','other'): + (types_upstream/name).mkdir() + (types_upstream/name/'f.txt').write_text(name+'\n') + (types_upstream/'retarget').symlink_to('a.txt') + (types_upstream/'to-file').symlink_to('a.txt') + git(types_upstream,'add','.') + git(types_upstream,'commit','-m','test: create typed paths') + types = workspace/'types' + subprocess.check_call(['git','clone','-q',str(types_upstream),str(types)],stderr=subprocess.DEVNULL) + git(types,'remote','set-url','origin','https://github.com/example/types.git') + git(root,'config','--global','url.'+str(types_upstream)+'.insteadOf','https://github.com/example/types.git') + git(types,'config','user.name','Checkpoint Fixture') + git(types,'config','user.email','fixture@example.invalid') + (types/'retarget').unlink() + (types/'retarget').symlink_to('b.txt') + (types/'to-file').unlink() + (types/'to-file').write_text('now a regular file\n') + (types/'c.txt').unlink() + (types/'c.txt').mkdir() + (types/'c.txt/inner.txt').write_text('file became a directory\n') + (types/'d/x.txt').unlink() + (types/'d').rmdir() + (types/'d').write_text('directory became a file\n') + (types/'linked/f.txt').unlink() + (types/'linked').rmdir() + (types/'linked').symlink_to('other') + git(types,'tag','light') + git(types,'-c','tag.gpgsign=false','tag','-a','-m','annotated, unpushed','annotated') + git(types,'branch','wip') + typed = checkpoint.pack(workspace,root/'types.tar.gz') + assert typed['version'] == 2 + assert next(r for r in typed['repositories'] if r['path'] == 'types')['base'] == 'origin' + checkpoint.restore(root/'types.tar.gz',root/'restored-types') + restored = root/'restored-types/types' + assert os.readlink(restored/'retarget') == 'b.txt' + assert not (restored/'to-file').is_symlink() + assert (restored/'to-file').read_text() == 'now a regular file\n' + assert (restored/'a.txt').read_text() == 'a.txt\n', 'a write went through the old symlink' + assert (restored/'c.txt/inner.txt').read_text() == 'file became a directory\n' + assert (restored/'d').read_text() == 'directory became a file\n' + assert os.readlink(restored/'linked') == 'other' + assert (restored/'other/f.txt').read_text() == 'other\n' + types_record = next(r for r in typed['repositories'] if r['path'] == 'types') + assert 'linked/f.txt' in types_record['deleted'] and 'linked/f.txt' not in types_record['files'] + assert git(restored,'status','--porcelain') == git(types,'status','--porcelain') + for ref in ('refs/tags/light','refs/tags/annotated','refs/heads/wip'): + assert git(restored,'rev-parse',ref) == git(types,'rev-parse',ref), ref + assert git(restored,'cat-file','-t','annotated') == 'tag' + assert git(restored,'cat-file','tag','annotated') == git(types,'cat-file','tag','annotated') + # An annotated tag message the agent adds is secret-scanned: the manifest + # carries it base64-encoded, and the manifest scan decodes base64. + git(types,'-c','tag.gpgsign=false','tag','-a','-m','github_token = "ghp_'+'aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6'+'"','leaky') + try: + checkpoint.pack(workspace,root/'leaky.tar.gz') + raise AssertionError('secret scanner admitted a credential in an annotated tag message') + except ValueError: + assert checkpoint.STEP[0] == 'scanning for secrets', checkpoint.STEP[0] + git(types,'tag','-d','leaky') + # A crafted deletion cannot reach through a symlinked parent. + forged = root/'forged-types' + with tarfile.open(root/'types.tar.gz') as archive: + archive.extractall(forged, filter='data') + forged_manifest = json.loads((forged/'snapshot/manifest.json').read_text()) + (root/'outside.txt').write_text('outside\n') + git(types_upstream,'rm','-rq','d') + (types_upstream/'escape').symlink_to(root) + git(types_upstream,'add','escape') + git(types_upstream,'commit','-m','test: add a symlink out of the repository') + record = next(r for r in forged_manifest['repositories'] if r['path'] == 'types') + record['head'] = git(types_upstream,'rev-parse','HEAD') + record['branch'] = '' + record['files'], record['refs'], record['tags'] = [], [], {} + record['deleted'] = ['escape/outside.txt'] + (forged/'snapshot/manifest.json').write_text(json.dumps(forged_manifest)) + with tarfile.open(root/'forged-types.tar.gz','w:gz') as archive: + archive.add(forged/'snapshot',arcname='snapshot') + checkpoint.restore(root/'forged-types.tar.gz',root/'restored-forged-types') + assert (root/'outside.txt').read_text() == 'outside\n' + # A malformed record is refused as a checkpoint error, not a crash. + record['refs'] = [['refs/heads/x', 2]] + (forged/'snapshot/manifest.json').write_text(json.dumps(forged_manifest)) + with tarfile.open(root/'malformed.tar.gz','w:gz') as archive: + archive.add(forged/'snapshot',arcname='snapshot') + try: + checkpoint.restore(root/'malformed.tar.gz',root/'restored-malformed') + raise AssertionError('a malformed origin record was restored') + except ValueError: + pass + assert not (root/'restored-malformed').exists() + print('PASS: origin restore replaces retargeted symlinks, symlinks turned files, and file/directory swaps in place; unpushed tags, annotated tag objects and branches survive; deletions never follow a symlink') + + # The bundle holds only what the agent owns, never origin's tags or + # remote-tracking refs. Restore fetches a prerequisite origin stopped + # advertising: here the branch the agent built on is deleted after the save. + topic_upstream = root/'topic-upstream' + topic_upstream.mkdir() + git(topic_upstream,'init','-b','main') + git(topic_upstream,'config','user.name','Checkpoint Fixture') + git(topic_upstream,'config','user.email','fixture@example.invalid') + # GitHub serves any commit in the repository by name. + git(topic_upstream,'config','uploadpack.allowAnySHA1InWant','true') + (topic_upstream/'base.txt').write_text('base\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: base') + git(topic_upstream,'tag','v1') + # A release tag on a commit no branch reaches, as bifrost's releases are. + git(topic_upstream,'checkout','-q','-b','release') + (topic_upstream/'release.txt').write_text('release\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: release') + git(topic_upstream,'tag','v2') + git(topic_upstream,'checkout','-q','main') + git(topic_upstream,'branch','-D','release') + git(topic_upstream,'checkout','-q','-b','topic') + (topic_upstream/'topic.txt').write_text('topic\n') + git(topic_upstream,'add','.') + # A prerequisite header line longer than one 256-byte read, with '-' at + # the split point. + git(topic_upstream,'commit','-m','test: '+'a'*208+'-topic') + git(topic_upstream,'checkout','-q','-b','pushed') + (topic_upstream/'pushed.txt').write_text('pushed\n') + git(topic_upstream,'add','.') + git(topic_upstream,'commit','-m','test: pushed') + git(topic_upstream,'checkout','-q','topic') + git(topic_upstream,'checkout','-q','main') + topic = workspace/'topic' + subprocess.check_call(['git','clone','-q',str(topic_upstream),str(topic)],stderr=subprocess.DEVNULL) + git(topic,'remote','set-url','origin','https://github.com/example/topic.git') + # file:// clones over the fetch protocol, so a restore clone lacks commits + # no ref reaches; a plain path would copy the whole object store. + git(root,'config','--global','url.file://'+str(topic_upstream)+'.insteadOf','https://github.com/example/topic.git') + git(topic,'config','user.name','Checkpoint Fixture') + git(topic,'config','user.email','fixture@example.invalid') + git(topic,'checkout','-q','-b','agent','origin/topic') + (topic/'agent.txt').write_text('built on topic\n') + git(topic,'add','agent.txt') + git(topic,'commit','-m','test: build on topic') + # A pushed branch with nothing unpushed, and a local tag on it, whose + # branch origin deletes after the save (as a squash merge does). + pushed = workspace/'pushed' + subprocess.check_call(['git','clone','-q','file://'+str(topic_upstream),str(pushed)],stderr=subprocess.DEVNULL) + git(pushed,'remote','set-url','origin','https://github.com/example/topic.git') + git(pushed,'checkout','-q','-b','pushed','origin/pushed') + git(pushed,'tag','mine') + saved = checkpoint.pack(workspace,root/'topic.tar.gz') + pushed_index = [r['path'] for r in saved['repositories']].index('pushed') + index = [r['path'] for r in saved['repositories']].index('topic') + unpacked = root/'topic-unpacked' + with tarfile.open(root/'topic.tar.gz') as archive: + archive.extractall(unpacked, filter='data') + heads = git(topic,'bundle','list-heads',str(unpacked/'snapshot/repositories'/str(index)/'history.bundle')) + assert git(topic,'rev-parse','-q','--verify','refs/tags/v2'), 'fixture clone lacks the release tag' + assert 'refs/remotes/' not in heads and 'refs/tags/' not in heads, heads + git(topic_upstream,'branch','-D','topic') + git(topic_upstream,'branch','-D','pushed') + assert not (unpacked/'snapshot/repositories'/str(pushed_index)/'history.bundle').exists() + lost = git(topic,'rev-parse','origin/topic') + probe = root/'topic-probe' + git(root,'clone','-q','--no-checkout','https://github.com/example/topic.git',str(probe)) + assert subprocess.run(['git','-C',str(probe),'cat-file','-e',lost]).returncode != 0, 'the clone still has the deleted branch' + checkpoint.restore(root/'topic.tar.gz',root/'restored-topic') + restored = root/'restored-topic/topic' + assert git(restored,'rev-parse','HEAD') == git(topic,'rev-parse','HEAD') + assert (restored/'topic.txt').read_text() == 'topic\n' + assert (restored/'agent.txt').read_text() == 'built on topic\n' + restored_pushed = root/'restored-topic/pushed' + assert git(restored_pushed,'rev-parse','HEAD') == git(pushed,'rev-parse','HEAD') + assert git(restored_pushed,'branch','--show-current') == 'pushed' + assert git(restored_pushed,'rev-parse','mine') == git(pushed,'rev-parse','mine') + assert (restored_pushed/'pushed.txt').read_text() == 'pushed\n' + print('PASS: the bundle excludes origin tags and remote-tracking refs; restore fetches a prerequisite, HEAD and a tag whose branches origin deleted after the save') + + # Thousands of origin branches do not grow any single scan argument. + many_upstream = root/'many-upstream' + many_upstream.mkdir() + git(many_upstream,'init','-b','main') + git(many_upstream,'config','user.name','Checkpoint Fixture') + git(many_upstream,'config','user.email','fixture@example.invalid') + (many_upstream/'base.txt').write_text('base\n') + git(many_upstream,'add','.') + git(many_upstream,'commit','-m','test: base') + stream = [] + for number in range(4000): + message = 'branch %d' % number + stream.append('commit refs/heads/b%d\ncommitter F 0 +0000\ndata %d\n%s\nfrom refs/heads/main\n\n' + % (number, len(message), message)) + subprocess.run(['git','-C',str(many_upstream),'fast-import','--quiet'],input=''.join(stream).encode(),check=True) + many = workspace/'many' + subprocess.check_call(['git','clone','-q',str(many_upstream),str(many)],stderr=subprocess.DEVNULL) + git(many,'remote','set-url','origin','https://github.com/example/many.git') + git(root,'config','--global','url.'+str(many_upstream)+'.insteadOf','https://github.com/example/many.git') + git(many,'config','user.name','Checkpoint Fixture') + git(many,'config','user.email','fixture@example.invalid') + (many/'agent.txt').write_text('unpushed\n') + git(many,'add','agent.txt') + git(many,'commit','-m','test: unpushed') + assert len(set(git(many,'for-each-ref','--format=%(objectname)','refs/remotes/').split())) > 3200 + crowded = checkpoint.pack(workspace,root/'many.tar.gz') + assert next(r for r in crowded['repositories'] if r['path'] == 'many')['base'] == 'origin' + shutil.rmtree(many) + print('PASS: an origin with 4,000 distinct branch tips packs; the history scan stays within argument limits') + + # Unreachable origins cost one shared deadline, not one per repository. + server = socket.socket() + server.bind(('127.0.0.1', 0)) + server.listen() + held = [] + threading.Thread(target=lambda: [held.append(server.accept()) for _ in range(8)], daemon=True).start() + hanging = 'https://127.0.0.1:%d/example.git' % server.getsockname()[1] + deadline = checkpoint.ORIGIN_SECONDS + checkpoint.ORIGIN_SECONDS = 2 + try: + started = time.monotonic() + listings = checkpoint.origin_listings([hanging]*4 + ['ssh://git@example.com/x.git']) + elapsed = time.monotonic() - started + finally: + checkpoint.ORIGIN_SECONDS = deadline + assert listings == [None]*5, listings + assert elapsed < 5, elapsed + print('PASS: four hanging origins and an SSH origin resolve to bundle saves within one shared deadline') + (root/'private').write_text('outside workspace') (repo/'escape').symlink_to(root/'private') try: