From 51b6adb85039148ac8f67a07820fac0f039c431f Mon Sep 17 00:00:00 2001 From: Josh Stevenson Date: Mon, 28 Sep 2026 01:00:00 -0500 Subject: [PATCH 1/2] ci(semantic-memory): prove pinned paired-root source --- .github/workflows/paired-root.yml | 84 ++++++++++++++ scripts/ci/paired_root.py | 182 ++++++++++++++++++++++++++++++ scripts/ci/test_paired_root.py | 115 +++++++++++++++++++ 3 files changed, 381 insertions(+) create mode 100644 .github/workflows/paired-root.yml create mode 100644 scripts/ci/paired_root.py create mode 100644 scripts/ci/test_paired_root.py diff --git a/.github/workflows/paired-root.yml b/.github/workflows/paired-root.yml new file mode 100644 index 0000000..a6bac16 --- /dev/null +++ b/.github/workflows/paired-root.yml @@ -0,0 +1,84 @@ +name: Pinned paired-root source proof + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: semantic-memory-paired-root-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + paired-root: + # Forks require separate maintainer admission; do not execute their scripts + # with the repository's Actions token merely because a PR was opened. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + LIBRARIES_SHA: 0b099ec416de60f6adafb182f1d1e83795d05c56 + # pull_request uses the synthetic base+head merge revision whose workflow + # file GitHub executes; push/dispatch use their exact event commit. + MIRROR_SHA: ${{ github.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || '' }} + steps: + - name: Checkout exact mirror source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: mirror + ref: ${{ env.MIRROR_SHA }} + persist-credentials: false + - name: Checkout exact Libraries owner source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: RecursiveIntell/Libraries + path: Libraries + ref: ${{ env.LIBRARIES_SHA }} + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install native prerequisites + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y build-essential libssl-dev + - name: Test paired-root admission rules + working-directory: mirror + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest scripts.ci.test_paired_root -v + - name: Assemble pinned disposable source pair + id: assemble + shell: bash + run: | + set -euo pipefail + scratch="$RUNNER_TEMP/semantic-memory-pair-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + PYTHONDONTWRITEBYTECODE=1 python3 mirror/scripts/ci/paired_root.py assemble \ + --libraries Libraries --mirror mirror \ + --libraries-sha "$LIBRARIES_SHA" --mirror-sha "$MIRROR_SHA" \ + --pr-head-sha "$PR_HEAD_SHA" \ + --scratch "$scratch" + printf 'root=%s\n' "$scratch/Libraries" >> "$GITHUB_OUTPUT" + printf 'lock_before=%s\n' "$scratch/original-Cargo.lock" >> "$GITHUB_OUTPUT" + - name: Compile, bind scratch lock delta, and test selected lanes + working-directory: ${{ steps.assemble.outputs.root }} + shell: bash + env: + ORIGINAL_LOCK: ${{ steps.assemble.outputs.lock_before }} + CARGO_TARGET_DIR: ${{ runner.temp }}/semantic-memory-paired-target + run: | + set -euo pipefail + # This first resolution may update only the disposable paired lock. + cargo check -p semantic-memory --example governed_append_canary + PYTHONDONTWRITEBYTECODE=1 python3 "$GITHUB_WORKSPACE/mirror/scripts/ci/paired_root.py" \ + check-lock --before "$ORIGINAL_LOCK" --after Cargo.lock + cargo fmt --package semantic-memory -- --check + cargo check --locked -p semantic-memory --example governed_append_canary + cargo clippy --locked -p semantic-memory --all-targets -- -D warnings + cargo test --locked -q -p semantic-memory --lib diff --git a/scripts/ci/paired_root.py b/scripts/ci/paired_root.py new file mode 100644 index 0000000..fdb1b65 --- /dev/null +++ b/scripts/ci/paired_root.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Assemble a disposable, source-pinned paired root for CI (not a sync).""" +from __future__ import annotations + +import argparse +import copy +import json +import re +import shutil +import subprocess +import tarfile +import tomllib +from pathlib import Path, PurePosixPath + + +REQUIRED_POLY_KV_DROP = 'fib-quant 0.1.0-alpha.1' + + +def require_exact_sha(observed: str, expected: str, owner: str) -> None: + if not re.fullmatch(r'[0-9a-f]{40}', expected) or observed != expected: + raise ValueError(f'{owner} source SHA differs from pinned exact 40-hex revision') + + +def require_pr_merge_parents(parents: list[str], pr_head_sha: str) -> None: + if not re.fullmatch(r'[0-9a-f]{40}', pr_head_sha) or len(parents) != 2 or parents[1] != pr_head_sha: + raise ValueError('pull_request merge revision does not bind exact PR head as second parent') + + +def validate_entry(path: str, mode: str) -> None: + posix = PurePosixPath(path) + if not path or '\\' in path or posix.is_absolute() or any(part in ('..', '.', '') for part in path.split('/')): + raise ValueError(f'unsafe source path: {path!r}') + if mode not in ('100644', '100755'): + raise ValueError(f'unsupported source mode for {path}: {mode}') + + +def claim_archive_member(path: str, kind: str, seen: dict[str, str]) -> None: + validate_entry(path, '100644') + if kind not in ('directory', 'file', 'symlink') or path in seen: + raise ValueError(f'duplicate or unsupported archive entry: {path}') + parts = path.split('/') + if any(seen.get('/'.join(parts[:index])) not in (None, 'directory') for index in range(1, len(parts))): + raise ValueError(f'archive path collides with non-directory ancestor: {path}') + if kind != 'directory' and any(existing.startswith(path + '/') for existing in seen): + raise ValueError(f'archive path collides with existing descendants: {path}') + seen[path] = kind + + +def validate_lock_delta(before: dict, after: dict) -> None: + expected = copy.deepcopy(before) + packages = [p for p in expected.get('package', []) if p.get('name') == 'poly-kv' and p.get('version') == '0.1.0-alpha.1'] + if len(packages) != 1 or REQUIRED_POLY_KV_DROP not in packages[0].get('dependencies', []): + raise ValueError('pinned lock has no unique PolyKV FibQuant dependency to drop') + packages[0]['dependencies'].remove(REQUIRED_POLY_KV_DROP) + if expected != after: + raise ValueError('scratch lock changed outside the exact declared PolyKV dependency drop') + + +def git(repo: Path, *args: str) -> bytes: + return subprocess.check_output(['git', '-C', str(repo), *args]) + + +def tracked_files(repo: Path, sha: str, *, strict: bool) -> dict[str, tuple[str, str]]: + records: dict[str, tuple[str, str]] = {} + for row in git(repo, 'ls-tree', '-r', '-z', sha).split(b'\0'): + if not row: + continue + meta, raw_name = row.split(b'\t', 1) + mode, kind, oid = meta.decode().split() + name = raw_name.decode('utf-8', 'strict') + if strict: + validate_entry(name, mode) + if kind != 'blob': + raise ValueError(f'unsupported mirror object: {name}') + records[name] = (mode, oid) + if not records: + raise ValueError(f'no tracked files in {repo}') + return records + + +def extract_archive(repo: Path, sha: str, destination: Path, *, owner: bool) -> None: + proc = subprocess.Popen( + ['git', '-C', str(repo), 'archive', '--format=tar', sha], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + try: + assert proc.stdout is not None + seen: dict[str, str] = {} + with tarfile.open(fileobj=proc.stdout, mode='r|') as archive: + for member in archive: + name = member.name.rstrip('/') + if not name: + continue + if owner and (name == 'semantic-memory' or name.startswith('semantic-memory/')): + continue + if member.isdir(): + claim_archive_member(name, 'directory', seen) + # Git archive directory entries are not durable source objects. + validate_entry(name, '100644') + (destination / name).mkdir(parents=True, exist_ok=True) + continue + if not member.isfile(): + if owner and member.issym(): + # Pinned Libraries archive has unrelated salvage symlinks. + claim_archive_member(name, 'symlink', seen) + continue + raise ValueError(f'non-regular archive entry: {name}') + claim_archive_member(name, 'file', seen) + mode = '100755' if member.mode & 0o111 else '100644' + validate_entry(name, mode) + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + raise ValueError(f'missing archive payload: {name}') + with source, target.open('wb') as handle: + shutil.copyfileobj(source, handle) + target.chmod(0o755 if mode == '100755' else 0o644) + stderr = proc.stderr.read() if proc.stderr is not None else b'' + if proc.wait() != 0: + raise ValueError(f'git archive failed: {stderr.decode(errors="replace")}') + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.stdout is not None: + proc.stdout.close() + if proc.stderr is not None: + proc.stderr.close() + + +def assemble(libraries: Path, mirror: Path, libraries_sha: str, mirror_sha: str, scratch: Path, + pr_head_sha: str = '') -> dict: + require_exact_sha(git(libraries, 'rev-parse', 'HEAD').decode().strip(), libraries_sha, 'Libraries') + require_exact_sha(git(mirror, 'rev-parse', 'HEAD').decode().strip(), mirror_sha, 'mirror') + if pr_head_sha: + parents = git(mirror, 'show', '-s', '--format=%P', mirror_sha).decode().split() + require_pr_merge_parents(parents, pr_head_sha) + files = tracked_files(mirror, mirror_sha, strict=True) + # Refuse to overwrite any existing source or artifact, including a symlink. + scratch.mkdir(parents=True, exist_ok=False) + paired = scratch / 'Libraries' + paired.mkdir() + extract_archive(libraries, libraries_sha, paired, owner=True) + package = paired / 'semantic-memory' + package.mkdir() + extract_archive(mirror, mirror_sha, package, owner=False) + for path, (mode, oid) in files.items(): + target = package / path + if not target.is_file() or git(mirror, 'hash-object', str(target)).decode().strip() != oid: + raise ValueError(f'assembled mirror content differs from Git source: {path}') + if bool(target.stat().st_mode & 0o111) != (mode == '100755'): + raise ValueError(f'assembled mirror executable mode differs: {path}') + shutil.copy2(paired / 'Cargo.lock', scratch / 'original-Cargo.lock') + result = {'libraries_sha': libraries_sha, 'mirror_sha': mirror_sha, 'pr_head_sha': pr_head_sha or None, + 'mirror_tracked_files': len(files), + 'paired_root': str(paired), 'original_lock': str(scratch / 'original-Cargo.lock')} + print(json.dumps(result, sort_keys=True)) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='action', required=True) + build = sub.add_parser('assemble') + for flag in ('libraries', 'mirror', 'libraries-sha', 'mirror-sha', 'scratch'): + build.add_argument('--' + flag, required=True) + build.add_argument('--pr-head-sha', default='') + lock = sub.add_parser('check-lock') + lock.add_argument('--before', required=True) + lock.add_argument('--after', required=True) + args = parser.parse_args() + if args.action == 'assemble': + assemble(Path(args.libraries), Path(args.mirror), args.libraries_sha, args.mirror_sha, + Path(args.scratch), args.pr_head_sha) + else: + validate_lock_delta(tomllib.loads(Path(args.before).read_text()), tomllib.loads(Path(args.after).read_text())) + print('scratch lock delta is exactly the declared PolyKV FibQuant dependency drop') + + +if __name__ == '__main__': + main() diff --git a/scripts/ci/test_paired_root.py b/scripts/ci/test_paired_root.py new file mode 100644 index 0000000..fbc4129 --- /dev/null +++ b/scripts/ci/test_paired_root.py @@ -0,0 +1,115 @@ +"""Fail-closed contract tests for the disposable paired-root CI projection.""" +from __future__ import annotations + +import copy +import gc +import subprocess +import tempfile +import unittest +import warnings +from pathlib import Path + +from scripts.ci import paired_root + + +class PairedRootTests(unittest.TestCase): + @staticmethod + def fixture_repo(path: Path, files: dict[str, str]) -> str: + path.mkdir() + subprocess.run(['git', 'init', '-q', str(path)], check=True) + subprocess.run(['git', '-C', str(path), 'config', 'user.name', 'Fixture'], check=True) + subprocess.run(['git', '-C', str(path), 'config', 'user.email', 'fixture@example.invalid'], check=True) + for name, content in files.items(): + target = path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + subprocess.run(['git', '-C', str(path), 'add', '--', '.'], check=True) + subprocess.run(['git', '-C', str(path), 'commit', '-qm', 'fixture'], check=True) + return subprocess.check_output(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True).strip() + + def test_paths_and_modes_cannot_escape_or_change_source_kind(self): + for path in ('../escape', '/absolute', 'src/../escape', 'src/./file', 'src//file', 'src\\escape', ''): + with self.subTest(path=path), self.assertRaises(ValueError): + paired_root.validate_entry(path, '100644') + for mode in ('120000', '160000', '040000'): + with self.subTest(mode=mode), self.assertRaises(ValueError): + paired_root.validate_entry('src/lib.rs', mode) + paired_root.validate_entry('src/lib.rs', '100644') + paired_root.validate_entry('examples/canary.rs', '100755') + + def test_lock_delta_allows_only_known_poly_kv_optional_dependency_removal(self): + before = {'version': 4, 'package': [ + {'name': 'poly-kv', 'version': '0.1.0-alpha.1', 'dependencies': ['fib-quant 0.1.0-alpha.1', 'serde']}, + {'name': 'semantic-memory', 'version': '0.5.15', 'dependencies': ['poly-kv']}, + ]} + after = copy.deepcopy(before) + after['package'][0]['dependencies'].remove('fib-quant 0.1.0-alpha.1') + paired_root.validate_lock_delta(before, after) + for mutation in ( + lambda doc: doc['package'][1].update(version='0.5.16'), + lambda doc: doc['package'][0]['dependencies'].remove('serde'), + lambda doc: doc['package'].append({'name': 'shadow', 'version': '1'}), + ): + with self.subTest(mutation=mutation): + invalid = copy.deepcopy(after) + mutation(invalid) + with self.assertRaises(ValueError): + paired_root.validate_lock_delta(before, invalid) + + def test_wrong_source_sha_is_rejected_before_assembly(self): + with self.assertRaises(ValueError): + paired_root.require_exact_sha('a' * 40, 'b' * 40, 'Libraries') + with self.assertRaises(ValueError): + paired_root.require_exact_sha('short', 'short', 'mirror') + paired_root.require_exact_sha('a' * 40, 'a' * 40, 'Libraries') + + def test_archive_member_rejects_duplicates_and_file_directory_collisions(self): + seen = {} + paired_root.claim_archive_member('src', 'directory', seen) + paired_root.claim_archive_member('src/lib.rs', 'file', seen) + with self.assertRaises(ValueError): + paired_root.claim_archive_member('src/lib.rs', 'file', seen) + with self.assertRaises(ValueError): + paired_root.claim_archive_member('src', 'file', seen) + paired_root.claim_archive_member('top', 'file', seen) + with self.assertRaises(ValueError): + paired_root.claim_archive_member('top/child', 'file', seen) + with self.assertRaises(ValueError): + paired_root.claim_archive_member('src/lib.rs/child', 'directory', seen) + + def test_pr_merge_must_name_exact_head_as_second_parent(self): + base = 'a' * 40 + head = 'b' * 40 + paired_root.require_pr_merge_parents([base, head], head) + for parents in ([base], [base, 'c' * 40], [head, base]): + with self.subTest(parents=parents), self.assertRaises(ValueError): + paired_root.require_pr_merge_parents(parents, head) + + def test_real_git_assembly_preserves_source_and_wrong_sha_leaves_no_scratch(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + owner, mirror = root / 'owner', root / 'mirror' + owner_sha = self.fixture_repo(owner, { + 'Cargo.lock': 'version = 4\n', + 'sibling.txt': 'canonical sibling', + 'semantic-memory/old.txt': 'old owner projection', + }) + mirror_sha = self.fixture_repo(mirror, {'src/lib.rs': 'mirror source'}) + scratch = root / 'pair' + with self.assertRaises(ValueError): + paired_root.assemble(owner, mirror, '0' * 40, mirror_sha, scratch) + self.assertFalse(scratch.exists()) + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter('always', ResourceWarning) + result = paired_root.assemble(owner, mirror, owner_sha, mirror_sha, scratch) + gc.collect() + self.assertFalse([w for w in captured if issubclass(w.category, ResourceWarning)]) + self.assertEqual(result['mirror_tracked_files'], 1) + self.assertEqual((scratch / 'Libraries/semantic-memory/src/lib.rs').read_text(), 'mirror source') + self.assertFalse((scratch / 'Libraries/semantic-memory/old.txt').exists()) + self.assertEqual((scratch / 'Libraries/sibling.txt').read_text(), 'canonical sibling') + self.assertEqual((scratch / 'original-Cargo.lock').read_text(), 'version = 4\n') + + +if __name__ == '__main__': + unittest.main() From ccc08cd14fbd45eab22c4b6a3179ed501e5660f0 Mon Sep 17 00:00:00 2001 From: Josh Stevenson Date: Mon, 28 Sep 2026 01:09:22 -0500 Subject: [PATCH 2/2] fix(ci): read PR merge parents from raw commit --- scripts/ci/paired_root.py | 17 +++++++++++++++-- scripts/ci/test_paired_root.py | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/scripts/ci/paired_root.py b/scripts/ci/paired_root.py index fdb1b65..fafbf5d 100644 --- a/scripts/ci/paired_root.py +++ b/scripts/ci/paired_root.py @@ -60,6 +60,20 @@ def git(repo: Path, *args: str) -> bytes: return subprocess.check_output(['git', '-C', str(repo), *args]) +def commit_parents(repo: Path, sha: str) -> list[str]: + # Git's pretty-format %P hides parents behind a shallow-checkout graft. + # The raw commit object retains both parent headers even at depth one. + header = git(repo, 'cat-file', '-p', sha).split(b'\n\n', 1)[0] + parents = [] + for line in header.splitlines(): + if line.startswith(b'parent '): + parent = line[len(b'parent '):].decode('ascii', 'strict') + if not re.fullmatch(r'[0-9a-f]{40}', parent): + raise ValueError('invalid parent in raw Git commit header') + parents.append(parent) + return parents + + def tracked_files(repo: Path, sha: str, *, strict: bool) -> dict[str, tuple[str, str]]: records: dict[str, tuple[str, str]] = {} for row in git(repo, 'ls-tree', '-r', '-z', sha).split(b'\0'): @@ -134,8 +148,7 @@ def assemble(libraries: Path, mirror: Path, libraries_sha: str, mirror_sha: str, require_exact_sha(git(libraries, 'rev-parse', 'HEAD').decode().strip(), libraries_sha, 'Libraries') require_exact_sha(git(mirror, 'rev-parse', 'HEAD').decode().strip(), mirror_sha, 'mirror') if pr_head_sha: - parents = git(mirror, 'show', '-s', '--format=%P', mirror_sha).decode().split() - require_pr_merge_parents(parents, pr_head_sha) + require_pr_merge_parents(commit_parents(mirror, mirror_sha), pr_head_sha) files = tracked_files(mirror, mirror_sha, strict=True) # Refuse to overwrite any existing source or artifact, including a symlink. scratch.mkdir(parents=True, exist_ok=False) diff --git a/scripts/ci/test_paired_root.py b/scripts/ci/test_paired_root.py index fbc4129..8a14b4b 100644 --- a/scripts/ci/test_paired_root.py +++ b/scripts/ci/test_paired_root.py @@ -85,6 +85,26 @@ def test_pr_merge_must_name_exact_head_as_second_parent(self): with self.subTest(parents=parents), self.assertRaises(ValueError): paired_root.require_pr_merge_parents(parents, head) + def test_commit_parent_headers_survive_shallow_checkout(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + origin = root / 'origin' + base = self.fixture_repo(origin, {'file.txt': 'base'}) + subprocess.run(['git', '-C', str(origin), 'checkout', '-qb', 'feature'], check=True) + (origin / 'file.txt').write_text('feature') + subprocess.run(['git', '-C', str(origin), 'commit', '-qam', 'feature'], check=True) + head = subprocess.check_output(['git', '-C', str(origin), 'rev-parse', 'HEAD'], text=True).strip() + subprocess.run(['git', '-C', str(origin), 'checkout', '-q', '--detach', base], check=True) + subprocess.run(['git', '-C', str(origin), 'merge', '--no-ff', '-qm', 'merge feature', 'feature'], check=True) + merged = subprocess.check_output(['git', '-C', str(origin), 'rev-parse', 'HEAD'], text=True).strip() + subprocess.run(['git', '-C', str(origin), 'branch', 'merge-fixture'], check=True) + shallow = root / 'shallow' + subprocess.run(['git', 'clone', '-q', '--branch', 'merge-fixture', '--depth', '1', + 'file://' + str(origin), str(shallow)], check=True) + self.assertEqual(subprocess.check_output( + ['git', '-C', str(shallow), 'show', '-s', '--format=%P', 'HEAD'], text=True).strip(), '') + self.assertEqual(paired_root.commit_parents(shallow, merged), [base, head]) + def test_real_git_assembly_preserves_source_and_wrong_sha_leaves_no_scratch(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary)