diff --git a/.github/workflows/github-r2-fallback.yml b/.github/workflows/github-r2-fallback.yml new file mode 100644 index 00000000..ca47d53c --- /dev/null +++ b/.github/workflows/github-r2-fallback.yml @@ -0,0 +1,187 @@ +name: GitHub R2 fallback and typed RPC canary + +on: + pull_request: + paths: + - ".github/workflows/github-r2-fallback.yml" + - "scripts/github_r2_fallback.py" + - "docs/github-r2-fallback-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit + required: false + type: string + zed_interfaces_ref: + description: Exact 40-character zed-pkg/zed-interfaces commit + required: false + type: string + workflow_call: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit + required: true + type: string + zed_interfaces_ref: + description: Exact 40-character zed-pkg/zed-interfaces commit + required: true + type: string + +permissions: + contents: read + +concurrency: + group: github-r2-fallback-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + syntax: + name: Canary script compiles + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: py_compile + run: python3 -m py_compile scripts/github_r2_fallback.py + + canary: + name: Registry-down CDN install + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Resolve product pins + id: pins + env: + CLI_IN: ${{ inputs.zed_cli_ref || '' }} + INTERFACES_IN: ${{ inputs.zed_interfaces_ref || '' }} + run: | + set -euo pipefail + cli="${CLI_IN:-}" + interfaces="${INTERFACES_IN:-}" + if [[ -z "$cli" ]]; then + echo "zed_cli_ref is required (workflow_call or workflow_dispatch)" >&2 + exit 1 + fi + if [[ -z "$interfaces" ]]; then + echo "zed_interfaces_ref is required (workflow_call or workflow_dispatch)" >&2 + exit 1 + fi + [[ "$cli" =~ ^[0-9a-f]{40}$ ]] + [[ "$interfaces" =~ ^[0-9a-f]{40}$ ]] + printf 'cli=%s\n' "$cli" >> "$GITHUB_OUTPUT" + printf 'interfaces=%s\n' "$interfaces" >> "$GITHUB_OUTPUT" + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pins.outputs.cli }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out matching zed-interfaces + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ steps.pins.outputs.interfaces }} + path: product/zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-clients contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-clients + path: product/zed-clients + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-lib-core contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-lib-core + path: product/zed-lib-core + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable product identity + run: | + set -euo pipefail + test "$(git -C product/zed-cli rev-parse HEAD)" = "${{ steps.pins.outputs.cli }}" + test "$(git -C product/zed-interfaces rev-parse HEAD)" = "${{ steps.pins.outputs.interfaces }}" + + - name: Install Rust + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt + + - name: Install Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v6.0.0 + with: + node-version: "22" + + - name: Contract tests (interfaces, clients, lib-core) + run: | + set -euo pipefail + cargo test --manifest-path product/zed-interfaces/src/rust/Cargo.toml --lib zed_api + cargo test --manifest-path product/zed-lib-core/src/rust-orm/Cargo.toml \ + --features read-write \ + publication::tests::canonical_artifact_key_is_content_addressed + (cd product/zed-clients/clients/typescript && npm ci --ignore-scripts && npm run build && npm test) + + - name: Build exact zed CLI + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path product/zed-cli/Cargo.toml + cargo build --release --manifest-path product/zed-cli/Cargo.toml --bin zed + install -Dm755 product/zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run registry-down CDN canary + run: | + set -euo pipefail + python3 -m py_compile scripts/github_r2_fallback.py + python3 scripts/github_r2_fallback.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$RUNNER_TEMP/github-r2-fallback" \ + --route-map product/zed-interfaces/route-maps/zed-api.route-map.json \ + --interfaces-generated-ts product/zed-interfaces/generated/typescript/zed_api.ts \ + --clients-generated-ts product/zed-clients/clients/typescript/src/generated/zed-api.routes.ts + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: github-r2-fallback-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/github-r2-fallback + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/README.md b/README.md index f28bb00e..d251ffe5 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,20 @@ python3 scripts/lifecycle.py \ The work root is disposable. Reusing the same path is safe because the harness requires a fresh root and GitHub-hosted jobs receive a new runner filesystem. + +## GitHub / R2 fallback (registry down, CDN up) + +`.github/workflows/github-r2-fallback.yml` is a reusable canary for the +`cdn.zpkg.net` guessable-object contract. `scripts/github_r2_fallback.py` +publishes a disposable package to a `file://` registry, copies the tarball onto +`packages/` and `github/` keys, points `--registry` at a closed loopback port, +and proves `zed install --frozen` restores the packed payload from a loopback +CDN. It also round-trips the typed `get_version` RPC frame over TCP NDJSON +(from `github.com/oresoftware/api-docs`). Pull requests only compile the +script; a full product run needs exact `zed-cli` and `zed-interfaces` commits +via `workflow_call` or `workflow_dispatch`. See +`docs/github-r2-fallback-canary.md`. + ## Durable first-install acceptance `.github/workflows/durable-first-install.yml` is the cross-organization diff --git a/docs/github-r2-fallback-canary.md b/docs/github-r2-fallback-canary.md new file mode 100644 index 00000000..b0000334 --- /dev/null +++ b/docs/github-r2-fallback-canary.md @@ -0,0 +1,41 @@ +# GitHub / R2 registry-fallback canary + +This repository proves that `zed install --frozen` still materializes a +package when the HTTP registry is down, as long as the packed tarball is at a +guessable public CDN key (`packages/{org}/{name}/{version}/…` or +`github/{owner}/{repo}/{tag}/…`). That is the `cdn.zpkg.net` contract. + +The canary is credential-free and uses only loopback HTTP: + +1. publish a disposable package to a `file://` registry; +2. copy the tarball onto the guessable CDN layout; +3. install once so a lock exists, then uninstall and drop the store; +4. point `--registry` at a closed `127.0.0.1` port; +5. set `ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK=true` (production loopback stays + hermetic) and `--r2-public-base http://127.0.0.1:`; +6. `zed install --frozen` must restore the packed payload from the CDN mock. + +It also round-trips the `get_version` JSON call/receipt frame over TCP NDJSON, +using the same key as `zed-interfaces/route-maps/zed-api.route-map.json` +(generated by `github.com/oresoftware/api-docs`). zed-cli itself still speaks +HTTP; the TCP check is the typed RPC wire, not a second CLI transport. + +When the TypeScript copies are passed in, the generated `zed-clients` routes +file must match `zed-interfaces/generated/typescript/zed_api.ts` (banner +comments excluded). + +## Local run + +From a sibling checkout of `zed-pkg`: + +```bash +cargo build --release --manifest-path ../zed-pkg/zed-cli/Cargo.toml --bin zed +python3 scripts/github_r2_fallback.py \ + --zed ../zed-pkg/zed-cli/target/release/zed \ + --work-root /tmp/zed-github-r2-fallback \ + --route-map ../zed-pkg/zed-interfaces/route-maps/zed-api.route-map.json \ + --interfaces-generated-ts ../zed-pkg/zed-interfaces/generated/typescript/zed_api.ts \ + --clients-generated-ts ../zed-pkg/zed-clients/clients/typescript/src/generated/zed-api.routes.ts +``` + +The work root must not already exist. diff --git a/scripts/github_r2_fallback.py b/scripts/github_r2_fallback.py new file mode 100644 index 00000000..80419787 --- /dev/null +++ b/scripts/github_r2_fallback.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Black-box canary: registry down, public CDN still installs. + +Publishes a disposable package through a file:// registry, copies the packed +tarball onto the guessable R2 keys (`packages/…` and `github/…`), takes the +HTTP registry down, and proves `zed install --frozen` fetches from a loopback +stand-in for `https://cdn.zpkg.net`. Also round-trips the oresoftware/api-docs +JSON call/receipt frame for `get_version` over TCP NDJSON. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.server +import json +import os +import re +import shutil +import socket +import socketserver +import subprocess +import sys +import threading +from pathlib import Path +from typing import Mapping, Sequence + + +ORG = "zed-pkg-test" +NAME = "fallback-canary" +VERSION = "0.0.1" +TAG = f"v{VERSION}" +FILENAME = f"{NAME}-{VERSION}.tar.gz" + + +def shell_quote(value: str) -> str: + if not value: + return "''" + if re.fullmatch(r"[A-Za-z0-9_@%+=:,./-]+", value): + return value + return "'" + value.replace("'", "'\\''") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path, + extra_env: Mapping[str, str] | None = None, +) -> str: + argv = [str(value) for value in command] + print(f"\n$ (cd {cwd} && {' '.join(shell_quote(value) for value in argv)})", flush=True) + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_TOKEN": "", + "ZED_PKG_INTERACTIVE": "false", + } + ) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(completed.stdout, end="", flush=True) + if completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {argv!r}\n{completed.stdout}" + ) + return completed.stdout + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def expand(template: str, **values: str) -> str: + out = template + for key, value in values.items(): + out = out.replace("{" + key + "}", value) + return out + + +def load_route_map(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_package(directory: Path) -> None: + directory.mkdir(parents=True) + (directory / "src").mkdir() + (directory / "src" / "payload.txt").write_text("cdn-fallback\n", encoding="utf-8") + (directory / ".zpkg.toml").write_text( + f'''[package] +org = "{ORG}" +name = "{NAME}" +version = "{VERSION}" +description = "Disposable GitHub/R2 fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{NAME}" + +[install] +dir = "zed_modules" +adapter = "none" +''', + encoding="utf-8", + ) + run(["git", "init", "-b", "main"], cwd=directory) + run(["git", "config", "user.email", "canary@zed-pkg-test.invalid"], cwd=directory) + run(["git", "config", "user.name", "zed-pkg-test canary"], cwd=directory) + run(["git", "add", "-A"], cwd=directory) + run(["git", "commit", "-m", "canary package"], cwd=directory) + commit = run(["git", "rev-parse", "HEAD"], cwd=directory).strip() + run(["git", "tag", TAG, commit], cwd=directory) + + +def write_consumer(directory: Path) -> None: + directory.mkdir(parents=True) + (directory / ".zpkg.toml").write_text( + f'''[package] +org = "zed-pkg-test" +name = "fallback-consumer" +version = "0.0.0" +description = "Consumes the GitHub/R2 fallback canary" + +[package.repository] +vcs = "git" +url = "https://github.com/zed-pkg-test/fallback-consumer" + +[install] +dir = "zed_modules" +adapter = "none" + +[dependencies] +"{ORG}/{NAME}" = "={VERSION}" +''', + encoding="utf-8", + ) + + +def start_cdn(root: Path, port: int) -> http.server.ThreadingHTTPServer: + class BoundHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(root), **kwargs) + + def log_message(self, format: str, *args) -> None: # noqa: A003 + print(f"cdn: {format % args}", flush=True) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", port), BoundHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +class VersionRpcHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + raw = self.rfile.readline() + call = json.loads(raw) + if call.get("v") != 1 or call.get("op") != "call" or call.get("key") != "get_version": + receipt = { + "v": 1, + "op": "receipt", + "id": call.get("id", ""), + "key": call.get("key", ""), + "transport": "tcp", + "ok": False, + "status": 400, + "error": {"code": "bad_call"}, + } + else: + path = call.get("path") or {} + receipt = { + "v": 1, + "op": "receipt", + "id": call["id"], + "key": "get_version", + "transport": "tcp", + "ok": True, + "status": 200, + "body": { + "org": path.get("org", ORG), + "name": path.get("name", NAME), + "version": path.get("version", VERSION), + "yanked": False, + }, + } + line = json.dumps(receipt, separators=(",", ":")) + "\n" + self.wfile.write(line.encode("utf-8")) + + +def start_tcp_rpc(port: int) -> socketserver.ThreadingTCPServer: + server = socketserver.ThreadingTCPServer(("127.0.0.1", port), VersionRpcHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +def tcp_get_version(port: int) -> dict: + call = { + "v": 1, + "op": "call", + "id": "e2e-get-version", + "key": "get_version", + "transport": "tcp", + "path": {"org": ORG, "name": NAME, "version": VERSION}, + } + payload = json.dumps(call, separators=(",", ":")) + "\n" + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload.encode("utf-8")) + sock.shutdown(socket.SHUT_WR) + raw = b"" + while not raw.endswith(b"\n"): + chunk = sock.recv(4096) + if not chunk: + break + raw += chunk + receipt = json.loads(raw.decode("utf-8")) + if not receipt.get("ok"): + raise AssertionError(f"tcp get_version failed: {receipt!r}") + return receipt + + +def copy_guessable_objects(artifact: Path, cdn_root: Path) -> None: + targets = [ + cdn_root / "packages" / ORG / NAME / VERSION / FILENAME, + cdn_root / "github" / ORG / NAME / TAG / FILENAME, + ] + for target in targets: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(artifact, target) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + parser.add_argument( + "--route-map", + type=Path, + help="zed-api.route-map.json (defaults to a sibling zed-interfaces checkout)", + ) + parser.add_argument( + "--interfaces-generated-ts", + type=Path, + help="zed-interfaces generated TypeScript to compare with zed-clients", + ) + parser.add_argument( + "--clients-generated-ts", + type=Path, + help="zed-clients copy of the generated TypeScript routes", + ) + return parser.parse_args() + + +def assert_route_map(route_map: dict) -> None: + mapping = route_map["map"] + get_version = mapping["get_version"] + if get_version["path"] != "/v1/packages/{org}/{name}/versions/{version}": + raise AssertionError(get_version["path"]) + if get_version["transports"] != ["http", "tcp"]: + raise AssertionError(get_version["transports"]) + if get_version.get("tcp_framing") != "ndjson": + raise AssertionError("get_version must use ndjson on TCP") + if mapping["registry_events"]["transports"] != ["websocket"]: + raise AssertionError(mapping["registry_events"]) + package_path = expand(mapping["get_package"]["path"], org=ORG, name=NAME) + if package_path != f"/v1/packages/{ORG}/{NAME}": + raise AssertionError(package_path) + cdn_path = expand( + mapping["cdn_package_object"]["path"], + org=ORG, + name=NAME, + version=VERSION, + filename=FILENAME, + ) + if cdn_path != f"/packages/{ORG}/{NAME}/{VERSION}/{FILENAME}": + raise AssertionError(cdn_path) + + +def strip_ts_banner(text: str) -> str: + return re.sub(r"^/\*\*.*?\*/\s*", "", text, count=1, flags=re.S).strip() + "\n" + + +def parse_args_and_compare_generated(args: argparse.Namespace) -> None: + if args.interfaces_generated_ts and args.clients_generated_ts: + left = strip_ts_banner(args.interfaces_generated_ts.read_text(encoding="utf-8")) + right = strip_ts_banner(args.clients_generated_ts.read_text(encoding="utf-8")) + if left != right: + raise AssertionError( + "zed-clients generated routes drifted from zed-interfaces generated TypeScript" + ) + + +def main() -> int: + args = parse_args() + zed = args.zed.resolve() + root = args.work_root.resolve() + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + + default_map = ( + Path(__file__).resolve().parents[3] + / "zed-pkg" + / "zed-interfaces" + / "route-maps" + / "zed-api.route-map.json" + ) + route_map_path = (args.route_map or default_map).resolve() + if not route_map_path.is_file(): + raise AssertionError(f"route map not found: {route_map_path}") + route_map = load_route_map(route_map_path) + assert_route_map(route_map) + parse_args_and_compare_generated(args) + + root.mkdir(parents=True) + registry = root / "file-registry" + registry.mkdir() + registry_url = f"file://{registry}" + package = root / "package" + consumer = root / "consumer" + publish_home = root / "publish-home" + install_home = root / "install-home" + fallback_home = root / "fallback-home" + cdn_root = root / "cdn" + cdn_root.mkdir() + + write_package(package) + write_consumer(consumer) + + def zed_cmd(*command: str | Path, cwd: Path, home: Path, extra_env: Mapping[str, str] | None = None) -> str: + return run( + [zed, "--registry", registry_url, "--home", home, *command], + cwd=cwd, + extra_env=extra_env, + ) + + zed_cmd("publish", cwd=package, home=publish_home) + metadata_path = registry / "packages" / ORG / NAME / "versions" / f"{VERSION}.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + sha256 = str(metadata["sha256"]) + artifact = registry / "artifacts" / f"{sha256}.tar.gz" + if not artifact.is_file(): + raise AssertionError(f"published artifact missing: {artifact}") + copy_guessable_objects(artifact, cdn_root) + content_copy = cdn_root / "artifacts" / f"{sha256}.tar.gz" + content_copy.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(artifact, content_copy) + + zed_cmd( + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + cwd=consumer, + home=install_home, + ) + lock = consumer / ".zpkg.lock" + lock_bytes = lock.read_bytes() + payload = consumer / "zed_modules" / ORG / NAME / "src" / "payload.txt" + if payload.read_text(encoding="utf-8") != "cdn-fallback\n": + raise AssertionError("file-registry install did not materialize the canary") + zed_cmd("uninstall", cwd=consumer, home=install_home) + if lock.read_bytes() != lock_bytes: + raise AssertionError("uninstall mutated the lock") + shutil.rmtree(install_home) + if (consumer / "zed_modules").exists(): + raise AssertionError("uninstall left materialization behind") + + dead_port = free_port() + cdn_port = free_port() + tcp_port = free_port() + cdn = start_cdn(cdn_root, cdn_port) + rpc = start_tcp_rpc(tcp_port) + try: + receipt = tcp_get_version(tcp_port) + if receipt["body"]["version"] != VERSION: + raise AssertionError(receipt) + if receipt["transport"] != "tcp": + raise AssertionError(receipt) + + fallback_registry = f"http://127.0.0.1:{dead_port}" + cdn_base = f"http://127.0.0.1:{cdn_port}" + run( + [ + zed, + "--registry", + fallback_registry, + "--home", + fallback_home, + "--r2-public-base", + cdn_base, + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env={ + "ZED_PKG_SOURCE_FALLBACK": "true", + "ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK": "true", + "ZED_PKG_R2_PUBLIC_BASE": cdn_base, + }, + ) + restored = consumer / "zed_modules" / ORG / NAME / "src" / "payload.txt" + if restored.read_text(encoding="utf-8") != "cdn-fallback\n": + raise AssertionError("CDN fallback did not restore the packed payload") + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + if digest != sha256: + raise AssertionError("packed artifact digest drifted") + print( + json.dumps( + { + "ok": True, + "registry": fallback_registry, + "cdn": cdn_base, + "tcp_rpc": f"127.0.0.1:{tcp_port}", + "sha256": sha256, + "cdn_keys": [ + f"packages/{ORG}/{NAME}/{VERSION}/{FILENAME}", + f"github/{ORG}/{NAME}/{TAG}/{FILENAME}", + f"artifacts/{sha256}.tar.gz", + ], + }, + indent=2, + ), + flush=True, + ) + finally: + cdn.shutdown() + rpc.shutdown() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 — canary must print the failure + print(f"github-r2-fallback canary failed: {error}", file=sys.stderr) + raise