From 2eced8916e58c8b067465751f3450b80e9a10d66 Mon Sep 17 00:00:00 2001 From: Shane Burrell Date: Sat, 29 Aug 2026 22:41:31 -0400 Subject: [PATCH] Cover live sshd in e2e so fake SSH is not the only remote path. CI still stays green where localhost SSH is missing: the new script skips instead of failing. --- .github/workflows/ci.yml | 3 ++ AGENTS.md | 12 ++---- Makefile | 8 ++++ scripts/e2e-live.sh | 85 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 8 deletions(-) create mode 100755 scripts/e2e-live.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a07d714..c62f5d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,3 +133,6 @@ jobs: run: make build - name: Round trip a synthetic model over the local and SSH transports run: ./scripts/e2e.sh + - name: Round trip over live sshd (skips when localhost SSH is unavailable) + if: matrix.os == 'ubuntu-latest' + run: ./scripts/e2e-live.sh diff --git a/AGENTS.md b/AGENTS.md index 1909e43..6657439 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,19 +28,15 @@ make build `make check` runs fmt-check, vet, lint, and test (not race, cover, or e2e). -After any change that touches copy/sync/verify/SSH, also run a **live -sshd** round trip on this machine (not the fake SSH in `scripts/e2e.sh`): +After any change that touches copy/sync/verify/SSH, also run the live +sshd smoke (skips when `ubuntu@127.0.0.1` is unreachable): ```sh make build -SRC=$(mktemp -d); DST=/tmp/modelmove-live-dst -# build a tiny HF-shaped tree in $SRC, then: -./bin/modelmove copy "$SRC" "ubuntu@127.0.0.1:$DST" --remote-bin "$PWD/bin/modelmove" --no-progress -./bin/modelmove verify "$DST" --no-progress +./scripts/e2e.sh +./scripts/e2e-live.sh ``` -Localhost SSH must work as `ubuntu@127.0.0.1` without a password. - ## Layout | Path | Role | diff --git a/Makefile b/Makefile index 5bfecf6..68f8b53 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,14 @@ fmt-check: ## Fail if the tree is not formatted tidy: ## Tidy go.mod go mod tidy +.PHONY: e2e +e2e: build ## Local + fake-SSH smoke + ./scripts/e2e.sh + +.PHONY: e2e-live +e2e-live: build ## Live sshd smoke (skips if ubuntu@127.0.0.1 is unreachable) + ./scripts/e2e-live.sh + .PHONY: check check: fmt-check vet lint test ## Everything CI runs diff --git a/scripts/e2e-live.sh b/scripts/e2e-live.sh new file mode 100755 index 0000000..9065667 --- /dev/null +++ b/scripts/e2e-live.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Live-sshd smoke: same synthetic HF tree as e2e.sh, but the destination is +# ubuntu@127.0.0.1 over a real sshd. Skips cleanly when localhost SSH is +# unavailable so GitHub Actions and laptops without sshd stay green. +set -euo pipefail + +BIN=${BIN:-"$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/bin/modelmove"} +if [ ! -x "$BIN" ]; then + echo "e2e-live: $BIN not found; run 'make build' first" >&2 + exit 1 +fi + +REMOTE_USER=${MODELMOVE_LIVE_SSH_USER:-ubuntu} +REMOTE_HOST=${MODELMOVE_LIVE_SSH_HOST:-127.0.0.1} +SSH=(ssh -o BatchMode=yes -o ConnectTimeout=2 -o StrictHostKeyChecking=accept-new) + +if ! "${SSH[@]}" "${REMOTE_USER}@${REMOTE_HOST}" true >/dev/null 2>&1; then + echo "e2e-live: skip (no live ssh to ${REMOTE_USER}@${REMOTE_HOST})" + exit 0 +fi + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"; "${SSH[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "rm -rf /tmp/modelmove-live-$$" >/dev/null 2>&1 || true' EXIT + +REMOTE_DST="/tmp/modelmove-live-$$" +SRC="$WORK/src" +mkdir -p "$SRC" + +echo "==> e2e-live: synthetic Hugging Face checkpoint" +python3 - "$SRC" <<'PY' +import os, random, sys +root = sys.argv[1] +random.seed(1234) +with open(os.path.join(root, "config.json"), "w") as f: + f.write('{"model_type":"llama","hidden_size":4096}') +with open(os.path.join(root, "tokenizer.json"), "w") as f: + f.write('{"vocab":[]}' + "x" * 20000) +with open(os.path.join(root, "model.safetensors.index.json"), "w") as f: + f.write('{"weight_map":{}}') +for i in (1, 2): + name = f"model-0000{i}-of-00002.safetensors" + with open(os.path.join(root, name), "wb") as f: + f.write(random.randbytes(4_000_000)) +PY + +fail() { echo "e2e-live: $1" >&2; exit 1; } + +TARGET="${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DST}" + +echo "==> e2e-live: copy over live sshd" +"$BIN" copy "$SRC" "$TARGET" --remote-bin "$BIN" --no-progress +"${SSH[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "'$BIN' verify '$REMOTE_DST' --no-progress" + +echo "==> e2e-live: unchanged resync should move nothing" +"$BIN" sync "$SRC" "$TARGET" --remote-bin "$BIN" --json --no-progress > "$WORK/noop.json" +python3 - "$WORK/noop.json" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +sent = r["summary"]["bytes_received"] +assert sent == 0, f"an unchanged resync sent {sent} bytes" +PY + +echo "==> e2e-live: sparse update after a small edit" +python3 - "$SRC/model-00001-of-00002.safetensors" <<'PY' +import sys +p = sys.argv[1] +d = bytearray(open(p, "rb").read()) +d[2_000_000:2_000_064] = b"FINETUNED" * 7 + b"!" +open(p, "wb").write(bytes(d)) +PY + +"$BIN" sync "$SRC" "$TARGET" --remote-bin "$BIN" --json --no-progress > "$WORK/delta.json" +python3 - "$WORK/delta.json" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +total = r["plan"]["total_bytes"] +sent = r["summary"]["bytes_received"] +assert sent < total // 8, f"a 64-byte edit moved {sent} of {total} bytes" +print(f" live ssh: moved {sent} of {total} bytes") +PY + +"${SSH[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "'$BIN' verify '$REMOTE_DST' --no-progress" + +echo +echo "e2e-live: all checks passed"