diff --git a/README.md b/README.md index 9702e7acd..c57613bec 100644 --- a/README.md +++ b/README.md @@ -121,11 +121,20 @@ resources): ```bash yeto launch --training-mode rl --gpu aws:8xa100@us-east-1,aws:8xa100@us-west-2 \ - --model org/model --data org/prompts \ - --reward-function project.rewards:score \ + --model Qwen/Qwen3-1.7B --model-revision 70d244cc86ccca08cf5af4e1e306ecf908b1ad5e \ + --data HuggingFaceH4/MATH-500 --data-revision 6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be \ + --rl-prompt-column problem --rl-label-column answer \ + --reward-function yeto.rl.math_reward:reward_func --trust-remote-code \ + --tuning lora --lora-targets attention --inner-lr 1e-5 \ --rollout-batch-size 16 --n-samples-per-prompt 4 --total-steps 8 ``` +RL needs immutable inputs: pass the 40-character Hub commits with +`--model-revision` / `--data-revision`. `--reward-function` takes a built-in +reward (`yeto.rl.*`, listed in [examples/README.md](examples/README.md)), any +`package.module:function` importable from the repository root or the +environment, or a `path/to/file.py:function` inside the repository. + The `--gpu` grammar, data formats, `--output`, QLoRA, resume/branch and every flag are described in [docs/CLI_REFERENCE.md](docs/CLI_REFERENCE.md). @@ -161,6 +170,10 @@ learners directly; see | [MILES_RL.md](docs/MILES_RL.md) | Miles RL: rollout/training boundaries, recovery, export, all RL flags | | [RL_ALGORITHMS.md](docs/RL_ALGORITHMS.md) | RL algorithm specs | | [CLOUDS.md](docs/CLOUDS.md) | per-cloud credentials, regions, planner signals, Modal islands | +| [KIMI_K31_ASYNC_RL_GUIDE.md](docs/KIMI_K31_ASYNC_RL_GUIDE.md) | Kimi K3.1 async GRPO (agentic RL) end to end: Modal trainer, NC samplers, run config, dashboard, operations | +| [K31_FULL_262K_LAYOUT.md](docs/K31_FULL_262K_LAYOUT.md) | Kimi K3.1 128-expert full-parameter trainer layout on 6x8 B300 | +| [NC_POOL.md](docs/NC_POOL.md) | National Compute sampler pool: config, dry run, status, stop, billing | +| [DISPENSER.md](docs/DISPENSER.md) | task dispenser over a shared bucket (store dispenser) | | [ASCEND.md](docs/ASCEND.md) | Ascend NPU islands and a pure-Ascend fleet | | [MEGATRON.md](docs/MEGATRON.md) | Megatron-Core island backend for large MoE | | [MLX.md](docs/MLX.md) | Apple-silicon islands | diff --git a/docker/image-assert/assert_image_versions.py b/docker/image-assert/assert_image_versions.py index 2166038f1..64c81bafd 100755 --- a/docker/image-assert/assert_image_versions.py +++ b/docker/image-assert/assert_image_versions.py @@ -130,13 +130,34 @@ def check_cann(cann: dict | None, report: Report) -> None: report.add("cann", have.startswith(cann["version"]), f"{have} from {source} (lock {cann['version']})") +def _nvidia_smi_driver() -> tuple[str | None, str | None]: + """The host driver as nvidia-smi reports it: sandboxed runtimes (Modal's gVisor) + mount the driver libraries but no /proc/driver/nvidia/version.""" + import shutil + import subprocess + + smi = shutil.which("nvidia-smi") + if not smi: + return None, None + try: + out = subprocess.run([smi, "--query-gpu=driver_version", "--format=csv,noheader"], + capture_output=True, text=True, timeout=30).stdout + except (OSError, subprocess.SubprocessError): + return None, None + first = out.strip().splitlines()[0].strip() if out.strip() else "" + return (first, "nvidia-smi") if re.fullmatch(r"[0-9][0-9.]*", first) else (None, None) + + def check_driver(driver: dict | None, report: Report) -> None: if not driver: return have, source = _read_version([driver["version_file"]]) + if have is None: + have, source = _nvidia_smi_driver() want = driver.get("min") if have is None: - report.add("driver", False, f"no version in {driver['version_file']} (is the host driver mounted?)") + report.add("driver", False, f"no version in {driver['version_file']} and no nvidia-smi answer " + "(is the host driver mounted?)") elif want is None: report.add("driver", True, f"{have} from {source} (recorded; no minimum in the lock)") else: diff --git a/docker/miles-k31/build.sh b/docker/miles-k31/build.sh new file mode 100755 index 000000000..4acc00bc7 --- /dev/null +++ b/docker/miles-k31/build.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Build + push the yeto Kimi K3.1 Miles image without a docker daemon: crane +# appends one overlay layer (docker/miles-ports/make_layer.py) to the upstream +# radixark/miles:kimi-k31 base pinned by digest in versions.lock.json. +# +# docker/miles-k31/build.sh [--push] +# +# The overlay holds (versions.lock.json "decisions"): +# * the Miles fork branch over /root/miles (MILES_BRANCH, default k31/f1) +# * the SGLang fork branch over /sgl-workspace/sglang (SGLANG_BRANCH, default yeto/k31) +# * peft and pyrage wheels unpacked into the base's site-packages +# * the lazy-modelopt patch (Megatron-LM) and Megatron-Bridge __init__ +# * /opt/yeto/image-manifest.json and the image-assert lock under /opt/yeto/image +# No TMS preload: the base's torch_memory_saver already carries the fix. +# +# Inputs (env): +# MILES_SRC / SGLANG_SRC required: local clones of the forks (fetch the branches first) +# MILES_BRANCH Miles fork branch or commit (default k31/f1) +# SGLANG_BRANCH SGLang fork branch or commit (default yeto/k31) +# MILES_ORIGIN / SGLANG_ORIGIN fork repository URLs written as the checkouts' origin +# (default: yeto.rl.adapters.miles.pins MILES_NEXT_REPOSITORY / SGLANG_NEXT_REPOSITORY) +# SGLANG_TAG the SGLang release tag the fork descends from (default v0.5.23) +# MEGATRON_LM_SRC clone containing the base's Megatron-LM commit (default: shallow fetch into $OUT) +# PEFT_VERSION / PYRAGE_VERSION wheels to add (defaults 0.20.0 / 1.4.0) +# TAG_SUFFIX appended to the k31-- tag (rebuilds of the same commits) +# ALLOW_INSTALL_METADATA_DRIFT=1 accept setup.py/requirements* differences between a fork +# and the base's checkout (the base's dist-info is kept; the +# paths are recorded in image-manifest.json). k31/integ needs it: +# its setup.py adds an extras_require entry and requirements.txt +# adds pyrage, which this layer ships as a wheel. +# DEST_REPO required for --push: target repository, e.g. ghcr.io//yeto-miles-k31 +# CRANE crane binary (go-containerregistry >= 0.20) +# OUT work directory (default: mktemp) +# Registry auth: crane reads ~/.docker/config.json; nothing is printed. +# Writes: $OUT/{layer.tar,image-manifest.json,commits.json,build-record.json} +set -euo pipefail +cd "$(dirname "$0")/../.." +PUSH=0; [ "${1:-}" = "--push" ] && PUSH=1 +CRANE=${CRANE:-crane} +LOCK=docker/miles-k31/versions.lock.json +MILES_SRC=${MILES_SRC:?set MILES_SRC to a local Miles fork clone} +SGLANG_SRC=${SGLANG_SRC:?set SGLANG_SRC to a local SGLang fork clone} +MILES_BRANCH=${MILES_BRANCH:-k31/f1} +SGLANG_BRANCH=${SGLANG_BRANCH:-yeto/k31} +SGLANG_TAG=${SGLANG_TAG:-v0.5.23} +PEFT_VERSION=${PEFT_VERSION:-0.20.0} +PYRAGE_VERSION=${PYRAGE_VERSION:-1.4.0} +OUT=${OUT:-$(mktemp -d /tmp/img-k31-XXXXXX)} +if [ $PUSH = 1 ]; then DEST_REPO=${DEST_REPO:?set DEST_REPO for --push}; else DEST_REPO=${DEST_REPO:-yeto-miles-k31}; fi + +lock() { python3 -c 'import json, sys; d = json.load(open(sys.argv[1])); +for k in sys.argv[2].split("."): d = d[k] +print(d)' "$LOCK" "$1"; } +BASE_REPO=$(lock base_image.name | cut -d: -f1) +BASE_INDEX_DIGEST=$(lock base_image.index_digest) +BASE_AMD64_DIGEST=$(lock base_image.manifest_digest) +BASE="$BASE_REPO@$BASE_AMD64_DIGEST" + +# Fork repositories (origin of the shipped checkouts), from the pins unless overridden. +# (pins.py is a stdlib-only constants module; run it instead of grepping, so an +# alias such as SGLANG_NEXT_REPOSITORY = SGLANG_K31_REPOSITORY resolves.) +pins=$(python3 - <<'PY' +import runpy +ns = runpy.run_path("yeto/rl/adapters/miles/pins.py") +for k in ("MILES_NEXT_REPOSITORY", "SGLANG_NEXT_REPOSITORY"): + print(f"{k}={ns[k]}") +PY +) +eval "$pins" +MILES_ORIGIN=${MILES_ORIGIN:-$MILES_NEXT_REPOSITORY} +SGLANG_ORIGIN=${SGLANG_ORIGIN:-$SGLANG_NEXT_REPOSITORY} + +# 1. The base as the registry describes it (read-only): manifest, config, and +# the build args that name the checkouts the base installed. Nothing here is +# written into the repository: it is re-read on every build and checked +# against the lock. +"$CRANE" manifest "$BASE" > "$OUT/base-manifest.json" +"$CRANE" config "$BASE" > "$OUT/base-config.json" +base_arg() { # value of an ARG the base recorded in its build history + python3 - "$OUT/base-config.json" "$1" <<'PY' +import json, re, sys +cfg = json.load(open(sys.argv[1])); name = sys.argv[2] +seen = [] +for h in cfg.get("history", []): + for m in re.finditer(rf"\b{name}=(\S+)", h.get("created_by", "")): + seen.append(m.group(1).rstrip('"')) +print(seen[-1] if seen else "") +PY +} +BASE_MILES_COMMIT=$(base_arg MILES_COMMIT) +BASE_SGLANG_COMMIT=$(base_arg SGLANG_COMMIT) +MEGATRON_LM_COMMIT=$(base_arg MEGATRON_COMMIT) +for v in BASE_MILES_COMMIT BASE_SGLANG_COMMIT MEGATRON_LM_COMMIT; do + [ -n "${!v}" ] || { echo "$v: not in the base image's build history" >&2; exit 1; } +done +for pair in "SGLANG_COMMIT=$BASE_SGLANG_COMMIT" "MILES_COMMIT=$BASE_MILES_COMMIT" "MEGATRON_COMMIT=$MEGATRON_LM_COMMIT"; do + want=$(lock "base_build_args.${pair%%=*}") + [ "$want" = "${pair#*=}" ] || { echo "$LOCK says ${pair%%=*}=$want, the base says ${pair#*=}" >&2; exit 1; } +done +# torch_memory_saver and Megatron-Bridge: the commits pip installed from git +TMS_COMMIT=$(grep -o 'torch_memory_saver\.git@[0-9a-f]\{40\}' "$OUT/base-config.json" | tail -1 | cut -d@ -f2) +MEGATRON_BRIDGE_COMMIT=$(grep -o 'Megatron-Bridge\.git@[0-9a-f]\{40\}' "$OUT/base-config.json" | tail -1 | cut -d@ -f2 || true) +if [ -z "$MEGATRON_BRIDGE_COMMIT" ]; then + MEGATRON_BRIDGE_COMMIT=$(grep -o 'MEGATRON_BRIDGE_COMMIT=[0-9a-f]\{40\}' "$OUT/base-config.json" | tail -1 | cut -d= -f2 || true) +fi +[ -n "$MEGATRON_BRIDGE_COMMIT" ] || { echo "Megatron-Bridge commit: not in the base image's build history; set MEGATRON_BRIDGE_COMMIT" >&2; exit 1; } +want_tms=$(lock commits.torch_memory_saver) +[ "$want_tms" = "$TMS_COMMIT" ] || { echo "$LOCK pins torch_memory_saver $want_tms, the base installed ${TMS_COMMIT:-?}" >&2; exit 1; } + +# 2. Fork commits and the pure-Python check (make_layer refuses anything else). +MILES_COMMIT=$(git -C "$MILES_SRC" rev-parse "$MILES_BRANCH^{commit}") +SGLANG_COMMIT=$(git -C "$SGLANG_SRC" rev-parse "$SGLANG_BRANCH^{commit}") +git -C "$MILES_SRC" cat-file -e "$BASE_MILES_COMMIT^{commit}" || { echo "fetch the base Miles commit $BASE_MILES_COMMIT into $MILES_SRC" >&2; exit 1; } +git -C "$SGLANG_SRC" cat-file -e "$BASE_SGLANG_COMMIT^{commit}" || { echo "fetch the base SGLang commit $BASE_SGLANG_COMMIT into $SGLANG_SRC" >&2; exit 1; } + +# 3. The base's sglang editable metadata, from the layer that wrote it (found by +# scanning; no layer number anywhere). +META="$OUT/base-sglang-meta" +if [ ! -d "$META" ]; then + layer="" + for digest in $(python3 docker/miles-ports/make_layer.py find-layer \ + --manifest "$OUT/base-manifest.json" --config "$OUT/base-config.json" \ + | python3 -c 'import json, sys; print("\n".join(l["digest"] for l in json.load(sys.stdin)))'); do + if "$CRANE" blob "$BASE_REPO@$digest" | tar -tzf - 2>/dev/null \ + | python3 docker/miles-ports/make_layer.py check-listing; then + layer=$digest; break + fi + done + [ -n "$layer" ] || { echo "no base layer holds the sglang editable-install metadata" >&2; exit 1; } + mkdir -p "$OUT/meta-x" + "$CRANE" blob "$BASE_REPO@$layer" | tar -xzf - -C "$OUT/meta-x" \ + --wildcards "opt/sglang/lib/python3.12/site-packages/__editable___sglang_*_finder.py" \ + "opt/sglang/lib/python3.12/site-packages/__editable__.sglang-*.pth" \ + "opt/sglang/lib/python3.12/site-packages/sglang-*.dist-info/*" + mv "$OUT/meta-x/opt/sglang/lib/python3.12/site-packages" "$META" +fi +BASE_SGLANG_VERSION=$(ls "$META" | sed -n 's/^sglang-\(.*\)\.dist-info$/\1/p' | head -1) +[ -n "$BASE_SGLANG_VERSION" ] || { echo "no sglang-*.dist-info in $META" >&2; exit 1; } + +# setuptools-scm version the fork would get from a real `pip install -e`: +# v0.5.23--g -> 0.5.24.dev+g; a fork without the tag keeps +# the base's public version plus the local part. +if desc=$(git -C "$SGLANG_SRC" describe --tags --long --match "$SGLANG_TAG" "$SGLANG_COMMIT" 2>/dev/null); then + n=$(echo "$desc" | awk -F- '{print $(NF-1)}') + base_public=${SGLANG_TAG#v} + next=$(python3 -c 'import sys; p = sys.argv[1].split("."); p[-1] = str(int(p[-1]) + 1); print(".".join(p))' "$base_public") + SGLANG_VERSION="${next}.dev${n}+g${SGLANG_COMMIT:0:7}" +else + SGLANG_VERSION="${BASE_SGLANG_VERSION%%+*}+g${SGLANG_COMMIT:0:7}" +fi +MS=${MILES_COMMIT:0:7}; SS=${SGLANG_COMMIT:0:7} +TAG="$DEST_REPO:k31-$MS-$SS${TAG_SUFFIX:+-$TAG_SUFFIX}" +MTIME=$(git -C "$MILES_SRC" log -1 --format=%ct "$MILES_COMMIT") + +# 4. Megatron-LM files patched in the layer start from the base's checkout commit. +MEGATRON_LM_REPOSITORY=https://github.com/radixark/Megatron-LM +if [ -z "${MEGATRON_LM_SRC:-}" ]; then + MEGATRON_LM_SRC="$OUT/megatron-lm" + if [ ! -d "$MEGATRON_LM_SRC" ]; then + git init -q "$MEGATRON_LM_SRC" + git -C "$MEGATRON_LM_SRC" fetch -q --depth 1 "$MEGATRON_LM_REPOSITORY" "$MEGATRON_LM_COMMIT" + fi +fi +git -C "$MEGATRON_LM_SRC" cat-file -e "$MEGATRON_LM_COMMIT^{commit}" + +# 5. Wheels: peft (pure Python) and pyrage (manylinux abi3, x86_64), no deps. +PIP=${PIP:-python3 -m pip} +mkdir -p "$OUT/wheels" +$PIP download -q --no-deps --only-binary=:all: --dest "$OUT/wheels" "peft==$PEFT_VERSION" +$PIP download -q --no-deps --only-binary=:all: --platform manylinux_2_17_x86_64 --platform manylinux2014_x86_64 \ + --python-version 3.12 --abi abi3 --abi cp312 --dest "$OUT/wheels" "pyrage==$PYRAGE_VERSION" +PEFT_WHEEL=$(ls "$OUT/wheels"/peft-"$PEFT_VERSION"-*.whl) +# the base ships an older peft; its dist-info is whited out (lock packages.peft.replaces_base) +PEFT_BASE_VERSION=$(lock packages.peft.replaces_base 2>/dev/null || true) +PYRAGE_WHEEL=$(ls "$OUT/wheels"/pyrage-"$PYRAGE_VERSION"-*x86_64.whl) + +# 6. Manifest, commits.json (image-assert) and the layer. +python3 - "$OUT/image-manifest.json" </dev/null + "$CRANE" mutate "$TAG" -t "$TAG" \ + --entrypoint "/opt/yeto/image/entrypoint.sh" \ + --label "org.opencontainers.image.source=$MILES_ORIGIN" \ + --label "org.opencontainers.image.revision=$MILES_COMMIT" \ + --label "org.opencontainers.image.base.name=$BASE_REPO@$BASE_INDEX_DIGEST" \ + --label "ai.yeto.miles.commit=$MILES_COMMIT" \ + --label "ai.yeto.sglang.commit=$SGLANG_COMMIT" \ + --label "ai.yeto.image.lock=docker/miles-k31/versions.lock.json" >/dev/null + DIGEST=$("$CRANE" digest "$TAG") + echo "pushed $TAG@$DIGEST" +fi +python3 - "$OUT/build-record.json" < bool: + base = name.rsplit("/", 1)[-1] + return base in INSTALL_METADATA or (base.startswith("requirements") and base.endswith(".txt")) + + +def is_native(name: str) -> bool: + return name.startswith(NATIVE_PREFIXES) or name.endswith(NATIVE_SUFFIXES) or name.rsplit("/", 1)[-1] == "CMakeLists.txt" +# The file the layer writes over Megatron-Bridge's package __init__ (the base +# installs Bridge with pip, not editable; see megatron_bridge_init.py). MEGATRON_BRIDGE_INIT = Path(__file__).with_name("megatron_bridge_init.py") -# Megatron-LM as the base checks it out (/root/Megatron-LM, on sys.path) and -# the patch the layer applies to it (see the comments it adds). -MEGATRON_LM_COMMIT = "a84b105473eca9c9c6e49dbf1c8aa460c5cf796e" +# The patch the layer applies to the base's Megatron-LM checkout +# (/root/Megatron-LM, on sys.path; see the comments it adds). MEGATRON_LM_PATCH = Path(__file__).with_name("megatron_lm_lazy_modelopt.patch") +IMAGE_ASSERT_SRC = Path(__file__).resolve().parent.parent / "image-assert" + +_DIST_INFO = re.compile(r"^sglang-(?P\d[^/]*)\.dist-info$") # not sglang-router-* +_FINDER = re.compile(r"^__editable___sglang_(?P.+)_finder\.py$") + + +def editable_module_name(version: str) -> str: + """The setuptools editable finder module of ``sglang==version``.""" + return "__editable___sglang_" + version.replace(".", "_").replace("+", "_").replace("-", "_") + + +def sglang_metadata_names(names) -> dict[str, str] | None: + """The base's SGLang editable-install files among ``names`` (paths of a layer or + a directory listing, with or without the site-packages prefix): ``version``, + ``dist_info``, ``pth`` and ``finder`` (file names); None when the listing has no + ``sglang-*.dist-info``.""" + found: dict[str, str] = {} + for raw in names: + name = str(raw).strip().lstrip("./") + if name.startswith(SITE + "/"): + name = name[len(SITE) + 1:] + head = name.split("/", 1)[0] + dist = _DIST_INFO.match(head) + if dist and "dist_info" not in found: + found["dist_info"], found["version"] = head, dist["version"] + elif "/" not in name: + if name.startswith("__editable__.sglang-") and name.endswith(".pth"): + found["pth"] = name + elif _FINDER.match(name): + found["finder"] = name + if "dist_info" not in found: + return None + found.setdefault("pth", f"__editable__.sglang-{found['version']}.pth") + found.setdefault("finder", f"{editable_module_name(found['version'])}_finder.py") + return found + + +def layer_holds_sglang_metadata(names) -> bool: + return sglang_metadata_names(names) is not None + + +_SGLANG_STEP_HINTS = ("/sgl-workspace/sglang", "sglang_deps", "pip install -e") + + +def candidate_metadata_layers(manifest: dict, config: dict) -> list[int]: + """Indices into ``manifest["layers"]`` to scan for the SGLang editable-install + metadata, most likely first: the layers whose build step (image config + history, non-empty entries map 1:1 onto the layers) mentions the SGLang + checkout and an editable install, newest first; then every other layer + from the last one down (the newest write wins in an image, so the last + layer that holds the files is the one that matters).""" + layers = manifest.get("layers") or [] + steps = [h for h in config.get("history") or [] if not h.get("empty_layer")] + if steps and len(steps) != len(layers): + steps = [] # a history that does not line up is no hint at all + scored: list[tuple[int, int]] = [] + for index in range(len(layers)): + created = str(steps[index].get("created_by", "")) if steps else "" + score = 0 + if "/sgl-workspace/sglang" in created and "install -e" in created: + score = 2 + elif "sglang" in created and "install" in created: + score = 1 + scored.append((score, index)) + scored.sort(key=lambda item: (-item[0], -item[1])) + return [index for _score, index in scored] + + +def _helper_mode(argv: list[str]) -> int | None: + if not argv or argv[0] not in ("find-layer", "check-listing"): + return None + if argv[0] == "check-listing": + return 0 if layer_holds_sglang_metadata(sys.stdin.read().splitlines()) else 1 + p = argparse.ArgumentParser(prog="make_layer.py find-layer") + p.add_argument("--manifest", type=Path, required=True, help="crane manifest of the base (amd64)") + p.add_argument("--config", type=Path, required=True, help="crane config of the base (amd64)") + a = p.parse_args(argv[1:]) + manifest = json.loads(a.manifest.read_text()) + config = json.loads(a.config.read_text()) + order = candidate_metadata_layers(manifest, config) + print(json.dumps([{"index": i, "digest": manifest["layers"][i]["digest"], + "size": manifest["layers"][i].get("size")} for i in order])) + return 0 def git(repo: Path, *argv: str) -> str: @@ -66,18 +187,32 @@ def git(repo: Path, *argv: str) -> str: capture_output=True, text=True).stdout -def check_pure_python(repo: Path, base: str, commit: str, packages: tuple[str, ...]) -> list[str]: +def check_pure_python(repo: Path, base: str, commit: str, packages: tuple[str, ...], + *, allow_install_metadata_drift: bool = False) -> tuple[list[str], list[str], list[str]]: + """``(changed, deleted, install_metadata)`` paths of ``base..commit``; exits when + the fork touches a native source or a non-Python file of an installed package + (``packages``), or install metadata without ``allow_install_metadata_drift``.""" status = git(repo, "diff", "--name-status", "--no-renames", base, commit).splitlines() - names = [] + changed: list[str] = [] + deleted: list[str] = [] + metadata: list[str] = [] for line in status: kind, name = line.split("\t", 1) - if kind == "D": - raise SystemExit(f"{name}: fork deletes a base file; add a whiteout first") - in_package = name.startswith(packages) and name.endswith(".py") - if not (in_package or name.startswith(EXEMPT_PREFIXES) or name.endswith(".md")): + if is_native(name): + raise SystemExit(f"{name}: native source changed; needs a real (docker) build") + if name.startswith(packages) and not (name.endswith(".py") or name.endswith(".md") or name.endswith(".txt") + or name.endswith(".json") or name.endswith(".yaml") or name.endswith(".yml")): raise SystemExit(f"{name}: not a pure-Python package change; needs a real (docker) build") - names.append(name) - return names + if is_install_metadata(name): + if not allow_install_metadata_drift: + raise SystemExit(f"{name}: install metadata differs from the base's (its dist-info is kept); " + "pass --allow-install-metadata-drift when the change is metadata only") + metadata.append(name) + if kind == "D": + deleted.append(name) + else: + changed.append(name) + return changed, deleted, metadata def shallow_clone(src: Path, commit: str, origin: str, dest: Path) -> None: @@ -114,16 +249,22 @@ def version_file(version: str) -> bytes: ).encode() -def wheel_files(wheel: Path) -> dict[str, bytes]: - """``{site-packages relative path: bytes}`` of a pure-Python wheel. +def wheel_files(wheel: Path, *, platform: str | None = None) -> dict[str, bytes]: + """``{site-packages relative path: bytes}`` of a wheel that installs by + unpacking alone (no scripts, headers or ``.data`` directory). - Only ``*-none-any`` wheels without a ``.data`` directory are accepted: - those install by unpacking alone (no scripts, headers or compiled code). + ``*-none-any`` wheels (pure Python) are always accepted. With ``platform`` + (``manylinux_x86_64`` style suffix of the image's platform) a binary wheel of + exactly that platform is accepted too: a ``*-abi3-manylinux*_x86_64`` wheel + such as pyrage unpacks to ``.so`` files that pip would copy as they are. An ``INSTALLER`` file is added, as pip writes one. """ if not wheel.name.endswith("-none-any.whl"): - raise SystemExit(f"{wheel.name}: only pure-Python (*-none-any) wheels are supported") + if platform is None or not re.search(rf"-(?:cp3\d+|abi3|py3)-manylinux[\w.]*_{re.escape(platform)}\.whl$", + wheel.name): + raise SystemExit(f"{wheel.name}: only pure-Python (*-none-any) wheels are supported" + + (f", or manylinux {platform} wheels with --wheel-platform" if platform else "")) files: dict[str, bytes] = {} with zipfile.ZipFile(wheel) as archive: for name in archive.namelist(): @@ -164,21 +305,47 @@ def main() -> None: p.add_argument("--sglang-commit", required=True) p.add_argument("--sglang-base-commit", required=True, help="commit in the base image") p.add_argument("--sglang-origin", required=True) - p.add_argument("--sglang-base-version", required=True) p.add_argument("--sglang-version", required=True) p.add_argument("--sglang-base-metadata", type=Path, required=True, - help="dir with the base image's sglang site-packages metadata") + help="dir with the base image's sglang site-packages metadata (its version is read " + "from the sglang-*.dist-info directory found there)") p.add_argument("--wheel", type=Path, action="append", default=[], - help="pure-Python wheel unpacked into site-packages (repeatable)") + help="wheel unpacked into site-packages (repeatable): pure Python, or a manylinux " + "wheel of --wheel-platform") + p.add_argument("--wheel-platform", default=None, + help="accept manylinux binary wheels of this platform (e.g. x86_64) besides pure-Python ones") + p.add_argument("--replace-base-dist", action="append", default=[], metavar="NAME=VERSION", + help="whiteout the base's NAME-VERSION.dist-info (a wheel of the same distribution is " + "unpacked over it; with both present importlib.metadata picks whichever lists first)") p.add_argument("--megatron-lm-src", type=Path, required=True, - help=f"clone containing Megatron-LM {MEGATRON_LM_COMMIT[:7]} (the base's checkout)") + help="clone containing the base's Megatron-LM checkout commit (--megatron-lm-commit)") + p.add_argument("--megatron-lm-commit", required=True, help="Megatron-LM commit the base checked out") + p.add_argument("--megatron-bridge-commit", required=True, + help="Megatron-Bridge commit the base installed (recorded; its __init__ is replaced)") + p.add_argument("--image-assert-dir", type=Path, default=None, + help="directory with versions.lock.json and commits.json: shipped with the " + "docker/image-assert checker under /opt/yeto/image") + p.add_argument("--allow-install-metadata-drift", action="store_true", + help="accept setup.py/setup.cfg/pyproject.toml/requirements* changes in the forks " + "(the base's dist-info is kept; the paths are recorded in the manifest)") p.add_argument("--mtime", type=int, required=True) p.add_argument("--out", type=Path, required=True) a = p.parse_args() # tools/: stand-alone scripts run from the checkout (e.g. convert_hf_to_fp8.py), nothing built - check_pure_python(a.miles_src, a.miles_base_commit, a.miles_commit, ("miles/", "miles_plugins/", "tools/")) - check_pure_python(a.sglang_src, a.sglang_base_commit, a.sglang_commit, ("python/sglang/",)) + deleted: dict[str, list[str]] = {} + drift: dict[str, list[str]] = {} + for key, src, base, commit, packages in ( + ("miles", a.miles_src, a.miles_base_commit, a.miles_commit, ("miles/", "miles_plugins/", "tools/")), + ("sglang", a.sglang_src, a.sglang_base_commit, a.sglang_commit, ("python/sglang/",)), + ): + _changed, deleted[key], drift[key] = check_pure_python( + src, base, commit, packages, allow_install_metadata_drift=a.allow_install_metadata_drift) + manifest = json.loads(Path(a.manifest_json).read_text()) + for key in ("miles", "sglang"): + manifest[key]["deleted_from_base"] = deleted[key] + manifest[key]["install_metadata_drift"] = drift[key] + Path(a.manifest_json).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") entries: dict[str, tuple[str, bytes | None, int, str | None]] = {} @@ -221,26 +388,34 @@ def add_tree(root: Path, dest: str) -> None: add_tree(clone, dest) # hide the base image's .git (other history/origin) entirely add_file(f"{dest}/.git/.wh..wh..opq", b"", 0o644) + # files the fork deleted: whiteouts, so the checkout is exactly the fork commit + for name in deleted["miles" if dest == "root/miles" else "sglang"]: + d, n = os.path.split(f"{dest}/{name}") + add_file(f"{d}/.wh.{n}", b"") - # SGLang editable metadata: same install, fork version. - old, new = a.sglang_base_version, a.sglang_version - old_mod = "__editable___sglang_" + old.replace(".", "_").replace("+", "_") - new_mod = "__editable___sglang_" + new.replace(".", "_").replace("+", "_") + # SGLang editable metadata: same install, fork version. The base's version + # and finder module come from the files themselves (no version is written here). meta = a.sglang_base_metadata + base_meta = sglang_metadata_names(os.listdir(meta)) + if base_meta is None: + raise SystemExit(f"{meta}: no sglang-*.dist-info directory (not the base's site-packages metadata)") + old, new = base_meta["version"], a.sglang_version + old_finder, old_pth = base_meta["finder"], base_meta["pth"] + old_mod = old_finder[: -len("_finder.py")] + new_mod = editable_module_name(new) subst = lambda b: b.replace(old.encode(), new.encode()).replace(old_mod.encode(), new_mod.encode()) - for stale in (f"__editable__.sglang-{old}.pth", f"{old_mod}_finder.py", - f"sglang-{old}.dist-info", f"__pycache__/{old_mod}_finder.cpython-312.pyc"): + for stale in (old_pth, old_finder, base_meta["dist_info"], f"__pycache__/{old_mod}_finder.cpython-312.pyc"): d, n = os.path.split(f"{SITE}/{stale}") add_file(f"{d}/.wh.{n}", b"") files = { - f"__editable__.sglang-{new}.pth": subst((meta / f"__editable__.sglang-{old}.pth").read_bytes()), - f"{new_mod}_finder.py": subst((meta / f"{old_mod}_finder.py").read_bytes()), + f"__editable__.sglang-{new}.pth": subst((meta / old_pth).read_bytes()), + f"{new_mod}_finder.py": subst((meta / old_finder).read_bytes()), } dist = f"sglang-{new}.dist-info" - for name in sorted(os.listdir(meta / f"sglang-{old}.dist-info")): + for name in sorted(os.listdir(meta / base_meta["dist_info"])): if name != "RECORD": - files[f"{dist}/{name}"] = subst((meta / f"sglang-{old}.dist-info" / name).read_bytes()) - old_record = (meta / f"sglang-{old}.dist-info" / "RECORD").read_text().splitlines() + files[f"{dist}/{name}"] = subst((meta / base_meta["dist_info"] / name).read_bytes()) + old_record = (meta / base_meta["dist_info"] / "RECORD").read_text().splitlines() record = [l for l in old_record if l.startswith("../")] # bin/ scripts unchanged record += [record_line(n, d) for n, d in sorted(files.items())] record += [f"__pycache__/{new_mod}_finder.cpython-312.pyc,,", f"{dist}/RECORD,,"] @@ -250,15 +425,28 @@ def add_tree(root: Path, dest: str) -> None: add_file("sgl-workspace/sglang/python/sglang/_version.py", version_file(new)) for wheel in a.wheel: - for name, data in sorted(wheel_files(wheel).items()): - add_file(f"{SITE}/{name}", data) + for name, data in sorted(wheel_files(wheel, platform=a.wheel_platform).items()): + mode = 0o755 if name.endswith(".so") or "/bin/" in name else 0o644 + add_file(f"{SITE}/{name}", data, mode) + for spec in a.replace_base_dist: + dist, _, version = spec.partition("=") + if not dist or not version: + raise SystemExit(f"--replace-base-dist {spec!r}: expected NAME=VERSION") + add_file(f"{SITE}/.wh.{dist.replace('-', '_')}-{version}.dist-info", b"") add_file(f"{SITE}/megatron/bridge/__init__.py", MEGATRON_BRIDGE_INIT.read_bytes()) - for name, data in patched_files(a.megatron_lm_src, MEGATRON_LM_COMMIT, MEGATRON_LM_PATCH).items(): + for name, data in patched_files(a.megatron_lm_src, a.megatron_lm_commit, MEGATRON_LM_PATCH).items(): add_file(f"root/Megatron-LM/{name}", data) add_dir("opt/yeto") add_file("opt/yeto/image-manifest.json", Path(a.manifest_json).read_bytes()) + if a.image_assert_dir is not None: + add_dir(IMAGE_ASSERT_DIR) + for name in ("versions.lock.json", "commits.json"): + add_file(f"{IMAGE_ASSERT_DIR}/{name}", (a.image_assert_dir / name).read_bytes()) + add_file(f"{IMAGE_ASSERT_DIR}/assert_image_versions.py", + (IMAGE_ASSERT_SRC / "assert_image_versions.py").read_bytes(), 0o755) + add_file(f"{IMAGE_ASSERT_DIR}/entrypoint.sh", (IMAGE_ASSERT_SRC / "entrypoint.sh").read_bytes(), 0o755) with tarfile.open(a.out, "w", format=tarfile.PAX_FORMAT) as tar: for path in sorted(entries): @@ -279,4 +467,8 @@ def add_tree(root: Path, dest: str) -> None: if __name__ == "__main__": - main() + code = _helper_mode(sys.argv[1:]) + if code is None: + main() + else: + sys.exit(code) diff --git a/docs/BREAKER.md b/docs/BREAKER.md new file mode 100644 index 000000000..12eb8d4e4 --- /dev/null +++ b/docs/BREAKER.md @@ -0,0 +1,90 @@ +# Run health breakers + +Two pieces watch an RL run's numerics: + +- `yeto breaker`: a read-only check over a run's event tapes. It prints every + breaker with its window value and limit and exits 1 when one trips, so a cron + job or a supervisor can page or call `yeto stop-run RUN`. It never stops + anything itself. +- `YETO_RL_XHW_GUARD`: the in-run guard of the trainer driver. It judges each + sampler source every trained round and can warn, quarantine, down-weight or + stop. + +## `yeto breaker` + +```bash +yeto breaker --trainer-log ~/.yeto/runs//launcher.log \ + --sampler-tapes [--window 3] \ + [--sampler-hardware auto|amd|nvidia] [--noise-floor X] \ + [--logprob-warn X] [--logprob-act Y] [--flips flips.json] [--json] +``` + +- `--trainer-log`: launcher logs (lines with `YETO_RL_EVENT {json}`), JSONL + event tapes, or directories of them. +- `--sampler-tapes`: sampler event tapes (`events..jsonl`), as + files, globs or directories (every `*.jsonl` below them). +- Exit codes: `0` all ok (warnings included), `1` a breaker tripped, `2` bad + input (a missing file, inconsistent limits). + +| Breaker | Value (trailing `--window` trained steps) | Trips | +|---|---|---| +| `logprob_diff` | mean `train_rollout_logprob_abs_diff` | above the hardware limit (below) | +| `is_clipped` | mean `sc_is_clipped` | > 0.05 | +| `is_weight` | mean `sc_importance_weight` | < 0.9 | +| `stale` | stale groups / (stale + trained) | > 0.30 | +| `aborted` | agentic trials lost / trials (last 3 x window sampler rounds) | > 0.20 | +| `zero_call` | agentic trials retried before any model call / trials | > 0.05 | +| `flips` | changed / total weights of the last release (`--flips`, `{release: {changed, total}}`) | > 0.01 | +| `nan_skip_consecutive` | rounds skipped in a row for a non-finite grad norm | > 2 | +| `nan_skip_fraction` | skipped / trained rounds (judged from 20 trained rounds on) | > 0.05 | +| `r3_rows` | sampler records reporting a routed-experts row mismatch | > 0 | +| `xhw /` | the last `rl_xhw_mismatch` row of each source | level `act`, or the guard stopped | + +### Log-prob mismatch limits by sampler hardware + +The train/rollout log-prob difference is not zero even with identical weights: +the sampler's card and engine round differently from the trainer's recompute. +The limits therefore depend on where the rollouts were sampled. + +| Hardware class | warn | trip | Measured (Kimi-K3.1, step 1, pure numerics) | +|---|---|---|---| +| `amd` (ROCm, MI-series cards) | 0.10 | 0.15 | MI355X vLLM -> B300 Megatron 0.076-0.099 | +| `nvidia` (CUDA cards) | 0.04 | 0.052 | B300 SGLang -> B300 Megatron 0.023-0.028 | + +The `amd` limits are the in-run guard's defaults. With `--sampler-hardware auto` +(the default) the class comes from the sampler tapes (`rl_sampler_engine_round` +`source`, `family/card/engine@version/...`) or the `rl_xhw_mismatch` rows. A run +with any AMD sampler is judged by the `amd` limits, since the step value mixes +the tokens of every source. When no tape tells, the `amd` limits apply and the +line says `sampler hardware unknown`. + +`--noise-floor X` (the mean abs diff of two repeat runs with the same weights) +raises the trip limit to at least `1.5 * X` and the warn limit to at least `X`. +`--logprob-warn` and `--logprob-act` set the limits directly. + +## `YETO_RL_XHW_GUARD` (in-run, per source) + +Opt-in. The launcher forwards it to the trainer island. With several sampler +sources (cards, engines), the driver attributes the step metrics to each source +by its token share and judges each one on a trailing window. It emits +`rl_xhw_mismatch` every trained round. + +```bash +YETO_RL_XHW_GUARD=1 # defaults +YETO_RL_XHW_GUARD=warn_abs_diff=0.10,act_abs_diff=0.15,warn_kl=0.03,act_kl=0.06,warn_clip=0.01,act_clip=0.03,window=3,min_rounds=2,action=quarantine,quarantine_rounds=8,weight=0.5 +``` + +| Key | Default | Meaning | +|---|---|---| +| `warn_abs_diff` / `act_abs_diff` | 0.10 / 0.15 | `train_rollout_logprob_abs_diff` limits | +| `warn_kl` / `act_kl` | 0.03 / 0.06 | `train_rollout_kl` limits | +| `warn_clip` / `act_clip` | 0.01 / 0.03 | `sc_is_clipped` limits | +| `window` | 3 | trailing trained steps per source | +| `min_rounds` | 2 | rounds a source needs before it can be acted on | +| `action` | `quarantine` | `warn`, `quarantine` (skip the source's groups for `quarantine_rounds` rounds, then let one probe round through), `downweight` (scale its advantages by `weight`; exact only without GRPO std normalization), `stop` (the driver raises) | +| `quarantine_rounds` | 8 | rounds of a quarantine | +| `weight` | 0.5 | advantage scale of `downweight` | + +Every key is optional; `0`, `off` or unset turns the guard off. The defaults +sit above the measured MI355X vLLM band. For an all-NVIDIA fleet, tighten them +to `warn_abs_diff=0.04,act_abs_diff=0.052`. diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index b0781bd6f..5db964642 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -166,6 +166,13 @@ Inspect and end runs. Each takes the run name (its `--cluster-prefix`). | `yeto logs RUN` | follow a run log | `--no-follow`: print what is captured so far and exit | | `yeto down RUN` | tear down every cluster of a run | — | | `yeto stop-run RUN` | stop a run cleanly | — | +| `yeto cost --run RUN` | cost per resource and per UTC hour, GPU effective utilization ([COST_GUARD.md](COST_GUARD.md)) | `--json`, `--no-billing`, `--prices FILE`, `--tapes GLOB...`, `--gap-minutes` | +| `yeto cleanup --run RUN` | release every resource the run's ledger still lists, each by its exact id | `--dry-run` | +| `yeto breaker --trainer-log LOG` | read-only health breakers over a run's event tapes; exit 1 on a trip ([BREAKER.md](BREAKER.md)) | `--sampler-tapes`, `--sampler-hardware`, `--noise-floor`, `--logprob-warn`, `--logprob-act`, `--window`, `--flips`, `--json` | + +`yeto down RUN` also releases what the run's resource ledger still lists (sampler +apps, NC leases). The launch flags of the cost guard (`--budget-usd`, idle +shutdown, `--no-cost-guard`) are in [COST_GUARD.md](COST_GUARD.md). ### `yeto merge` @@ -517,10 +524,11 @@ injection and are omitted here. | Flag | Default | Description | |---|---|---| -| `--reward-function` | — | reward callable `package.module:function` | +| `--reward-function` | — | reward callable `package.module:function`, or `path/file.py:function` inside the repository | | `--rl-prompt-column` | — | prompt column (default: `messages`, `prompt` or `input`) | | `--rl-label-column` | — | label column passed to the reward (default `label`) | | `--rl-allow-local-data` | off | accept a local prompt file for `--data` | +| `--rl-sampler-engine` | — | `sglang` or `vllm`: engine of the separate samplers of a `--rl-sample-role trainer` run (checks Kimi-K3.1 LoRA targets) | | `--n-samples-per-prompt` | `4` | group size | | `--rollout-batch-size` | `32` | prompts per rollout round | | `--rollout-max-response-len` | `32768` | max generated tokens per response | @@ -581,7 +589,7 @@ injection and are omitted here. | `--rl-sample-relay` | off | the syncer relays spot sampler groups to the trainer | | `--rl-spot-subscribers` | `0` | receive-only spot sampler subscribers on the syncer | | `--rl-release-delta-full-every` | `0` | upload releases as bf16 deltas, with a full release every N | -| `--rl-sample-wait-s` | — | trainer: max wait for one full round from the buffer (default 3600) | +| `--rl-sample-wait-s` | — | trainer: max seconds without a new group while a round fills (default 3600) | **Checkpoints, recovery and resume** diff --git a/docs/CLOUDS.md b/docs/CLOUDS.md index ddb1f827a..20df77fe7 100644 --- a/docs/CLOUDS.md +++ b/docs/CLOUDS.md @@ -82,6 +82,208 @@ from Modal's cluster info. Facts to keep in mind: - Manual join: launch with `--external-learners 1`, then `python -m yeto.modal_runner --learner-id N --num-learners M --syncer-addr H:P --run-script ...`. +## Default container images + +RL islands and engine samplers run digest-pinned images (`yeto-miles-k31`, +`yeto-miles-ports`, `yeto-sampler-k31`) from `ghcr.io/agentenv` by default. +The digest is fixed in the code; only the registry prefix is configurable: + +```bash +export YETO_IMAGE_REGISTRY=/ # a mirror holding the same digests +``` + +| Image | Used by | +|---|---| +| `yeto-miles-k31@sha256:63ad0d2a…` | RL trainer default (Kimi-K3.1 LoRA and other recipes) | +| `yeto-miles-k31@sha256:2de5d796…` | Kimi-K3.1 `--tuning full` trainer default | +| `vllm-openai-rocm@sha256:5f47cc10…` | Kimi-K3.1 A-card vLLM sampler | + +`--rl-image docker:@sha256:` replaces the image of one +run, and a sampler spec's `image` replaces the sampler's. + +**The packages are private for now.** Pulls need a registry token with +`read:packages` on the organization: + +```bash +export SKYPILOT_DOCKER_SERVER=ghcr.io # or your mirror's host +export SKYPILOT_DOCKER_USERNAME= +export SKYPILOT_DOCKER_PASSWORD= +yeto launch ... --rl-image-private # forward the login to the islands +``` + +The same variables (or a `ghcr.io` entry in `~/.docker/config.json`) serve SkyPilot +VMs and Modal islands (yeto passes them to Modal as a registry secret). An A-card +sampler node pulls with `docker login ` using the same token before +`docker pull`. Without credentials the pull fails with an authorization error. + +## Modal B300 clusters (Kimi K3.1 trainer) + +A K3.1 trainer island is a Modal clustered function on whole B300 nodes, +`--gpu modal:6x8xb300` (`MODAL_GPUS["B300"]`, 8 per container like the other +whole-node types). Facts to keep in mind: + +- The decorator is `modal.clustered(size=N, rdma=True)` (Modal Clusters GA); + a client that only has `modal.experimental.clustered` is used through the + same path (`modal_runner.clustered_decorator`). Modal sets the NCCL + interface variables for RDMA itself; yeto exports `NCCL_IB_DISABLE=0` and + nothing about `NCCL_IB_HCA` / `NCCL_SOCKET_IFNAME`. +- Every Modal island gets `NCCL_RAS_ENABLE=0` (`MODAL_ISLAND_DEFAULT_ENVS`): + NCCL 2.30's RAS thread crashes the process after the first engine memory + release. `--modal-env KEY=VALUE` overrides any default; VM-passthrough hosts + need `--modal-env NCCL_NVLS_ENABLE=0`. +- A preempted or failed container restarts the whole cluster. With a + `modal-volume://` or `s3://` checkpoint store the re-run is a resume from + the store's `LATEST` (`storage.resumes_in_new_container`), not a failure. +- Image: `docker/miles-k31/` (one overlay on `radixark/miles:kimi-k31`, CUDA + 13.0.3, NCCL 2.30.7). Modal's GPU page says "B300 requires CUDA 13.1+" + while the image toolkit is 13.0.3; the Modal host driver (610.x, CUDA 13.3) + runs 13.x runtimes. Whether the 13.0.3 toolkit satisfies the B300 JIT paths + (flashinfer, deep_gemm, triton) is open until the first smoke test on two + `B300:8` containers; the fallback is a 13.1 toolchain layer with + `CUDA_HOME` pointed at it. The same test decides IB verbs vs EFA hosts + (`experimental_options={"efa_disabled": True}` forces IB). +- Shape planning (`yeto shape`) does not price B300 on Modal: the price table + (`MODAL_GPU_USD_PER_SECOND`) has no entry and none is guessed; `yeto launch + --gpu modal:...xb300` works. + +## S3-compatible stores (Cloudflare R2) on Modal + +A Modal island takes an `s3://BUCKET[/PREFIX]` `--rl-checkpoint-store`. Modal +mounts no bucket, so every node keeps a local stage of the store and moves +the whole store by object (`yeto.cloud.s3_store`, `STAGE_ENV`): round cuts, +`LATEST`, the sample buffer, policy releases and trainer commands. The head +(trainer scheduler, convertible pool) reads the same store by object key. + +- Endpoint: `--rl-object-store-endpoint https://.r2.cloudflarestorage.com` + (or `AWS_ENDPOINT_URL` in the launcher environment). The region becomes + `auto` unless one is configured; the launcher exports the endpoint to its + own environment too so every launcher-side boto3 client (head object API, + code pack store) talks to the same endpoint. +- A standard AWS setup works as is. The endpoint is, in order: + `YETO_RL_STORE_S3_ENDPOINT_URL`, `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL`, + then `endpoint_url` of the store's AWS profile (top level or in its `s3` + section). The region is `YETO_RL_STORE_S3_REGION`, `AWS_REGION` / + `AWS_DEFAULT_REGION`, the profile's `region`, else `auto` for an R2 endpoint. + The profile is `YETO_RL_STORE_S3_PROFILE`, else `AWS_PROFILE`; `yeto launch + --rl-object-store-profile NAME` sets it for a launch. The head reads with the + profile, and without `--modal-store-secret` its key, endpoint and region are + shipped to the islands (an island has no profile to read). A profile for R2: + + ```ini + # ~/.aws/config + [profile my-r2] + region = auto + endpoint_url = https://.r2.cloudflarestorage.com + ``` + + with the key pair under `[my-r2]` in `~/.aws/credentials`. Store CLIs + (`python -m yeto.rl.engine.store_dispense status`) take `--endpoint-url` and + `--profile`. A credential or endpoint error names what to set: for example, + an R2 key sent to AWS S3 because no endpoint was set reports + `AWS S3 (no endpoint set) refused the credentials (InvalidAccessKeyId)` and + the endpoint variables. +- Credentials: `--modal-store-secret NAME` attaches a Modal Secret holding + `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (and optionally + `AWS_ENDPOINT_URL`, `AWS_REGION`) to the island function; the island reads + that pair because the launch says so (`YETO_RL_STORE_S3_CREDENTIALS=aws-env`). + Create it once: + `modal secret create NAME AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_ENDPOINT_URL=https://.r2.cloudflarestorage.com AWS_REGION=auto` + (an R2 API token with Object Read & Write on the bucket). + Without the flag the launcher's store credential (`YETO_RL_STORE_S3_*`) is + shipped as on sky. The launcher's own `AWS_*` variables are never a store + credential, and no island uses the default credential chain. +- Uploads: a file larger than one part (`YETO_RL_STORE_PART_MB`, default 64) + goes as a resumable multipart upload. The UploadId and acknowledged parts + are journaled next to the source (`.upload.tmp`); a failed try + continues from the parts already stored, up to `YETO_RL_STORE_UPLOAD_ATTEMPTS` + tries (default 3), and a later push or a restarted rank continues it too. +- Part checksums, `YETO_RL_STORE_PART_CHECKSUM` (forwarded to every island): + `1` sends a SHA-256 with every part and checks the store's composite + checksum after the upload; `0` sends none; `auto` (the default) sends them + except to a Cloudflare R2 endpoint (`*.r2.cloudflarestorage.com`), which + answers `NotImplemented` to a checksummed `UploadPart`. Any other endpoint + that answers `NotImplemented` turns checksums off for that store with one + warning per process (`does not take SHA-256 part checksums`) and the file + is uploaded again without them; set `0` for such an endpoint to skip the + probe. Without part checksums a cut is still checked by size and by the + manifest's SHA-256 when it is read. +- Account ids, bucket names and endpoints live in the launch flags, the + environment or the Secret, never in the code. +- Open until a run: the per-node stage of a multi-node trainer (each node + pushes its own shards and pulls every subtree it does not have) and the + store throughput from Modal to R2. + +## National Compute sampler nodes (MI355X, ROCm) + +An account holds one 8-card node at a time. A lease is granted for up to 4 +hours and cannot be renewed. Nodes can be preempted, and the local disk is +destroyed at release. The rollout side of a K3.1 run is therefore a set of +short-lived sampler islands over the object store. + +> **Billing warning.** A lease does **not** end on its own at `lease_until`. +> When `lease_until` passes, the ticket stays `active` and **keeps billing** +> until it is deleted. Each node must be released explicitly: +> +> - `python -m yeto.launch.nc pool-stop --config ` releases every +> node of a pool; +> - `python -m yeto.launch.nc down --account --ticket ` +> releases a single node. +> +> The pool does this for you only while its control loop runs. A lease recorded +> in a run's resource ledger is also shown by `yeto cost --run RUN`, deleted by +> `yeto cleanup --run RUN` / `yeto down RUN`, and deleted by the cost guard 5 +> minutes after its `lease_until` ([COST_GUARD.md](COST_GUARD.md)). + +**Recommended path:** the vLLM sampler pool, `python -m yeto.launch.nc pool`. +It orders the nodes, starts and replaces the samplers, and releases the nodes +before their leases end. The pool config, `--dry-run`, `pool-status`, +`pool-stop` and the control-host requirement are covered in +[NC_POOL.md](NC_POOL.md). The task dispenser that its samplers share is +covered in [DISPENSER.md](DISPENSER.md). + +**Legacy path (manual SGLang node scripts).** `scripts/sampler_node/` holds +scripts that set up one node by hand with the SGLang server. Each value is a +parameter; no site, account or model name is written in them. Nothing +releases a node set up this way except `down` (see the billing warning +above). The scripts: + +- `nc_lease.sh --account N` polls `nc-query status` until the ticket is + active, writes `node.json` (ip, user, host key, lease_until, rate) and pins + the host key. +- `bootstrap.sh --image IMAGE --model MODEL` reports the host (a node without + `amd-smi` or `rocm-smi` is refused: the launcher cannot check its cards), + makes the NVMe directories, pulls the SGLang ROCm image and fetches the + base model (HF repo, HF bucket, or `--peer user@host:/path` from a node + that has it) in the background. With `--release-store DIR --release-prefix + RUN [--release-peers host:port,...] [--serve-port P]` it also starts the + policy release puller (`python -m yeto.rl.spot.s3_delivery pull-loop`): + releases come from the peer nodes first (near-region fan-out, + `yeto.rl.spot.peer_fanout`) and from the store only when no peer has them, + and the node serves its own placed releases on `--serve-port`. The bucket, + endpoint and credentials come from `--r2-env-file` (mode 0600; rclone takes + the endpoint through `RCLONE_S3_ENDPOINT`). +- `run_sglang.sh --image IMAGE --model-dir DIR --tp 8` starts the server in + docker with the K3.1 contract on: `--enable-return-routed-experts`, + `--enable-fp32-lm-head`, `--attn-dp-size TP`, `--moe-a2a-backend deepep` + (each a parameter), and the ROCm environment + `ROCM_QUICK_REDUCE_QUANTIZATION=NONE SGLANG_ROCM_RUNTIME_RESERVE_GB=10 + AITER_SITUV2_A8W4=1 HSA_NO_SCRATCH_RECLAIM=1` (no `MX_PATCH`); the store env + file goes in as the container environment and must be mode 0600. +- `run_engine_sampler.sh --spec SPEC.json` starts + `python -m yeto.rl.engine.engine_sampler` with the spec; the store + credentials come from `--r2-env-file`; the interpreter must import + `safetensors` and `boto3` (`--pip-install` adds them). + +Launcher side, an MI355X node of an SSH inventory is a `rocm` device-family +node (`yeto/gpu_spec.py`): the card preflight runs `amd-smi static --json`, +then `rocm-smi --showproductname --showmeminfo vram --json`, and compares the +cards, count and HBM (288 GiB) with the inventory (`yeto/hw/card_tools.py`). +The JSON shapes follow the ROCm documentation and are confirmed on the first +lease. `scripts/prune_layers.py` cuts the first N layers of a checkpoint +(K3.1: `--layers 4` keeps the dense layer, two KDA layers and the first +MLA/DSA layer with all 896 experts, 7 of 96 shards) for CPU and single-node +checks. + ## Verified on real machines Verification results decide runtime behaviour: the planner only plans RL @@ -96,7 +298,7 @@ for multi-node islands only on `RDMA_CLOUDS` (all in | RunPod | yes | yes | not verified | single node | | Nebius | yes | not verified | not verified (the S3-compatible store needs `aws configure --profile nebius`) | not verified (InfiniBand expected on 8-GPU SXM) | | Verda | not verified | not verified | not available in sky | single node | -| Modal | yes | yes (8xH100, single container) | not verified (Modal Volume) | not verified (RoCE, whole nodes) | +| Modal | yes | yes (8xH100, single container) | not verified (Modal Volume); `s3://` (R2) wired, not verified | not verified (RoCE, whole nodes; B300 clusters wired, not verified) | A mixed fleet (learner islands on Nebius and Modal, syncer head on AWS) and a manual Modal join next to a Nebius island have also run to completion, with diff --git a/docs/COST_GUARD.md b/docs/COST_GUARD.md new file mode 100644 index 000000000..e1fc22f19 --- /dev/null +++ b/docs/COST_GUARD.md @@ -0,0 +1,99 @@ +# Cost protection for unattended runs + +yeto protects every `yeto launch` run against three ways of spending money while nobody watches: +a run that costs more than planned, GPUs that sit idle, and resources that outlive the run. + +## 1. Resource ledger (always on) + +Every billed resource a run creates is appended to `~/.yeto/runs//resources.jsonl` the moment +it exists: Modal apps (by app id), Modal FunctionCalls, National Compute leases (ticket + account), and sky +clusters. A second line is written when a resource is released. Writes are fsynced, so the ledger +stays correct even if the launcher is killed with SIGKILL. + +Processes started for the same run find the ledger through `YETO_RESOURCE_LEDGER`, which `yeto launch` +sets for its worker; a process that records into it writes an `nc_lease` entry with `account`, the ticket +as id, `lease_until` and a rate. + +> The NC lease pool (`python -m yeto.launch.nc pool`) does not write this ledger yet: it keeps its own +> ticket ledger (`YETO_NC_STATE_DIR`). Its leases are not in `yeto cost`, the budget or `yeto cleanup`; +> stop its nodes with `python -m yeto.launch.nc down --account N --ticket T`. + +## 2. Budget breaker + +| Flag | Default | Meaning | +|---|---|---| +| `--budget-usd` | none (off) | When the run's cost reaches this, stop gracefully: the trainer takes a cut, the fleet ends, and every resource is torn down. | +| `--budget-warn-usd` | 80 % of `--budget-usd` | Emit `cost_budget_warn` once. | +| `--budget-stop-grace-minutes` | 20 | If the run is still up this long after a graceful stop, release every ledger resource by force. | +| `--cost-poll-seconds` | 60 | How often the guard checks. | +| `--cost-prices` | none | Price table used to estimate time that has not been billed yet. | + +A reused run name (`--cluster-prefix`) keeps its ledger. The budget of a new launch counts the +resources still open plus those created since it started; what an earlier launch of that name released +before is not counted (`yeto cost --run` still reports the whole ledger). A new launch also removes the +`STOP` flag an earlier launch of that name left, so a budget or idle stop does not end the next launch. + +How cost is computed: +- **Modal**: hourly rows from the Modal billing API, matched by exact app id, plus an estimate for the + time after the newest billed hour (billing data lags behind). The estimate uses the rate recorded in + the ledger, then the price table, then the app's newest complete billed hour. +- **National Compute**: lease duration × rate. A lease past its `lease_until` keeps billing until it is + deleted, and it is counted that way. Set `YETO_NC_USD_PER_GPU_HOUR` (or `YETO_NC_USD_PER_UCR`) to price it. +- A resource without a known rate is reported as **unpriced** (`cost_unpriced` event), never as $0, and + is not counted toward the budget. Configure a rate for it. + +## 3. Idle shutdown + +| Flag | Default | Meaning | +|---|---|---| +| `--idle-warn-minutes` | 15 | `idle_warn` after this long with no progress event, or with every reporting GPU at 0 %. `0` turns idle shutdown off. | +| `--idle-stop-minutes` | 15 | Stop the run (same graceful-then-forced path as the budget) this long after `idle_warn` if it is still idle. | +| `--load-grace-minutes` | 45 | No idle verdict for this long after the start or after any loading event (weight loading, a trainer restart). Loading the K3.1 weights takes about 20 minutes with 0 % GPU. | +| `--max-load-minutes` | 120 | Loading events alone cannot hold off the idle verdict for longer than this since the last progress (catches a loader stuck in a restart loop). | +| `--cost-tapes` | none | Extra event tape globs the detector reads, for example sampler tapes pulled from NC nodes. | + +Progress means trained rounds, sampled rounds, cuts and policy releases. Heartbeats are not progress, +because a hung process still sends heartbeats. A sampler that waits while the trainer trains does not +make the run idle, because idleness is judged for the run as a whole. + +## 4. Lease expiry + +An NC lease still open 5 minutes after its `lease_until` is released (`nc_lease_overdue`). The +lease pool normally drains nodes before that point; this check is a safety net. + +## 5. Cleanup + +- When the run ends normally or with an error, the launcher tears down its resources. The worker then + releases anything the ledger still lists. +- On SIGTERM (`yeto down`), the launcher unwinds through its teardown. +- If the launcher is killed outright, a detached **reaper** process (one per run) sees the worker exit and + releases what the ledger still lists. +- Manual cleanup: `yeto cleanup --run [--dry-run]`. It releases only what the ledger lists, each by + its exact id. It never matches by name pattern and never uses `--all`. Running it again is safe. +- `--keep` runs are left up by the reaper and the backstop. Release them with `yeto down `, which + also cleans the ledger. + +### Control host + +The guard and the reaper run on the machine that ran `yeto launch`. If that machine goes offline +(sleep, network loss, shutdown), no budget, idle or lease-expiry check runs and nothing releases the +run's resources; Modal apps and NC leases keep billing. Run long launches from a host that stays up +(a small always-on VM, in `tmux` or under `systemd`), and after an outage run `yeto cost --run RUN` +and `yeto cleanup --run RUN`. + +## 6. Cost report + +`yeto cost --run [--json] [--no-billing] [--prices FILE] [--tapes GLOB...]` shows: +- cost per resource: billed $, estimated $ and total +- cost per UTC hour +- **GPU effective utilization**: productive time ÷ billed compute time. Productive time is the time + between progress events that are at most `--gap-minutes` (default 10) apart. This is an estimate taken + from event tapes. When hardware utilization samples are available, their mean is shown next to it. + +## Events + +All events go to `~/.yeto/runs//cost-events.jsonl` and to the launcher log (`[cost-guard] ...`): +`cost_tick`, `cost_budget_warn`, `cost_budget_exceeded`, `cost_unpriced`, `idle_warn`, `idle_cleared`, +`idle_shutdown`, `cost_guard_stop` (mode graceful / forced), `nc_lease_overdue`. + +`--no-cost-guard` turns the guard off. The ledger, the reaper and `yeto cleanup` keep working. diff --git a/docs/DISPENSER.md b/docs/DISPENSER.md new file mode 100644 index 000000000..0e4588a86 --- /dev/null +++ b/docs/DISPENSER.md @@ -0,0 +1,176 @@ +# Task dispenser over a shared bucket + +This page covers the **store dispenser**: how agentic samplers that share nothing but one +object-store bucket (for example, A-card samplers on separate cloud accounts writing one R2 bucket) +divide the training tasks between them. There is no syncer, no TCP endpoint and no coordinator +process. The whole state lives in the bucket. + +If all your samplers can reach one syncer over TCP, the syncer's task dispenser +(`--rl-data-dispense`) is still available. The two dispensers give you the same guarantees. + +## Guarantees + +- **Each task goes to one sampler at a time.** Two samplers can never hold the same attempt of a + task. +- **Tasks are not lost.** A task held by a sampler that crashed, was evicted or was fenced goes out + again. +- **A trained task is not dispensed again**, even when every trajectory scored 0. A wrong answer is + training signal. +- **Infrastructure failures are retried, with a bound.** A task whose group could not be written + because its trials failed on infrastructure (sandbox errors, aborted generations) goes out again, + at most `max_redo` more times. After that it is marked `exhausted`. +- **Restarts are safe.** A restarted sampler first marks as trained the tasks whose groups it had + already written. Then it releases the rest of its old claims. +- **The data ends cleanly.** Once every task of every pass is settled, every sampler prints + `[engine-sampler] data ended`, writes `rl_sampler_data_ended` and exits with code 0. The NC pool + ([NC_POOL.md](NC_POOL.md)) takes exit 0 as the normal end of the run: it does not restart the + sampler or replace its node. + +## Requirements + +- The bucket must support conditional writes (`PutObject` with `If-None-Match: *`). Cloudflare R2 + and AWS S3 both support them. We measured R2 directly: in 32-way races on one key, exactly one + writer won every time, and every loser got HTTP 412. The dispenser refuses to run on a store + without atomic create, such as a FUSE bucket mount or an endpoint that answers `NotImplemented`. +- Every sampler of the run must use **the same prompt file** (byte for byte), the same seed, the + same number of passes and the same hold-out list. The first sampler writes a plan file, and any + sampler that disagrees with it refuses to start. + +## Configuration + +The dispenser lives in the `prompts.dispense` section of the sampler spec: + +```json +"prompts": { + "path": "/work/sampler/prompts.jsonl", + "input_key": "messages", + "groups_per_round": 8, + "format": "agentic", + "dispense": { + "backend": "store", + "namespace": "/task-dispense", + "lease_s": 900, + "max_redo": 2, + "passes": 1, + "seed": 0, + "holdout_ids": ["", "..."], + "require_holdout": true + } +} +``` + +| Key | Meaning | Default | +|---|---|---| +| `backend` | `"store"` selects this dispenser. | `tcp` | +| `namespace` | Path of the ledger under the run's store. Use **the same value on every sampler** of the run. Give a new run a new namespace. | `task-dispense` | +| `store` | Another store root for the ledger. | the spec's `buffer_root` / `store` | +| `lease_s` | Claim lease. The holder renews it every `lease_s / 3`, so a trajectory may run far longer than the lease. A claim counts as expired only after the holder stops renewing for `lease_s`. | 900 | +| `max_redo` | How many `infra` or `expired` endings a task may have before it is marked `exhausted`. Releases (fenced, restarted, shut down) do not count. | 2 | +| `passes`, `seed` | Number of passes over the data. Pass 0 is in file order; each later pass is reshuffled by `seed`. | 1, 0 | +| `holdout_ids` / `holdout` | Task ids that must never be trained on, given inline or as a file path on the sampler. A training row that has one of these ids makes the sampler refuse to start. | none | +| `require_holdout` | Refuse to start without a hold-out list. | false | +| `mix` | `--data-mix` source weights (rows carry `metadata.yeto_data_source`). | none | +| `by_label`, `labels` | One stream per task label (`task_label`). `labels` limits this sampler to those labels. | none | +| `scan_s`, `max_scan_s` | How often an idle sampler lists the ledger (backs off up to `max_scan_s`). | 30, 120 | + +A single task can override the lease through `metadata.dispense_lease_s` in its row. + +The NC sampler launcher sets the section with flags. Pass them identically to every sampler: + +``` +--dispense-namespace /task-dispense \ +--dispense-lease-s 900 --dispense-max-redo 2 \ +--dispense-holdout ./holdout.json # read locally, embedded as holdout_ids +[--dispense-mix 0.7,0.3] [--dispense-labels swe,terminal] [--dispense-by-label] +``` + +In a pool config ([NC_POOL.md](NC_POOL.md)), put the same object under `sampler.dispense`. Don't combine the dispenser with +`--data-slot`: the dispenser replaces fixed slots. + +## Behaviour + +Each task (a "slot" of the global sequence) moves through these states: + +| Outcome of an attempt | What happens next | +|---|---| +| Group written to the buffer, any reward | `trained`. The task is not dispensed again. | +| Trials lost to infrastructure (group too small to write) | `infra`. Dispensed again, counted toward `max_redo`. | +| Holder stopped renewing (sampler died) | `expired` after `lease_s` of server time. Dispensed again, counted. | +| Holder fenced, restarted, shutting down, or round not written | `release`. Dispensed again at once, not counted. | +| More than `max_redo` counted failures | `exhausted`. Not dispensed again. | +| Row is malformed (no prompt or no task) | `dropped`. | + +- **Time.** Leases are measured in the bucket's time: the object `LastModified` and the listing + `Date` header. The sampler hosts' own clocks are never used. +- **Ownership check.** Right before a sampler writes a group, it checks that it still holds the + task: no newer attempt exists, its attempt was not ended, and its incarnation is not fenced. If + the check fails, the group is dropped with reason `dispense_lost`. +- **Fencing.** When the pool fences a sampler's shard (the node is taken for gone), that + sampler's incarnation lease moves to the fence. From then on: + - the fenced sampler renews nothing, writes no group and takes no new task; + - any other sampler that lists the ledger releases the fenced sampler's claims immediately, without + waiting for the lease to run out. + + If a fenced node writes a group anyway, the trainer rejects it, because the trainer's + incarnation gate drops groups of a superseded incarnation. +- **Mix and labels.** The weighted interleave of streams is fixed in the plan, so the mix holds + over the whole run, whichever sampler takes which task. A sampler limited to some labels takes + only those tasks and leaves the rest to the other samplers. + +## Checking status + +``` +python -m yeto.rl.engine.store_dispense status --store s3:/// \ + --namespace /task-dispense +``` + +The command prints a totals line and one row per task. The totals count tasks in each state +(`trained`, `exhausted`, `dropped`, `running`, `stale`, `queued`) plus `unclaimed`, `slots` and +`ended`. Each task row has these columns: + +| Column | Meaning | +|---|---| +| `slot` | Position of the task in the global sequence. | +| `state` | `trained` / `exhausted` / `dropped`: settled. `running`: held, lease alive. `stale`: lease ran out, not yet reclaimed. `queued`: waiting for its next attempt. | +| `attempts` | Number of attempts so far. | +| `task_id` | The task's id. | +| `holder` | Sampler of the latest attempt. | +| `elapsed_s` | Duration of the settling attempt. | +| `beat_age_s` | Time since the last renewal. | +| `failure_reasons` | One entry per failed attempt, as `attempt:kind:reason`. | + +Options: + +- `--json` prints the full report. +- `--log FILE` writes one `rl_dispense_task` JSON line per task, for dashboards. +- `--brief` uses one listing request plus one read of each task's latest claim (so `task_id` and + `holder` are shown); it skips the settlement and failure reads (`elapsed_s`, `failure_reasons`). + +The command needs read access to the bucket only. Credentials come from the usual AWS environment +or profile variables. + +Samplers also record dispenser events on their event tape: + +| Event | When | +|---|---| +| `rl_dispense_claim` | A sampler took a task. | +| `rl_dispense_settle` | A task was settled: trained, exhausted or dropped. | +| `rl_dispense_requeue` | An infrastructure failure sent a task out again. | +| `rl_dispense_release` | A claim was released (unwritten, closing, or the holder was fenced). | +| `rl_dispense_expired` | A claim expired. | +| `rl_dispense_lost` | This sampler lost a task it held: superseded or fenced. | +| `rl_dispense_recover` | The sampler restarted and recovered its earlier claims. | +| `rl_dispense_written` | Per-round summary. | +| `rl_dispense_close` | The sampler shut down. | + +## Limits + +- **Request cost.** Each listing of the ledger is one request, and an idle sampler lists at most + once per `scan_s`. A held task costs one renewal write every `lease_s / 3`, plus two or three + small reads before its group is written. +- **Expired holders.** An expired holder that is still alive (cut off from the bucket for longer + than the lease) may write its group in the short window before it notices it lost the task. The + task is then trained twice, which only happens past a full lease of lost connectivity. Fenced + holders are covered by the trainer's incarnation gate. +- **Ledger size.** The ledger keeps one object per claim, renewal, failure and settlement. Delete + the namespace when the run is archived. diff --git a/docs/K31_FULL_262K_LAYOUT.md b/docs/K31_FULL_262K_LAYOUT.md new file mode 100644 index 000000000..ecdf2cb95 --- /dev/null +++ b/docs/K31_FULL_262K_LAYOUT.md @@ -0,0 +1,279 @@ +# Kimi K3.1 full-parameter trainer: 128-expert layout on 6 x 8 B300 + +This document describes the recommended trainer layout for full-parameter RL +on Kimi-K3.1-Pro with 128 trainable experts per MoE layer. It is the single +reference for this layout. The end-to-end guide +([KIMI_K31_ASYNC_RL_GUIDE.md](KIMI_K31_ASYNC_RL_GUIDE.md), section 10) links +here. + +## Validation status + +| Configuration | Status | +|---|---| +| 64-expert full-parameter, 2 x 8 B300, end to end (NC samplers, Modal trainer) | Validated on hardware | +| 128-expert, 6 x 8 B300, **524k** total context, PP split **32/28/33** | Measured on hardware: one full training step with the stage-0 gradient fix active | +| 128-expert, 6 x 8 B300, **262k** total context, PP split **32/29/32** | Recommended production setting. Derived from a per-stage memory model fitted to the 524k measurement | +| 128-expert, 6 x 8 B300, 1M total context | Current scope: does not fit with PP3 and no activation offload (measured) | + +Numbers in this document are labelled **[measured]** or **[estimate]**. + +## 1. Which "262k"? + +The layout uses context parallelism CP=2, so each CP rank holds half of the +sequence. + +| Name | `--seq-len` (total) | `--rl-max-tokens-per-gpu` (per CP rank) | Status | +|---|---|---|---| +| 262k (recommended) | 262144 | 131072 | [estimate] | +| 524k (measured upper bound) | 524288 | 262144 | [measured] | +| 1M | 1048576 | 524288 | Does not fit [measured] | + +## 2. Scope + +- Model: `moonshotai/Kimi-K3.1-Pro`. It has 93 decoder layers, 92 of them + MoE (HF layers 1..92), with 896 routed experts per layer and top-16 routing. +- Training form: 128 uniformly spaced trainable routed experts per MoE layer + (about 389B trainable parameters). All other parameters are frozen in + MXFP4. The run uses routed-expert MXFP4 QAT, full activation recompute, R3 + routing replay, and the score_centering loss over the rollout top-128 + log-probs. +- Hardware: Modal clustered, 6 nodes x 8 B300 = 48 GPUs, with about 267.7 GiB + visible to torch per GPU. Inter-node NCCL runs over IB/RoCE. Each container + has 64 vCPUs (the Modal per-function limit) and requests 2400 GiB of memory. + +## 3. Expert selection file + +The selection file is JSON, `schema: 1`, with 92 layer keys `"1"`..`"92"`: + +```json +{ + "schema": 1, "model": "kimi_k31", "num_experts": 896, "num_trainable": 128, + "ep_size": 16, + "layers": { "1": { "trainable": [0, 7, 14, "... 128 ids"], "slot_order": ["... 896 ids"] }, "...": {} }, + "source": { "method": "uniform", "tokens": 0 }, + "sha256": "", + "trainable_sha256": "" +} +``` + +`slot_order` depends on `ep_size`. This layout runs EP16, so the file must be +built with `--ep-size 16`. A file built for another EP size is refused at +load (`slot_order was built for ep_size 8, the run uses 16`). + +```bash +# from scratch +python -m yeto.rl.adapters.miles.models.partial_experts.selection --model kimi_k31 \ + --num-experts 896 --num-trainable 128 --ep-size 16 --moe-layer-offset 1 \ + --mode uniform --num-moe-layers 92 --output +# from a file built for another EP size (same trainable set, same trainable_sha256) +python -m yeto.rl.adapters.miles.models.partial_experts.selection \ + --input --ep-size 16 --output +python -m yeto.rl.adapters.miles.models.partial_experts.selection --validate +``` + +## 4. Layout + +| Item | Value | +|---|---| +| Parallelism | PP3 x TP8 x CP2 x EP16, DP1, sequence parallel on. TP groups span nodes: pass `--rl-allow-cross-node-tp`. | +| PP split, 262k | **32 / 29 / 32** [estimate]: `--decoder-first-pipeline-num-layers 32 --decoder-last-pipeline-num-layers 32` | +| PP split, 524k | **32 / 28 / 33** [measured]: `--decoder-first-pipeline-num-layers 32 --decoder-last-pipeline-num-layers 33` | +| Tokens per CP rank | `--rl-max-tokens-per-gpu = seq-len / CP` (131072 at 262k). A full-length sample then never splits across microbatches. A smaller value is refused. | +| Microbatches per step (m) | At least PP (3). Read the actual value from the `num_microbatches=[...]` log line. Packing short traces lowers m. | +| Distributed timeout | `--rl-distributed-timeout-minutes 60` at 262k, `120` at 524k. The first step's JIT plus long-sequence pipeline traffic took 662 s [measured], longer than the default. | + +The split flags reach Miles through `YETO_MILES_TRAIN_EXTRA_ARGV`. The +launcher validates the split with `check_pipeline_split` before any GPU +starts (`yeto launch ... --dry-run` shows the result). + +Why the split is uneven: under 1F1B, stage 0 holds the most in-flight +microbatches and is the tightest stage. A split such as 31/29/33 moves the +pressure to stage 1 and is not recommended. + +## 5. Software + +### 5.1 Trainer image + +Use the full-parameter default image: + +``` +docker:ghcr.io/agentenv/yeto-miles-k31@sha256:2de5d796de19d38f4e3165efce2a9528a7f7d3e60e3cb32fe29fd00c6abf9b5b +``` + +It carries Miles `k31/integ-trainer` `1f29e43043`, which contains every Miles +change this layout depends on: + +| Miles change | Purpose | Setting | +|---|---|---| +| DSA indexer scored in query blocks (`c5393bbeb4`) | Keeps the frozen indexer's score matrix bounded at long context | `MILES_K31_INDEXER_QUERY_CHUNK` (default 8192) | +| Attention-residual aggregation in token blocks (`a0cef1d401`) | Bounds the fp32 temporaries at 262k per rank | `MILES_K3_ATTN_RES_TOKEN_CHUNK` (default 4096) | +| Stage-0 expert gradients under full recompute (`2fc2b14adc`) | Ensures the trainable experts on PP stage 0 receive gradients | built in | +| Finite score-centering loss at head mass >= 1 (`0cb40f2fcf`) | Keeps loss and gradients finite | built in | + +The 524k measurement ran with these same Miles changes applied on an earlier +base image. The full-parameter image above is the one this repository pins +and the one the validated Modal end-to-end run used. + +### 5.2 yeto + +This repository includes the stage-0 gradient hook +(`yeto.rl.adapters.miles.models.kimi_k31.first_stage_grad.before_train_step`) +and the uneven PP split support. Recommended practice: keep the hook enabled +as a second safeguard for stage-0 gradients, as in the measured run. + +### 5.3 Environment + +| Setting | Measured (524k) | Recommended (262k) | +|---|---|---| +| `MILES_K31_INDEXER_QUERY_CHUNK` | 2048 | 1024 [estimate] | +| `MILES_K3_ATTN_RES_TOKEN_CHUNK` | 4096 | 2048 [estimate] | +| `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` | on | off: a colocated Miles island launched through yeto does not accept it (torch_memory_saver) [estimate] | +| `NCCL_RAS_ENABLE` | 0 | 0 | +| `OMP_NUM_THREADS` | 8 | 8 | +| `MILES_MXFP4_QAT_TRITON` | 1 | 1 | +| `--log-probs-chunk-size` (Miles) | 4096 | 16384; lower to 4096 if the log-prob phase runs out of memory | +| Optimizer | CPU offload 1.0 | CPU offload 1.0 | + +## 6. Launch command + +The trainer-island part of `trainer.argv`. All site values are placeholders. + +```bash +SEQ=262144; CP=2; MTPG=$((SEQ / CP)) # 524k: SEQ=524288, last stage 33, timeout 120 + +yeto launch --training-mode rl \ + --gpu modal:6x8xb300 --cluster-prefix \ + --rl-image "docker:ghcr.io/agentenv/yeto-miles-k31@sha256:2de5d796de19d38f4e3165efce2a9528a7f7d3e60e3cb32fe29fd00c6abf9b5b" \ + --model moonshotai/Kimi-K3.1-Pro --model-revision \ + --modal-model-volume \ + --data --data-revision \ + --rl-prompt-column --rl-label-column \ + --reward-function \ + --trust-remote-code --tuning full --rl-model-recipe kimi_k31 \ + --rl-expert-selection --rl-routed-expert-qat mxfp4 \ + --expert-parallel 16 --tensor-parallel 8 --pipeline-parallel 3 --context-parallel 2 \ + --rl-allow-cross-node-tp \ + --rl-sample-role trainer --rl-single-island-no-sync --controller local \ + --rl-buffer-shards -0,-1,-2 --rl-stall-timeout 7200 \ + --rl-checkpoint-store s3:///runs/ \ + --rl-object-store-endpoint \ + --modal-store-secret \ + --inner-lr 1e-4 --rl-lr-schedule constant \ + --rl-max-policy-age 12 --rl-algorithm-spec \ + --rl-release-every 2 --rl-release-skip-busy \ + --rl-loss score_centering --rl-score-centering-is tis --rl-score-centering-tis-clip 2.0 \ + --rl-rollout-top-logprobs-num 128 \ + --rl-allow-unverified-mechanism losses:score_centering \ + --rl-allow-unverified-mechanism features:rollout_logprobs_as_old \ + --rl-allow-unverified-mechanism features:no_grpo_std_normalization \ + --rl-allow-unverified-mechanism loss_aggregations:token \ + --rollout-batch-size 8 --n-samples-per-prompt 8 \ + --rollout-max-response-len $((SEQ - 16384)) --seq-len $SEQ \ + --rl-activation-recompute --rl-max-tokens-per-gpu $MTPG \ + --rl-distributed-timeout-minutes 60 \ + --rl-optimizer-offload-fraction 1.0 --modal-memory-gib 2400 --modal-cpu 64 \ + --modal-env OMP_NUM_THREADS=8 --modal-env MILES_MXFP4_QAT_TRITON=1 \ + --modal-env NCCL_RAS_ENABLE=0 \ + --modal-env MILES_K31_INDEXER_QUERY_CHUNK=1024 \ + --modal-env MILES_K3_ATTN_RES_TOKEN_CHUNK=2048 \ + --modal-env "YETO_MILES_TRAIN_EXTRA_ARGV=--log-probs-chunk-size 16384 --no-check-for-nan-in-loss-and-grad --custom-megatron-before-train-step-hook-path yeto.rl.adapters.miles.models.kimi_k31.first_stage_grad.before_train_step --decoder-first-pipeline-num-layers 32 --decoder-last-pipeline-num-layers 32" \ + --modal-env YETO_RL_NAN_SKIP=max_consecutive=2,max_fraction=0.05,min_rounds=20 \ + --total-steps --rl-stop-after-rounds --rl-cut-every +``` + +Notes: + +- Run it with `--dry-run` first (`yeto run up --dry-run` does this). The dry + run checks the selection against `--expert-parallel` and the PP split, and + prints the learner command. +- `--no-check-for-nan-in-loss-and-grad` with `YETO_RL_NAN_SKIP` turns a + non-finite step into a counted skip. Remove both for strict behaviour. +- The algorithm values (learning rate, policy age, release cadence, TIS clip) + are the formal-run defaults, not layout requirements. + +## 7. Measured data (6 x 8 B300) + +Synthetic long-context rollouts replayed through the Miles train loop. Length +is per CP rank. Memory is torch `max_memory_allocated / max_memory_reserved` +in GiB per stage. + +| Round | Length / rank (total) | m | PP split | Stage-0 fix | step s | tok/s | s0 | s1 | s2 | +|---|---|---|---|---|---|---|---|---|---| +| **upper bound** | **262k (524k)** | **4** | **32/28/33** | **on** | **1019.8** | **2423** | **224.5 / 241.2** | **209.5 / 231.1** | **209.6 / 232.4** | +| 1M | 524k (1M) | 4 | 40/28/25 | on | out of memory on s0 | | | | | + +- At 524k, nvidia-smi peaks were about 269.8k / 259.8k / 261.3k MiB per + stage. Non-torch overhead was about 21-22 GiB per GPU, and stage 0 had about + 4 GiB of headroom. +- Numerics: all 92 MoE layers received non-zero gradients, and no step + produced NaN. Chunked and unchunked runs gave a bit-identical loss at 131k + per rank. +- Initialisation took about 19 minutes (1125.6 s) from launch to first + generate. Budget it on every relaunch. +- Earlier rounds at m=1 (31/31/31, 38/29/26) understate memory and time and + are not a basis for sizing. + +## 8. Current scope and recommended practice + +**Current scope** + +- The 262k split 32/29/32 comes from a per-stage memory model: + peak = 2.8 GiB x layers + k x 0.4375 GiB x sum(1 + rows) + about 30 GiB, + with k = 3/2/1 in-flight microbatches on s0/s1/s2. The model matched the + 524k run within about 5 GiB. It predicts about 67 GiB of headroom at 262k + [estimate]. +- The recommended 262k settings differ from the measured run in three ways: + expandable segments off, smaller chunk sizes, and log-prob chunk 16384. +- With real sampler buffers, the 128-expert path has completed + initialisation and reached generation. A first optimizer step on that path + is the next validation milestone. +- 1M total context needs about 456 GiB on stage 0 against about 268 GiB + available. It requires a different layout (deeper PP or activation offload). +- Optimizer CPU offload takes a noticeable share of short test steps. At + production batch sizes it is estimated at 2-5% [estimate]. +- A tensor-parallel split of the DSA indexer exists in Miles + (`MILES_K31_INDEXER_TP_SPLIT=1`, off by default). It is not yet validated + for production. + +**Recommended practice** + +- Build the selection file with `--ep-size` equal to `--expert-parallel` (16). +- Keep `--rl-max-tokens-per-gpu` at `seq-len / CP`, and keep m >= PP. +- Set `--rl-distributed-timeout-minutes` (60 at 262k, 120 at 524k). +- Keep the stage-0 gradient hook in `YETO_MILES_TRAIN_EXTRA_ARGV`. +- Use learning rates around 1e-4 for the full-parameter form. The MXFP4 + release rounds to nearest, so very small updates (around 1e-6) can leave the + released weights unchanged. +- Do not enable debug probes (per-layer NaN probes allocate extra memory). +- Request `--modal-cpu 64`; higher values are refused by Modal. + +## 9. Validating on your cluster + +Run these in order and stop at the first failure. + +1. **Dry run.** It must exit 0, report 48 GPUs, and show the split flags and + the stage-0 hook in `YETO_MILES_TRAIN_EXTRA_ARGV`. +2. **First step memory.** Record torch `max_memory_reserved` / + `max_memory_allocated` for one rank per stage, and sample nvidia-smi + `memory.used`. Treat less than 15 GiB of nvidia-smi headroom on any stage + as a stop signal. Confirm `num_microbatches=[m]` with m >= 3. +3. **Stage-0 gradients.** After step 1, check that the trainable experts of + stage-0 layers (for example HF layers 1, 10, 30) have non-zero gradient + norms, like layers on later stages. +4. **Numerics.** Loss and grad_norm are finite, and + `train_rollout_logprob_abs_diff` stays at your sampler's noise floor. +5. **Time.** Discard step 1 (JIT and autotune). Record `perf/step_time` and + `perf/actor_train_time` from step 2. + +**If a stage runs out of memory**, adjust in this order: + +1. Identify the stage from the GPU in the error message. +2. Stage 0: lower `MILES_K31_INDEXER_QUERY_CHUNK` (2048, 1024, 512) and + `MILES_K3_ATTN_RES_TOKEN_CHUNK` (4096, 2048, 1024), then move one layer off + stage 0 (for example 31/30/32). Each layer is about 2.8 GiB. +3. Stage 1 or 2: move a layer toward stage 0 if it has headroom, or lower the + chunk sizes. +4. Log-prob phase: lower `--log-probs-chunk-size` to 4096. +5. Large "reserved but unallocated" memory indicates fragmentation; reduce + the chunk sizes. +6. If memory is still short, use a shorter context. diff --git a/docs/KIMI_K31_ASYNC_RL_GUIDE.md b/docs/KIMI_K31_ASYNC_RL_GUIDE.md new file mode 100644 index 000000000..0f723e8d0 --- /dev/null +++ b/docs/KIMI_K31_ASYNC_RL_GUIDE.md @@ -0,0 +1,821 @@ +# Kimi K3.1 async GRPO (agentic RL): end-to-end guide + +This guide takes you from an empty control host to a running, monitored and +cleanly stopped asynchronous GRPO run of Kimi K3.1 with agentic (Harbor) +environments. One run config drives everything through one command, +`yeto run up`. + +Detailed references live in their own documents and are linked rather than +repeated here: + +| Topic | Reference | +|---|---| +| NC sampler pool: every `pool.json` field, `pool-*` commands, billing | [NC_POOL.md](NC_POOL.md) | +| Store task dispenser (`sampler.dispense`) | [DISPENSER.md](DISPENSER.md) | +| Sandbox hosts, `yeto sandbox`, `agentic.envs` | [SANDBOX_HOSTS.md](SANDBOX_HOSTS.md) | +| Run health breakers, log-prob guards | [BREAKER.md](BREAKER.md) | +| Resource ledger, budget breaker, cleanup | [COST_GUARD.md](COST_GUARD.md) | +| Trainer flags, rounds, cuts, releases, padding | [MILES_RL.md](MILES_RL.md) | +| 128-expert full-parameter layout on 6×8 B300 | [K31_FULL_262K_LAYOUT.md](K31_FULL_262K_LAYOUT.md) | +| Dashboard details, alert thresholds | [DASHBOARD.md](DASHBOARD.md) | +| Every `yeto` command and flag | [CLI_REFERENCE.md](CLI_REFERENCE.md) | + +## Contents + +1. [Architecture](#1-architecture) +2. [Prerequisites](#2-prerequisites) +3. [Configuration](#3-configuration) +4. [Launch](#4-launch) +5. [Operating a running job](#5-operating-a-running-job) +6. [Dashboard](#6-dashboard) +7. [Breakers and cost protection](#7-breakers-and-cost-protection) +8. [Troubleshooting FAQ](#8-troubleshooting-faq) +9. [Images](#9-images) +10. [Full-parameter parallel layout](#10-full-parameter-parallel-layout) + +--- + +## 1. Architecture + +The run is split into four independent parts that only talk through the +object store and a few well-defined network paths: + +- **Trainer island** on Modal B300 GPUs (`yeto launch --training-mode rl + --rl-sample-role trainer ...`). It reads sample groups from the buffer, + trains, and publishes policy releases. +- **Sampler islands** on National Compute (NC) nodes, one sampler per node, + managed by the NC pool loop (`python -m yeto.launch.nc pool`). Each sampler + runs the inference engine described by the sampler spec, claims tasks from + the dispenser, runs agentic trials and writes groups to its own buffer + shard. +- **Object store** (R2 or any S3-compatible bucket): the sample buffer + (`sample-buffer/shards/`), the task dispenser ledger, the policy + releases, the run's event tapes and the trainer's checkpoint store. +- **Sandbox host(s)** running the Harbor task containers. Samplers reach them + through the reverse tunnel configured in the spec (`agentic.setup.tunnel`) + and the Docker endpoint (`agentic.setup.sandbox.docker_host`). + +The control host (your workstation or a small VM) runs `yeto run up`, which +in turn runs the pool loop, the trainer launcher, the dashboard and the +monitor. + +``` + +-----------------------------------------------+ + | Control host | + | yeto run up --config run.json | + | |- preflight (dry runs, argument checks) | + | |- NC pool loop (order / setup / replace) | + | |- trainer launcher (yeto launch) | + | |- dashboard (loopback HTTP, reads store) | + | '- monitor (budget, wall clock, silence) | + +------+-------------------+--------------------+ + | | + orders, SSH setup | | launches + v v + +------------------------------------+ +---------------------------------+ + | NationalCompute sampler islands | | Modal B300 trainer island | + | node -0 .. -N | | Miles trainer | + | engine (vLLM) + agentic sampler | | reads groups, trains, | + | claims tasks, runs trials | | publishes policy releases, | + +----+------------------+------------+ | writes cuts | + | ^ +---------+---------------+-------+ + | trials | releases ^ | + v | | groups | releases, + +------------------+ | +--------------------+---------------v-----+ + | Sandbox host(s) | +------+ Object store (R2 / S3): | + | Harbor task | | sample-buffer/shards/ (groups) | + | containers | | task dispenser ledger (tasks) | + +------------------+ | policy releases (weights) | + ^ | event tapes, checkpoint store | + '-- reverse tunnel ---+--------------------------------------------+ + (sampler <-> sandbox) +``` + +Data flow per step: samplers claim tasks from the dispenser, generate +trajectories against the sandboxes, and write complete groups to their shard; +the trainer reads groups up to `--rl-max-policy-age` versions old, trains, +and releases a new policy; each sampler loads the release and continues. + +## 2. Prerequisites + +### Accounts and credentials + +All values below are placeholders. Keep real values in files outside the +repository and point the configs at them. + +| What | Used by | How it is provided | +|---|---|---| +| Modal account and token | trainer | Modal CLI login on the control host; `modal` SDK installed (preflight checks it) | +| Modal secret holding the store credentials | trainer | trainer flag `--modal-store-secret ` | +| Object store bucket and credentials | everything | run config `env.AWS_PROFILE: ""`; pool `r2_env_file: ""` for the nodes | +| NC accounts | pool | `YETO_NC_ACCOUNTS` in the run config `env` and `accounts` in `pool.json` (`` list) | +| NC tool directory | pool | `env.NC_TOOL_DIR` (see [NC_POOL.md, Before you start](NC_POOL.md#before-you-start)) | +| SSH keys for sandbox host and tunnel | samplers | spec `agentic.setup.sandbox.key.path`, `agentic.setup.tunnel.key.path` | +| Agentic environment file | samplers | pool `agentic_env_file: ""` (HARBOR_*, DOCKER_HOST, ...) | + +An NC account holds one node at a time. To run `N` samplers and keep a +replacement ready, provide at least `N + 1` accounts. + +### Images + +- **Sampler image** on NC: the vLLM build the sampler spec pins + (`engine.server.engine_version`, `engine.options.pin_version`). The pool + pulls it during node setup; override with `image` in `pool.json`. +- **Trainer image** on Modal: selected by the trainer flags (see + `examples/k31/launch-trainer-modal-b300.sh` and [MILES_RL.md](MILES_RL.md)). + The run's code is shipped as a code pack (`--rl-code-pack`, default `auto`). + Building and pinning the image: [section 9](#9-images). + +### Model weights + +- **Trainer**: the base checkpoint on a Modal volume + (`--model `, `--model-revision `, + `--modal-model-volume `). +- **Samplers**: the MXFP4 checkpoint in the bucket. The pool pulls it onto + each new node through `model_pull` (`src_key: ""`, + `max_mb_s`, `timeout_s`); the spec's `engine.server.base_dir` points at the + local copy. +- **Expert selection**: the trainer's `--rl-expert-selection ` and the + spec's `expert_selection_sha256` must describe the same file. Format and + generation: [K31_FULL_262K_LAYOUT.md](K31_FULL_262K_LAYOUT.md#3-expert-selection-file). + +### Sandbox host + +Prepare one or more hosts with Docker reachable over SSH. Task sets and +readiness are described in [SANDBOX_HOSTS.md](SANDBOX_HOSTS.md). `yeto run up` +calls `yeto sandbox status` in preflight and `yeto sandbox prepare` before +ordering any node. + +## 3. Configuration + +A run uses three files: + +- `run.json`: the run config read by `yeto run up / status / down / scale`; +- `pool.json`: the NC pool config ([NC_POOL.md](NC_POOL.md)); +- the sampler spec: the engine, sampling, prompts and agentic sections of each + sampler (`pool.json` `spec`). Starting points are in `examples/k31/` + (`sampler-spec.nc-mi355x-vllm.json`). + +### 3.1 `run.json` + +```json +{ + "comment": "Kimi K3.1 async GRPO, agentic", + "run": "", + "store": "s3:///runs/", + "env": { + "AWS_PROFILE": "", + "YETO_NC_ACCOUNTS": ",,", + "YETO_NC_STATE_DIR": "", + "NC_TOOL_DIR": "" + }, + "preflight": {"commands": [["python", ""]]}, + "sandbox": { + "envs": "", + "docker_host": "ssh://", + "max_run_containers": 40 + }, + "pool": {"config": "", "interval_s": 30}, + "trainer": { + "argv": ["--training-mode", "rl", "--gpu", "modal:x8xb300", "..."], + "usd_per_h": 0, + "progress_paths": [] + }, + "first_group": {"timeout_s": 5400, "poll_s": 60}, + "dashboard": {"port": 8795}, + "budget": { + "usd": 0, + "wall_h": 3.0, + "nc_usd_per_h": 0, + "silence_min": 30, + "release_nc_on_breach": false + } +} +``` + +Fill the `0` prices with your contracted rates; they drive the spend estimate. + +| Field | Meaning | +|---|---| +| `run` | Run name. Also the state directory `~/.yeto/rl-up//` (pids, logs, step times) and the `ai.yeto.run` label of sandbox containers. Required. | +| `store` | Run store URI. The dashboard reads it; the first-group wait lists `sample-buffer/shards/` under it. Required. | +| `env` | Environment for every subprocess (paths may use `~`). `YETO_RESOURCE_LEDGER` defaults to `~/.yeto/runs//resources.jsonl`. | +| `preflight.commands` | Extra checks run during preflight; any non-zero exit stops `up`. | +| `sandbox.envs` | The `agentic.envs` file for `yeto sandbox status / prepare / teardown`. Optional. | +| `sandbox.docker_host` | Docker endpoint used to count and remove this run's trial containers. | +| `sandbox.max_run_containers` | Warning threshold for this run's container count (a warning, not a breaker). | +| `pool.config` | Path of `pool.json`. Required. | +| `pool.interval_s` | Seconds between pool control ticks (default 30). | +| `trainer.argv` | The `yeto launch` arguments of the trainer. Required. | +| `trainer.usd_per_h` | Trainer price per hour for the spend estimate. | +| `trainer.progress_paths` | Extra files whose growth counts as trainer progress for the silence backstop. | +| `first_group.timeout_s`, `poll_s` | How long `up` waits for the first buffer object before starting the trainer (default 5400 s, polled every 60 s). | +| `dashboard.port` | Loopback port of the dashboard (default 8795). | +| `budget.usd` | Spend cap (trainer plus NC estimate). | +| `budget.wall_h` | Wall-clock cap from the pool start (default 3). | +| `budget.nc_usd_per_h` | Price of one NC node hour for the estimate. | +| `budget.silence_min` | Minimum silence window of the backstop; the effective window is `max(4 x --rl-stall-timeout, silence_min)`. | +| `budget.release_nc_on_breach` | `true`: a budget or wall-clock breach performs the full `down` including the NC pool. Default `false` (see section 7). | + +### 3.2 Trainer arguments (`trainer.argv`) + +The full trainer command, with comments for every flag, is in +`examples/k31/launch-trainer-modal-b300.sh`; all flags are documented in +[MILES_RL.md](MILES_RL.md). The ones you will tune most: + +| Flag | Meaning | +|---|---| +| `--rl-sample-role trainer` | This island trains only; samplers live elsewhere. | +| `--rl-buffer-shards -0,-1,...` | The buffer shards the trainer reads. List every shard index the pool may ever use (see `shard_limit` below). | +| `--rollout-batch-size` | Prompts (groups) per training round. | +| `--n-samples-per-prompt` | Group size; must match the spec's `sampling.n`. | +| `--rl-max-policy-age` | How many versions old a group may be; needs a matching algorithm spec (`--rl-algorithm-spec`, e.g. `examples/k31/algorithm-spec.async-age2.json`). | +| `--rl-checkpoint-store` | Where round cuts and the ledger are kept; enables resume across launches. | +| `--rl-cut-every N` | Keep a round cut every `N` rounds (plus the last round). See also `--rl-cut-keep`, `--rl-cut-auto-mtbf-s`. | +| `--rl-release-every N` | Release every N-th policy version to the samplers (default 1). Needs `--rl-max-policy-age >= N`. | +| `--rl-stall-timeout` | Trainer hang detection in seconds. `yeto run up` refuses `0`. | +| `--rl-stop-after-rounds N` | Stop after N rounds with a final cut. | + +### 3.3 `pool.json` + +```json +{ + "run": "", + "route": "ssh", + "target": 2, + "accounts": ["", "", ""], + "shard_base": "", + "shard_limit": 8, + "machine_type": "mi355x", + "spec": "", + "run_prefix": "runs/", + "r2_env_file": "", + "agentic_env_file": "", + "model_pull": {"src_key": "", "max_mb_s": 2500, "timeout_s": 7200}, + "replace_lead_s": 3600, + "end_margin_s": 600, + "sampler": {"dispense": {"backend": "store", "namespace": "task-dispense"}}, + "elastic": {"max_nodes": 3, "max_orders": 10, "engine_stall_s": 600} +} +``` + +Key fields (the complete list is in +[NC_POOL.md, the pool config](NC_POOL.md#the-pool-config-pooljson)): + +| Field | Meaning | +|---|---| +| `target` | Samplers to keep running. At most the number of `accounts`. Change it at run time with `yeto run scale`. | +| `accounts` | NC accounts the pool may order on; each must also be in `YETO_NC_ACCOUNTS`. Accounts beyond `target` are used for replacements. | +| `shard_base` | Shard prefix: nodes write `-0`, `-1`, ... Also names the pool state file `pool-.json`. | +| `shard_limit` | How many shard indexes the run may take; set it to the number of shards in the trainer's `--rl-buffer-shards`. The usable range is also bounded by the port range of `agentic.setup.tunnel.listen` (one port per shard). Shard indexes are not reused, so replacements consume new indexes. | +| `elastic.max_nodes` | Most billed nodes at once (default `2 x target`). | +| `elastic.max_orders` | Most orders over the whole run. | +| `elastic.engine_stall_s` | Seconds without generated tokens while requests are pending before an engine counts as stalled (default 600). | +| `elastic.max_sampler_restarts` | In-place restarts of a crashed sampler before its node is replaced (default 2). | +| `sampler.dispense` | Task dispenser settings ([DISPENSER.md](DISPENSER.md)). | +| `model_pull` | Model download from the bucket onto each new node. | + +`pool --dry-run` (part of preflight) refuses unknown keys, so a typo is +reported before anything is ordered. + +### 3.4 The sampler spec + +Start from `examples/k31/sampler-spec.nc-mi355x-vllm.json`. Placeholders such +as `` are filled per node by the pool. Key fields: + +| Field | Meaning | +|---|---| +| `engine.adapter` | `vllm-server` for the K3.1 sampler. | +| `engine.server.base_dir` | Local path of the MXFP4 checkpoint on the node. | +| `engine.server.max_model_len` | Engine context length. | +| `engine.server.extra_args` | Extra engine flags. They come after the preset's flags and take precedence. | +| `engine.options.k31_preset` | Serving preset (`amd-article`, `rl-precision`, `vllm-recipe`, `rl-precision-cuda`). The ROCm presets set `--gpu-memory-utilization 0.88`. | +| `engine.options.tp`, `max_num_seqs` | Tensor parallel degree and concurrent sequences per engine. | +| `sampling.n` | Samples per prompt (group size). | +| `sampling.max_tokens`, `temperature`, `top_p`, `top_k` | Generation settings. | +| `max_backlog_groups` | Groups a sampler may hold ahead of the trainer before it pauses. | +| `reload_max_defer_s` | Longest time, in seconds, a downloaded policy release may wait for running agentic trials before the sampler loads it anyway (default 600; `null` waits indefinitely). A forced load is reported as `rl_sampler_policy_loaded` with `forced: true` and `deferred_s`; trials that span the switch are excluded from training (lost as `served_by_other_version`, or aborted when the switch falls inside a model call) and counted in `rl_agentic_group`. | +| `prompts.groups_per_round` | Groups a sampler produces per round. | +| `reward`, `reward_kind` | Reward function import path and kind. | +| `agentic.*` | Agentic trial settings: `max_in_flight`, `turn_max_tokens`, `max_response_len`, `infra_retries`, `task_defaults`, `setup` (sandbox, tunnel, tasks), `envs` ([SANDBOX_HOSTS.md](SANDBOX_HOSTS.md)). | + +**GPU memory utilization.** To change it, add the flag to +`engine.server.extra_args`, for example +`["--gpu-memory-utilization", "0.85"]`. +Recommended practice: keep the preset's 0.88 on MI355X TP8. The collectives' +scratch memory lives outside vLLM's budget, and the preset keeps that +headroom free. + +## 4. Launch + +### 4.1 Dry run + +``` +yeto run up --config --dry-run +``` + +Preflight runs every check and orders nothing: + +1. the Modal SDK check (when the trainer targets `modal:`), and a refusal of + `--rl-stall-timeout 0`; +2. `python -m yeto.launch.nc pool --config --dry-run`: the pool + config, the accounts, and the sampler spec built locally the way the + sampler builds it at start (island config, engine adapter, rollout fields, + reward import, `agentic.setup`); +3. `yeto sandbox status` (when `sandbox.envs` is set); +4. `yeto launch --dry-run`; +5. every `preflight.commands` entry. + +Output goes to `~/.yeto/rl-up//preflight.log`. Each step logs its return +code (`rc=0` passes). The run ends with either +`dry run: preflight ok, nothing ordered` or +`preflight failed: nothing ordered (see preflight.log)`. If an earlier trainer +launch of this run exists, preflight also notes that the trainer will start +with `--rl-ledger-set-aside-before` (see section 5.3). + +### 4.2 Start + +``` +yeto run up --config +``` + +Steps: preflight, `yeto sandbox prepare`, the pool loop (detached), the wait +for the first sample group, the trainer (detached), the dashboard, then the +monitor in the foreground. `up` is resumable: running it again skips steps +whose process is still alive. + +Options: + +| Option | Use | +|---|---| +| `--dry-run` | Preflight only. | +| `--skip-preflight` | Resume after a preflight that already passed. | +| `--trainer-now` | Start the trainer immediately instead of waiting for the first group, so the Modal GPUs and the NC node setup come up in parallel. This shortens time to first step. The trainer may idle until samplers deliver groups. | +| `--no-monitor` | Return after the dashboard starts. Recommended practice: keep the monitor running, since it enforces the budget, wall-clock and silence guards. | + +Status at any time: + +``` +yeto run status --config +``` + +It prints the step history, process liveness, buffer object count, the spend +estimate, `pool-status`, the dispenser status, and the sampler target/nodes. + +## 5. Operating a running job + +### 5.1 Scaling sampler islands + +``` +yeto run scale --config --samplers +yeto run scale --config --samplers --retire # pick which node leaves +``` + +This runs `python -m yeto.launch.nc pool-scale --config --target ` +on the running pool. You can also call `pool-scale` directly. + +A scale request is checked before anything is ordered. It is refused (and +nothing changes) when the target exceeds the number of `accounts`, +`data_slots`, `elastic.max_nodes`, `elastic.max_orders`, +`elastic.max_run_spend_ucr`, or the shard range (`shard_limit` / +tunnel ports). On scale down the pool fences the leaving node's shard first, +so its claims return to the dispenser. Details: +[NC_POOL.md, pool-scale](NC_POOL.md#change-the-number-of-samplers-while-the-run-trains-pool-scale). + +### 5.2 Automatic node replacement + +The pool loop probes every node each tick and replaces lost capacity without +operator action: + +1. **Detect.** A node that is evicted, unreachable for longer than + `elastic.unreachable_grace_s`, or past its lease is treated as lost. +2. **Fence.** The node's shard is fenced (`nc_elastic_shard_fenced`), so a + node that comes back cannot add groups the trainer would take; its + dispenser claims are released. +3. **Release and re-order.** The ticket is deleted and a new node is ordered + on a free account (`nc_elastic_ordered`). Before a planned lease end, the + replacement is ordered `replace_lead_s` ahead so there is no gap. +4. **Setup.** The new node runs `agentic.setup` (sandbox access, tunnel, + tasks), the `model_pull`, `node_hooks`, and the sampler start. +5. **Ready.** The sampler joins (`nc_elastic_member_ready`) and continues the + shard's prompts on a new shard index. + +The whole sequence appears in the dashboard's NC pool events. Event names and +the per-case behaviour are listed in +[NC_POOL.md, What the pool does](NC_POOL.md#what-the-pool-does). + +### 5.3 Resuming training + +- **From the latest cut.** With `--rl-checkpoint-store`, the trainer keeps + round cuts (`--rl-cut-every`, `--rl-cut-keep`). Run + `yeto run up --config ` again with the same run config; the + trainer resumes from the latest cut. A changed run configuration is refused + unless you add `--rl-resume-allow-config-change` (the differences are + recorded in the `rl_resume` event). +- **From a policy release.** Add `--rl-init-from-release latest` (or + `r-v` / a version) to `trainer.argv`. The trainer loads that + release instead of the base checkpoint, and version numbering continues + after it. Current scope: the optimizer state starts fresh on this path. +- **From the base, on an existing run.** When `yeto run up` restarts a trainer + of a run that launched before, it adds + `--rl-ledger-set-aside-before `. A start from the base then moves the + earlier batch ledger aside (`ledger.before-`) instead of stopping on it. + No manual cleanup is needed. + +### 5.4 Upgrading the code of a running run + +The run's code is pinned as a code pack at the first start. Resumed and +replacement islands get the same pack, so a relaunch never picks up local +changes by accident. + +To move the run to the current code, add +`--rl-code-pack-upgrade ""` to `trainer.argv` (with `--rl-code-pack` +`git-patch` or `release`), then run `yeto run up` again. The upgrade is +accepted when the algorithm's semantic hash is unchanged, and the manifest +records the old and new pack with your reason. To resume deliberately on the +pinned pack, use `--rl-code-pack-keep` instead. For algorithm changes, +Recommended practice: start a new run (another `--cluster-prefix`). + +### 5.5 Stopping + +``` +yeto run down --config +``` + +`down` performs, in order: + +1. writes a read-only dashboard export to `~/.yeto/rl-up//dashboard.html`; +2. stops the dashboard and the trainer (`yeto down `); +3. releases the NC pool (`pool-stop`); +4. tears down the sandboxes (`yeto sandbox teardown`) and removes this run's + trial containers; +5. verifies with `pool-status` and reports `down: verified` or + `down: NOT verified, see down.log`. + +Recommended practice: run `down` when you finish. NC leases bill until their +tickets are deleted. `pool-stop` alone releases only the NC nodes +([NC_POOL.md, Stopping a pool](NC_POOL.md#stopping-a-pool)). + +## 6. Dashboard + +`yeto run up` starts `yeto dashboard serve --store --port ` on +the control host. It binds to loopback only and prints +`http://127.0.0.1:/`. If the control host is remote, open a tunnel from +your laptop: + +``` +ssh -N -L :127.0.0.1: +``` + +Then browse to `http://127.0.0.1:/`. For an offline copy, use +`yeto dashboard export --store -o `. `down` writes one +automatically. More detail: [DASHBOARD.md](DASHBOARD.md). + +All numbers come from the event tapes; `n/a` means the value is not on the +tapes. + +### Async GRPO panel + +**Trainer rounds.** One row per trained round: + +| Column | Meaning | +|---|---| +| reward mean +- std, p10 / p50 / p90 | Reward of the trained samples. The main learning signal. | +| train-rollout KL, logprob \|diff\| | Gap between trainer and sampler log-probs. Breaker limits: [BREAKER.md](BREAKER.md). | +| TIS clipfrac | Fraction of importance weights clipped. | +| grad_norm | Gradient norm. | +| resp len mean / p95, truncated | Response lengths and the share cut at the token limit. | +| groups | Groups consumed (trained) this round. | +| stale dropped, policy ages | Groups dropped as older than `--rl-max-policy-age`, and the age histogram of trained groups. | + +The totals line above the table sums trained groups, stale drops, +off-policy-gate drops, short groups, carried-over groups and **padded +samples**. Padding tops up a round that cannot be filled with whole groups, +and the real samples' gradient is scaled by the padded fraction. +The trainer reports the ratio as the metric `sample_buffer/padded_fraction` +([MILES_RL.md](MILES_RL.md)). Current scope: the dashboard shows the padded +sample count; the fraction is read from the trainer metrics. A fraction that +stays high means samplers are not keeping up. Add samplers or raise +`--rl-max-policy-age`. + +**Samplers.** One row per sampler island: + +| Column | Meaning | +|---|---| +| status | `healthy`; `starting`; `waiting: backlog full (trainers behind)`; `waiting: waiting for a policy release`; `stale`; `lost`; `left the run ()`; `finished`; `stopped`. | +| policy | `v` the sampler currently generates with. | +| policy age p50 / max, histogram | Trainer version minus the version a round was generated with. | +| generate s p50 / max | Seconds per generation round. | +| groups written | Groups the sampler wrote to its shard. | +| zero-variance seen / dropped, truncated, dropped groups | Group quality counters. | +| trials retried / lost | Agentic trials retried, and trials lost (`rl_agentic_trial_lost`). | +| reward | Last reward the sampler reported. | + +**Buffer backlog.** Groups written by all samplers minus the trainers' read +position. A growing backlog means trainers are behind. A backlog near zero +means samplers are the bottleneck. + +**Policy releases.** Each trainer release and when each sampler finished +loading it. NC pool events appear as dashed lines. + +**NC pool events.** The `nc_elastic_*` stream (ordered, member_ready, +shard_fenced, evicted, node_lost, engine_stalled, released, ...). Use it to +follow a replacement end to end. + +### Other panels + +- **Island health**, **Event stream**: per-island state and the raw events. +- **Cost (estimate, not billing data)**: spend per resource from the resource + ledger and the price table (`--prices`). The cost bar uses `--budget`. + Current scope: the figures are estimates; your provider invoices are + authoritative. `yeto run status` shows the same spend estimate the monitor + uses (`spend_usd_estimate`). + +## 7. Breakers and cost protection + +The monitor in `yeto run up` checks the run every minute: + +| Guard | Trigger | Action | +|---|---|---| +| Budget | spend estimate > `budget.usd` | Stop the trainer. | +| Wall clock | time since pool start > `budget.wall_h` | Stop the trainer. | +| Trainer exit | launcher finished `FAILED` or non-zero | Stop the trainer path cleanly. | +| Silence backstop | trainer log and `progress_paths` unchanged for `max(4 x --rl-stall-timeout, silence_min)` | Stop the trainer. | +| First group | no buffer object within `first_group.timeout_s` | Stop before the trainer starts. | +| Sandbox containers | count > `sandbox.max_run_containers` | Warning only. | + +**Trainer first, NC kept.** By design, a breach stops only the trainer +(`yeto down `). The NC pool, its nodes and the sandboxes stay up, so a +resume does not repeat node setup and model pull. The monitor prints which +nodes still bill and the release command. + +**Forced NC release at 1.2x.** After a budget or wall-clock breach, the +monitor stays up and prints the billing banner on every poll. Once the spend +passes `1.2 x budget.usd`, or the wall time passes `1.2 x wall_h`, it runs +`pool-stop` itself. It exits when no node bills. + +**Opt-in full stop.** Set `"release_nc_on_breach": true` in `budget` to make a +budget or wall-clock breach perform the full `yeto run down` immediately. + +The pool also has its own guards (`elastic.max_nodes`, `max_orders`, +`max_run_spend_ucr`, order breaker); see [NC_POOL.md](NC_POOL.md). Log-prob +and health breakers inside the run are described in [BREAKER.md](BREAKER.md). +The resource ledger, idle shutdown and cleanup are described in +[COST_GUARD.md](COST_GUARD.md). + +## 8. Troubleshooting FAQ + +Logs are in `~/.yeto/rl-up//` (`preflight.log`, `sandbox.log`, +`pool.log`, `trainer.log`, `dashboard.log`, `down.log`). + +**A sampler stays `waiting`.** +Check the reason shown in its status: +- `backlog full (trainers behind)`: the sampler has `max_backlog_groups` + ready and pauses by design. The trainer is the bottleneck. Check the trainer + rounds and `trainer.log`. Scaling samplers down saves cost while this + persists. +- `waiting for a policy release`: the sampler cannot generate more within + `--rl-max-policy-age` until a newer release arrives. Check the Policy + releases panel for the latest release and load time. If the trainer has + not released recently, look at the trainer first. + +**A sampler shows `starting` for a long time.** +Node setup includes the image pull, `model_pull` and engine start. Follow the +NC pool events and `pool.log`. The pool gives up after +`elastic.start_timeout_s` and orders a replacement. + +**Trials are lost (`trials retried / lost` grows).** +Each lost trial emits `rl_agentic_trial_lost` with the error, the call, the +token window and the trial directory. Typical causes: a model reply that +cannot be aligned to the recorded tokens, or a prompt longer than the +trainer's prompt width. Infrastructure failures are retried up to +`agentic.infra_retries`. A small, steady rate is normal for agentic tasks. +A sudden rise across all samplers usually points at the sandbox host: +run `yeto sandbox status --envs --store ` +([SANDBOX_HOSTS.md](SANDBOX_HOSTS.md)). + +**`engine_stalled` in the NC pool events, then the sampler recovers.** +The engine had requests pending but produced no tokens for +`elastic.engine_stall_s`. The pool fenced the shard and restarted the engine +in place. This is the automatic recovery path and needs no action. After +`max_stall_restart_failures` consecutive restarts without progress, the node +is retired and replaced (`engine_stall_loop`). An idle engine (waiting for the +trainer) is never counted as stalled. Current scope: the separate +event-tape stall detector (`elastic.stall_s`) is off by default; set it above +your longest round if you want it. + +**A release is not loaded by a sampler.** +The sampler emits `rl_sampler_release_not_loaded` once per release and cause, +with `reason` and `detail`. Check, in order: that the release finished +uploading (Policy releases panel), that the node can read the store +(`r2_env_file`), and that the release matches the sampler (the reason names +the mismatch). The sampler keeps generating with its current version +within `--rl-max-policy-age` and retries on the next poll. + +**The pool logs `CONFIGURATION ERROR`.** +A sampler exited with code 2 because its spec or arguments were refused. The +pool keeps the node and orders nothing more. Fix the spec, then run +`python -m yeto.launch.nc pool-retry --config ` to restart the +sampler on the same node(s) +([NC_POOL.md, pool-retry](NC_POOL.md#retry-after-a-configuration-error-pool-retry)). + +**`yeto run scale` is refused.** +The message names the limit (accounts, `max_nodes`, `max_orders`, spend, or +the shard range). Shard indexes are not reused. For a long run with many +replacements, set a generous `--rl-buffer-shards` list and `shard_limit` up +front. + +**`up` stops at the first group.** +No buffer object appeared within `first_group.timeout_s`. Check `pool.log` +for node setup progress and the samplers' event tapes. On the next attempt, +consider `--trainer-now` so the trainer start overlaps the node setup. + +**The trainer stopped with `BREAKER: trainer: log / tape silent`.** +The launcher was alive but produced no output for the silence window. +Inspect `trainer.log` and the Modal app. Resume with `yeto run up` (section +5.3). NC nodes are kept, so the restart is quick. + +**`down` reports `NOT verified`.** +Read `down.log`. Usually a node release is still in progress. Run +`python -m yeto.launch.nc pool-status --config ` and, if nodes are +listed, `pool-stop` again. + +## 9. Images + +### 9.1 Trainer image (Miles) + +Use this image for the full-parameter recipe: + +``` +docker:ghcr.io/agentenv/yeto-miles-k31@sha256:2de5d796de19d38f4e3165efce2a9528a7f7d3e60e3cb32fe29fd00c6abf9b5b +``` + +It is the default `--rl-image` for `--rl-model-recipe kimi_k31 --tuning full` +(`MILES_K31_FULL_IMAGE` in `yeto/rl/adapters/miles/pins.py`). It is also the +image the trainer ran in the validated Modal end-to-end run. It carries Miles +`k31/integ-trainer` `1f29e43043`. Recommended practice: pin this digest +explicitly in `trainer.argv` +(`--rl-image docker:ghcr.io/agentenv/yeto-miles-k31@sha256:2de5d796...`, the +full value above) so the run does not depend on a default. + +The image `yeto-miles-k31@sha256:63ad0d2a...` (`MILES_K31_IMAGE`) is for the +LoRA recipe. + +**Registry.** The default registry is `ghcr.io/agentenv`, and no setting is +needed to use it. Set `YETO_IMAGE_REGISTRY=/` only when you +pull from a mirror that carries the same digests. Because images are +referenced by digest, a mirror cannot serve different bytes under the same +pin. For a private registry, add `--rl-image-private` with +`SKYPILOT_DOCKER_USERNAME` / `SKYPILOT_DOCKER_PASSWORD` (see +[CLOUDS.md](CLOUDS.md)). + +The island checks its Miles checkout against the commit recorded for the +image digest (`IMAGE_MILES_COMMITS` in `pins.py`). + +### 9.2 Sampler image (vLLM ROCm) + +NC MI355X samplers run the upstream vLLM ROCm image, pinned in +`yeto/rl/sampler_pins.py` (`VLLM_SAMPLER_IMAGE_ROCM`): + +``` +docker:docker.io/vllm/vllm-openai-rocm@sha256:5f47cc1004d9a7ac65f8a913725edb20c62c99a07e31cbf4ba21741bd0b4f9b1 +``` + +The image reports vLLM version `0.0.0+mum.14d1ec2336`. The sampler spec's +`engine.server.engine_version` and `engine.options.pin_version` refer to it. +The node installs yeto into the image at start. + +### 9.3 Building your own Miles image + +Source: `docker/miles-k31/` in this repository. Most customers use the +published digest above. This section applies when you rebuild it. + +- `docker/miles-k31/versions.lock.json` pins the upstream base image by index + and manifest digest. It also pins the base's build arguments (the Miles, + SGLang and Megatron-LM commits, CUDA, NCCL, FlashInfer) and the expected + package versions. +- `docker/miles-k31/build.sh` builds without a Docker daemon. It uses `crane` + to append one overlay layer to the pinned base. The layer holds the Miles + fork checkout, the SGLang fork checkout, the `peft` and `pyrage` wheels, the + Megatron patches, `/opt/yeto/image-manifest.json` and the image-assert lock + under `/opt/yeto/image`. + +``` +MILES_SRC= SGLANG_SRC= \ +MILES_BRANCH= SGLANG_BRANCH= \ +DEST_REPO=//yeto-miles-k31 \ + docker/miles-k31/build.sh --push +``` + +Before it builds anything, the script compares the base's recorded build +arguments with the lock and stops on any mismatch. It tags the image +`k31--`. After a push it prints the digest and writes +`build-record.json`. The Miles commit inside the image is the commit +`MILES_BRANCH` resolved to at build time. It is recorded in +`/opt/yeto/image-manifest.json`, in the `ai.yeto.miles.commit` label and in +the image-assert `commits.json`. + +### 9.4 Code packs: `git-patch` and `release` + +The image fixes the frameworks. The yeto code on the island is pinned +separately as the run's **code pack** (`--rl-code-pack`): + +| Mode | What runs on the island | Pinned by | +|---|---|---| +| `git-patch` (the `auto` default with a bucket or `modal-volume://` checkpoint store) | Your local git tree (commit plus a patch of uncommitted and untracked files), packed once at the first start | The pack digest in the run manifest | +| `release` | `/opt/yeto/src` inside a yeto release image; no local tree is shipped | The release image digest | +| `off` | The local tree at each launch | Nothing | + +`scripts/build_yeto_release_image.sh [--push]` +(`DEST_REPO=//yeto-release`) builds a release image. It adds a +`git archive` of one clean commit to the image pinned as `MILES_NEXT_IMAGE`, +and refuses a dirty tree. See `docker/yeto-release/README.md`. + +Current scope: release images are built on `MILES_NEXT_IMAGE`, which is the +LoRA-recipe image. Recommended practice for K3.1 full-parameter runs: use +`git-patch` (or `auto`) with the full-parameter image in section 9.1. + +Moving a running run between packs or images goes through +`--rl-code-pack-upgrade ""` ([section 5.4](#54-upgrading-the-code-of-a-running-run)). + +## 10. Full-parameter parallel layout + +The recommended 128-expert layout, its measured data, the selection file +format, the full launch command and the validation steps are in +**[K31_FULL_262K_LAYOUT.md](K31_FULL_262K_LAYOUT.md)**. This section is a +summary. Where the two differ, that document applies. + +### 10.1 Validation status + +- Validated: 64-expert full-parameter on 2×8 B300 (end to end with NC + samplers). +- Measured: 128-expert at 524k total context on 6×8 B300, PP split 32/28/33. +- Recommended for production: 128-expert at 262k total context on 6×8 B300, + PP split 32/29/32. This split is derived from the memory model fitted to + the 524k measurement. + +### 10.2 Layouts + +| Trainable experts | GPUs | Layout | Context | Status | +|---|---|---|---|---| +| 64 | 16 (2×8 B300) | TP8 PP1 EP16 CP1 | 32k | Validated | +| 128 | 48 (6×8 B300) | PP3 x TP8 x CP2 x EP16, DP1, split 32/29/32 | 262k | Recommended | +| 128 | 48 (6×8 B300) | PP3 x TP8 x CP2 x EP16, DP1, split 32/28/33 | 524k | Measured upper bound | + +The 128-expert layout needs `--rl-allow-cross-node-tp`, +`--rl-max-tokens-per-gpu` = `seq-len / CP`, the split flags and the stage-0 +gradient hook in `YETO_MILES_TRAIN_EXTRA_ARGV`, and +`--rl-distributed-timeout-minutes 60`. See +[K31_FULL_262K_LAYOUT.md](K31_FULL_262K_LAYOUT.md#4-layout). + +Parallelism notes, from the launcher: + +- `--expert-parallel` shares ranks with TP x DP; it is not a further + multiplier. It must divide both 896 and the trainable count, and it must + equal the selection file's `ep_size`. +- Expert tensor parallel is fixed at 1 by the Miles backend. +- An uneven PP split is passed with + `--decoder-first/last-pipeline-num-layers` and checked before any GPU + starts. + +### 10.3 Expert selection and moving from 64 to 128 + +The selection file format and commands are in +[K31_FULL_262K_LAYOUT.md, section 3](K31_FULL_262K_LAYOUT.md#3-expert-selection-file). +To move from 64 to 128 experts: + +1. Generate a new selection with `--num-trainable 128 --ep-size 16`. +2. Update `--rl-expert-selection` and `--expert-parallel` in `trainer.argv`. +3. Update `expert_selection_sha256` in the sampler spec. + +The trainable set changes, so this starts a new run rather than resuming a +64-expert one. + +### 10.4 Memory estimator and headroom + +The trainer dry run prints a first-order per-GPU estimate +(`kimi_k31_full.memory`, from `memory_estimate` in +`yeto/launch/k31_checks.py`). It includes `host_optimizer_gib_per_gpu` and +warns above 85% of the card's capacity. The estimate leaves out allocator +fragmentation, NCCL buffers and CUDA graphs. On the validated 64-expert +layout: + +| | Estimator | Observed | +|---|---|---| +| HBM peak per GPU | ~162 GiB | ~251 GiB | +| Host memory per node | ~1.1 TiB offloaded optimizer | ~1.9 TB of 2 TB | + +Recommended practice: plan HBM at about 1.6x the estimator and host memory +at about 1.8x the offloaded-optimizer estimate. For the 128-expert layout, +size from the measured per-stage figures in +[K31_FULL_262K_LAYOUT.md](K31_FULL_262K_LAYOUT.md#7-measured-data-6-x-8-b300), +and confirm headroom on the first step as described there. + +## Planned enhancements + +- The padded-sample fraction as a dashboard column next to the padded sample + count. +- Pool configuration changes (accounts, limits) without a pool restart. Today, + adding accounts takes a pool restart; the target can change at run time. +- Release-mode images on the K3.1 full-parameter base. +- A measured first step of the 262k layout (split 32/29/32) under production conditions. diff --git a/docs/MILES_RL.md b/docs/MILES_RL.md index 951cecf19..e278a2432 100644 --- a/docs/MILES_RL.md +++ b/docs/MILES_RL.md @@ -439,12 +439,21 @@ store-coupled pair or a convertible pool (no syncer), and `--rl-data-passes`. `/opt/yeto/image-manifest.json` records every SHA. `--rl-image` overrides the image; a private one takes its login from `SKYPILOT_DOCKER_*` or `--rl-image-private`. -- **Fork overlay.** Some critic-family Miles flags are not in the pinned image - commit. With `--rl-miles-overlay auto` (default) the island setup applies a - sha256-checked patch from `yeto/rl/adapters/miles/overlays/` over the - image's Miles checkout, only when the algorithm spec uses a fork-only flag; - `critic-c357` forces it, `off` never applies it. The run manifest then - records the overlay sha256. +- **Sampler-side SGLang pin (separate).** The engine sampler + (`yeto.rl.engine.engine_sampler` with the `sglang-server` adapter) drives a + stand-alone SGLang HTTP server and only needs its public RL surface, so it + follows upstream `sgl-project/sglang` main, not the Miles fork: + `SGLANG_SAMPLER_UPSTREAM_COMMIT` / `SGLANG_SAMPLER_BASE_IMAGE_CUDA` (the + `lmsysorg/sglang` nightly the yeto sampler image `SGLANG_SAMPLER_IMAGE_CUDA` + is built on), `SGLANG_SAMPLER_IMAGE_ROCM` (the upstream MI35x dev image of + PR #48427; yeto is pip-installed on the node). The adapter records the + server's `SGLANG_BUILD_COMMIT` in `server_info` and whether it matches the + pin (`sglang_pin_match`; a mismatch is recorded, not refused). Moving the + sampler pin never moves `SGLANG_K31_COMMIT` (the Miles rollout worker's + SGLang) and vice versa. +- **No source overlay.** The island runs exactly the image's pinned Miles + checkout; critic-family flags that only a fork branch carries are refused + by their spec-level `*_not_at_pin` rejections until that fork is in the pin. ### Variance-aware GRPO sampling @@ -885,7 +894,7 @@ the adapter declares it) and accepted with them. {"schema": "yeto-rl-algorithm-spec-v2", "correction": {"method": "custom", "tis_clip_low": 0.5, "tis_clip": 5.0, "mismatch_metrics": true, - "function": {"path": "miles.backends.training_utils.loss_hub.corrections.icepop_function", + "function": {"path": "miles.backends.training_utils.loss.hub.corrections.icepop_function", "sha256": "971ccb0bf00b43b0582839c5b8dc05e91162c878ab7ec0ca878e3b1e668f5318"}}} ``` @@ -908,7 +917,7 @@ the adapter declares it) and accepted with them. "correction": {"method": "custom", "mis_level": "geometric", "mis_mode": "mask", "mis_lower_bound": 0.9999, "mis_upper_bound": 1.0001, "function": {"path": "yeto.rl.algos.vendor.miles_mis.compute_mis_weights_with_cp", - "sha256": "5c1dce0763f645f7491140e45c6c2b47774364f81ecd7c724cc71560424fefb4"}}} + "sha256": "5f0fc101735dbe49e2c20e0758f9746b64535c7511fa234e789966c1f8bcbf93"}}} ``` ## Flash-Next (Qwen3.8-Flash-Next) path @@ -1018,6 +1027,33 @@ budget (`scripts/benchmark_rl.py --arms single,federated,decoupled` runs both). Held-out reward and pass@k must not regress beyond seed noise, and the invariants in the [Development Guide](#development-guide) must hold. +## Kimi-K3.1 trainable forms + +`--rl-model-recipe kimi_k31` trains one of two forms: + +- `--tuning full` (production): the routed experts named by + `--rl-expert-selection` train in place (64 of 896 per MoE layer is the + production setting; any count `--expert-parallel` divides works), next to the + frozen MXFP4 experts. The launcher validates the selection file (schema, + sha256, EP size, and the MoE layers when the checkpoint's `config.json` is + readable) and the dry run prints `kimi_k31_full.memory`, a rough per-GPU + budget with warnings for long sequences (for example 262144 tokens with + `--context-parallel 1`). Build a selection with + `python -m yeto.rl.adapters.miles.models.partial_experts.selection` + (`--validate FILE` checks one). +- `--tuning lora` (infrastructure tests): an adapter over the frozen checkpoint. + `--lora-targets auto` is `attention`. `all-linear` also adapts the dense MLP, + the shared expert and every routed expert; an A-card vLLM sampler keeps the + routed experts as MXFP4 bytes and merges each release into BF16 weights, so it + can only apply attention (and BF16 dense/shared) adapters. A + `--rl-sample-role trainer` launch therefore refuses `all-linear` unless + `--rl-sampler-engine sglang` declares samplers that serve MoE LoRA; with + `--rl-sampler-engine vllm` it names the reason. Behind the syncer only + `attention` is accepted. + +A trainer island (`--rl-sample-role trainer`) reads groups from the store, not +`--data` rows: its `--data` may be a local file without a revision. + ## Buffer-trainer releases and spot rollout islands These options apply to a buffer trainer (`--rl-sample-role trainer`) that @@ -1043,6 +1079,41 @@ publishes policy releases to separate rollout islands. (`step_mfu`, `wall_mfu`) from the trainer's event tape, for models whose framework MFU counter does not cover them (Nemotron-H). +- **Role switch of an engine sampler island** (`python -m + yeto.rl.engine.role_switch --spec SPEC`, `yeto/rl/engine/role_switch.py`): + the island starts as an engine sampler (its spec plus a `role_switch` + section) and, when the section's trigger fires at a round boundary + (`after_seconds`, `after_groups` written by this node, the local marker + `$YETO_ROLE_SWITCH_MARKER`, or the store marker + `convertible//switch.json` that `role_switch.write_switch_marker` + writes), stops its engine process, checks the device memory is free and + runs `role_switch.trainer.command` (the trainer island's learner command, + `--rl-sample-role trainer --rl-single-island-no-sync --tuning full`) in the + same container. The trainer reads every island's buffer shard from the + newest cut's watermark; the role record `convertible//role.json` + carries the watermark per shard. With `then_sampler` the node comes back + as a sampler on the newest release (`base_dir`/`base_version`, gated by the + spec's `expert_selection_sha256`). This is the full-parameter sibling of + the convertible pool below: a Miles colocated node converts in place; an + engine sampler island restarts as a trainer process. Unlike the pool, + `--tuning full` is accepted on convertible nodes and on synced K3.1 + trainers (`--rl-model-recipe kimi_k31`), which release whole checkpoints. +- **Short rounds and padding** (buffer reader): Miles trains exactly + `rollout_batch_size x n_samples_per_prompt` samples. A round the buffer + cannot fill with whole groups (short groups, groups dropped as too small, + a group carried over to the next take) is topped up with padding samples: + copies of the round's shortest real sample with `loss_mask` 0, + `remove_sample` set and `metadata.yeto_padding`, each in a one-sample group + of its own (advantage 0). They add no loss term, but Miles still divides + the round's loss by the full global batch, so **the real samples' gradient + shrinks by the padded fraction** (a round 25% padded trains its real + samples with 0.75 of their gradient). yeto does not rescale it. Watch + `sample_buffer/padded_fraction` (and `sample_buffer/padded_samples`) on the + trainer's metrics: a fraction that stays high means the samplers do not + keep up (raise the sampler count or `max_policy_age`) and the effective + learning rate is lower by that factor. The rollout metrics + (`rollout/*`) leave padding samples out. + Spot rollout islands on Nebius have four operator tools in `yeto/rl/spot/` (each with `--help`): diff --git a/docs/MODELS.md b/docs/MODELS.md index 45d7fcf2d..0494e709d 100644 --- a/docs/MODELS.md +++ b/docs/MODELS.md @@ -48,6 +48,7 @@ Supported by the alias table but not yet run end to end. | `kimi-k2-thinking` | `moonshotai/Kimi-K2-Thinking` | 2060 | | `kimi-k25` | `moonshotai/Kimi-K2.5` | 2060 | | `kimi-k26` | `moonshotai/Kimi-K2.6` | 2060 | +| `kimi-k31-pro` | `moonshotai/Kimi-K3.1-Pro` | (Hub) | | `deepseek31` | `deepseek-ai/DeepSeek-V3.1` | 1343 | | `deepseek-r1` | `deepseek-ai/DeepSeek-R1` | 1343 | | `deepseek4pro` | `deepseek-ai/DeepSeek-V4-Pro` | 3200 | diff --git a/docs/NC_POOL.md b/docs/NC_POOL.md new file mode 100644 index 000000000..04dca9b05 --- /dev/null +++ b/docs/NC_POOL.md @@ -0,0 +1,400 @@ +# National Compute sampler pool + +`python -m yeto.launch.nc pool` keeps a set of sampler nodes on National +Compute (NC) running for a whole RL run. Each node runs one engine sampler +that writes its own shard of the run's sample buffer in your object store. +The pool orders nodes, starts the samplers, replaces nodes before their +leases end, and recovers from evictions, lost nodes and crashed samplers. + +> **Billing warning.** NC does **not** end a lease at `lease_until`. When +> `lease_until` passes, the ticket stays `active` and keeps billing until it +> is deleted. The pool always deletes its nodes before `lease_until`, but +> only while its control loop is running. If the loop is gone, the nodes +> keep billing until you run `pool-stop` (see [Stopping](#stopping-a-pool)). + +This page covers: + +- [Before you start](#before-you-start) +- [The control host](#the-control-host) +- [The pool config](#the-pool-config-pooljson) and every field's default +- [Commands](#commands): `--dry-run`, `pool`, `pool-status`, `pool-stop` +- [What the pool does](#what-the-pool-does) +- [Billing](#billing) + +## Before you start + +You need: + +- **NC accounts.** An NC account holds **one node at a time**. To run N + samplers you need at least N accounts. To have a replacement ready before + each lease ends (with no gap in sampling), you need more than N. +- **The NC order tools** `nc-manage` and `nc-query`, plus the controller tool + directory, which provides the workspace relay. The pool reaches nodes over + SSH through a small relay in each account's NC workspace. +- **An object store bucket** (S3 or R2) that is shared with the trainer. +- **A sampler spec** for the engine sampler. Start from + `examples/k31/sampler-spec.nc-mi355x-vllm.json`. + +Set these environment variables on the control host: + +| Variable | Meaning | Default | +|---|---|---| +| `YETO_NC_ACCOUNTS` | Comma-separated list of the NC accounts this tool may order on. Required. An account in `pool.json` that is not on this list is refused. | none | +| `NC_MANAGE`, `NC_QUERY` | Command lines of the order tools. Can be overridden in `provider`. | `nc-manage`, `nc-query` | +| `NC_TOOL_DIR` | Directory of the controller tool (`tool/core.mjs`, `tool/wsrun.mjs`). | none | +| `NC_NODE_BIN` | The `node` binary that runs the tool. | `node` | +| `YETO_NC_STATE_DIR` | The pool's state directory: ticket ledger, leases, pool state, events, pid and stop files. | `~/.yeto/nc` | +| `YETO_RESOURCE_LEDGER` | The run's cost ledger (set by `yeto launch` for its processes; set it yourself for a pool started by hand, e.g. `~/.yeto/runs//resources.jsonl`). Every order, grant and release of an NC ticket is recorded there as an `nc_lease`, so `yeto cost`, the budget and `yeto cleanup` see the pool's nodes. | none (not recorded) | +| `YETO_NC_RELAY_DIR` | Directory of the relay inside the NC workspace. | the workspace home `/data/home/.yeto-nc` | + +The **store env file** (`r2_env_file`) must be mode `0600`. It holds the +sampler's store credentials, in this format: + +``` +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_ENDPOINT_URL=https:// +AWS_REGION=auto +R2_BUCKET= +``` + +The bucket variable is named `R2_BUCKET` by default. If your env file names +it differently, set `bucket_env` in the pool config. The samplers write to +`s3:///runs/`. + +## The control host + +The pool is a **foreground loop**. It decides every order, replacement and +release, so it must run for the whole RL run. Run it: + +- on an always-on machine (a small VM is enough; it needs no GPU), not on a + laptop; +- under `tmux`, `screen` or a `systemd` unit, so an SSH disconnect does not + kill it. + +Once the loop stops, nothing replaces or releases the nodes. What happens +depends on how it stopped: + +| How the loop ends | What happens to the nodes | +|---|---| +| `pool-stop`, `SIGTERM`, `SIGINT` (Ctrl-C), `SIGHUP` (the terminal closed) | The loop drains every node (stops the sampler, deletes the ticket, confirms it reads `ended`), then exits. If a drain keeps failing, it prints `NODES STILL BILLING` with the tickets and the stop command. | +| The process crashes or is killed with `SIGKILL`, or the host loses power or network | **The nodes keep billing**, past `lease_until`. Run `pool-stop --config ` from the same host and state dir as soon as you can. `pool-status` reports this case with a `NODES STILL BILLING` banner. | +| The control host is gone for good | The state dir is gone with it. Release each ticket in the NC console, or run `python -m yeto.launch.nc down --account --ticket ` from a host with the same ledger. Keep `YETO_NC_STATE_DIR` on a disk that outlives the host. | + +The pool refuses to start a second loop on a pool whose loop is still +running. + +The trainer has the same requirement: a no-sync trainer runs with +`--controller local`, so its control loop also lives on your host for the +whole run. + +## The pool config (`pool.json`) + +A minimal config: + +```json +{ + "target": 2, + "accounts": [, , ], + "shard_base": "nc", + "run": "", + "machine_type": "mi355x", + "spec": "sampler-spec.json", + "run_prefix": "", + "r2_env_file": "/store.env", + "sampler": {"dispense": {"backend": "store", "namespace": "/task-dispense"}}, + "model_pull": {"src_key": "models/"}, + "elastic": {"max_run_spend_ucr": 2000000000} +} +``` + +The pool refuses unknown keys, at the top level and inside `sampler`, +`model_pull`, `node_hooks`, `backlog`, `elastic` and `provider`. A typo is an +error, never silently ignored. + +### Top-level fields + +| Field | Meaning | Default | +|---|---|---| +| `target` | Samplers to keep running. Must be at least 1 and at most the number of `accounts` (one node per account). | required | +| `accounts` | NC accounts the pool may order on. No repeats; each must be in `YETO_NC_ACCOUNTS`. Accounts beyond `target` are used for replacements. | required | +| `shard_base` | Prefix of the buffer shards (`-0`, `-1`, ...). Also names the state file `pool-.json`. | required | +| `run` | Run tag written to every ticket in the ledger. The pool and `pool-stop` only ever delete tickets that carry this tag. One word, no spaces. | `shard_base` | +| `machine_type` | NC machine type to order. This is **not** a Hugging Face model; the served model is set in the sampler spec. A value containing `/` is refused. | `mi355x` | +| `model` | Deprecated name of `machine_type`. It still works but prints a deprecation note; it cannot be set together with `machine_type`. | none | +| `route` | `ssh`: SSH + docker on the node (the verified route). `sky`: SkyPilot SSH node pool (refused on nodes with the provider's health k3s). | `ssh` | +| `spec` | Sampler spec file. `__RUN_PREFIX__` in it is replaced by `run_prefix`. | required (ssh) | +| `run_prefix` | The run's path under `s3:///runs/`. | required (ssh) | +| `r2_env_file` | Store env file (mode `0600`), see above. | required (ssh) | +| `bucket_env` | Name of the bucket variable in the store env file. | `R2_BUCKET` | +| `image` | Sampler container image. | the pinned digest of the K3.1 ROCm vLLM image (`yeto.rl.sampler_pins.VLLM_SAMPLER_IMAGE_ROCM`) | +| `protected_prefixes` | Run prefixes the sampler must never write to (for example, another live run's prefix in the same bucket). | `[]` | +| `agentic_env_file` | Second env file for agentic samplers (sandbox hosts, records proxy). | none | +| `sampler` | Per-run sampler knobs, see below. | `{}` | +| `model_pull` | Pulls the model from the bucket onto each new node, see below. | none (the model must already be on the node) | +| `backlog` | Sizes `max_backlog_groups` from the sampler layout, see below. | none (the spec's value) | +| `node_hooks` | Commands to run on each new node and before each in-place restart, see below. | `[]` | +| `data_slots` | Split the prompts into N fixed slots (`i % N`). Must be at least `target`. Leave it unset when you use the dispenser. | none | +| `replace_lead_s` | How long before a lease ends the pool orders its replacement. Cover node setup, the model pull and the engine start. | `3600` | +| `end_margin_s` | A node is drained this long before `lease_until`, whether or not its replacement is ready. | `600` | +| `stop_below_ucr` | When the shared balance drops below this, the pool stops ordering and drains every node. | `20000000` | +| `min_balance_to_order_ucr` | Minimum balance for each new order. | `stop_below_ucr` + one node hour | +| `event_command` | Shell command that gets each pool event as JSON on stdin (a pager, a chat hook). A failure is logged and never stops the pool. | none | +| `probe_timeout_s`, `probe_connect_timeout_s` | Time limits of the per-tick SSH probe of each node. | `90`, `20` | +| `provider` | `{"kind": "nc", "manage": ..., "query": ..., "tool_dir": ..., "node_bin": ...}` overrides the NC tools. `{"factory": "package.module:function"}` plugs in another supply of nodes. | `{"kind": "nc"}` | +| `model_src`, `model_dest`, `workdir` | Model pull and work directory of the `sky` route. | none | +| `comment` | Free text, ignored. | none | + +### `sampler` + +These knobs override the sampler spec on every node of the pool: + +| Field | Meaning | +|---|---| +| `dispense` | The store task dispenser (`prompts.dispense`), the same on every sampler. See [DISPENSER.md](DISPENSER.md). | +| `n` | Samples per prompt. | +| `max_tokens` | Response length cap. | +| `thinking_effort` | Thinking effort passed to the engine. | +| `drop_zero_variance` | Drop groups whose rewards are all equal. | +| `reload` | How a new policy release is loaded: `hot` or `restart`. | + +### `model_pull` + +| Field | Meaning | Default | +|---|---|---| +| `src_key` | Model directory under the bucket. You upload it there once, before the run. | required | +| `dest` | Path on the node. | `/mnt/nvme/models/Kimi-K3.1-Pro` | +| `include` | Glob of the files to pull. | all files | +| `max_mb_s` | Pull rate limit. | `2500` | +| `timeout_s`, `poll_s` | Pull deadline and status poll interval. | `3600`, `30` | + +### `backlog` + +The fields `synced` (required), `trainers`, `fragments`, `local_rounds`, +`pipeline` and `explicit_rounds` size `max_backlog_groups`, the same way the +launcher sizes it (`--synced`, `--trainers`, ... of `ssh-sampler start`). + +### `node_hooks` + +Each hook is `{"name": ..., "command": ..., "where": "node" | "local", +"timeout_s": 600}`. The command is formatted with `{alias}`, `{ip}`, +`{ticket}`, `{account}`, `{user}`, `{shard}` and `{run}`. Each hook must be +idempotent and exit 0; a non-zero exit fails the node start. + +### `elastic` + +| Field | Meaning | Default | +|---|---|---| +| `max_run_spend_ucr` | Cap on the run's estimated spend; no order is placed past it. Set it. | none (no cap) | +| `max_nodes` | Most billed nodes at once. | `2 x target` (in practice at most one per account) | +| `max_orders` | Most orders over the whole run. | none | +| `unreachable_grace_s` | How long probes must fail before a node counts as lost. Shorter failures count as network blips. | `300` | +| `control_plane_alert_s` | How long NC status calls must fail before an alert. The pool never acts on a failed status call alone. | `600` | +| `stall_s` | A joined sampler whose event tape has not grown for this long counts as crashed. Set it above your longest round. | none (off) | +| `max_sampler_restarts` | In-place restarts of a crashed sampler before its node is replaced. | `2` | +| `start_timeout_s` | Time allowed from the order to a joined sampler. | `3600` | +| `overdue_grace_s` | How long a ticket may read `active` past `lease_until` before the pool deletes it. | `0` | +| `lost_handoff_wait_s` | How long the replacement of a lost node waits for its predecessor's handoff. | `60` | +| `breaker_failures`, `breaker_cooldown_s`, `breaker_max_cooldown_s`, `breaker_max_trips` | Circuit breaker on consecutive provisioning failures: it opens for a cooldown that doubles on each trip; after `breaker_max_trips` trips, ordering stops for good. | `3`, `600`, `3600`, `3` | +| `stop_on_finish` | A sampler that exits 0 (its data ended) stops the whole pool. | `true` | + +## Commands + +### Check a config: `pool --dry-run` + +``` +python -m yeto.launch.nc pool --config pool.json --dry-run +``` + +The dry run validates the config, renders the sampler spec, checks the store +env file (it exists, has mode `0600`, and contains the bucket key) and the +order tools, then prints: + +- the orders the pool would place at start; +- the accounts left over for replacements; +- the estimated spend per hour: steady, and the peak while replacements + overlap. + +It orders nothing and calls no NC API. Exit codes: 0 (ok), 1 (problems +found), 2 (invalid config). + +### Run: `pool` + +``` +python -m yeto.launch.nc pool --config pool.json [--interval 60] +``` + +This runs the control loop in the foreground, one tick every `--interval` +seconds. The loop writes: + +- the state to `$YETO_NC_STATE_DIR/pool-.json`; +- events to `events-.jsonl`; +- its pid to `pool-.pid`. + +If the state file says the pool stopped, the loop exits at once. Move the +file away to start a new pool with the same `shard_base`. + +### Inspect: `pool-status` + +``` +python -m yeto.launch.nc pool-status --config pool.json [--events 20] +``` + +This prints the loop's process (`alive`), the members that may still hold a +ticket (`billed`, with `lease_until` and `past_lease_until`), the state file +and the last events. It is read-only. If the state file holds members and no +loop is alive, it prints the `NODES STILL BILLING` banner and exits 1. +It also prints `target` (the target the loop holds, from `pool-scale` when one +was given, else `pool.json`), `nodes` (members that may hold a ticket) and +`samplers` (`nodes/target`). `yeto run status` shows the same pair under +`samplers`. + +### Change the number of samplers while the run trains: `pool-scale` + +``` +python -m yeto.launch.nc pool-scale --config pool.json --target N [--retire ...] +yeto run scale --config run.json --samplers N [--retire ] # the same, from the run config +``` + +`pool-scale` writes the wanted target to `pool-.scale.json` in the +state dir. It never takes the pool lock: a running loop keeps it and reads the +file at its next tick (within `--interval` seconds). A loop started later reads +it at start, so the target outlives a restart; it takes precedence over +`target` in `pool.json`. Concurrent `pool-scale` commands serialize on +`pool-.scale.lock`; the last one wins. + +- **Scale up.** The loop orders the missing nodes like any fill. `pool-scale` + refuses (exit 2, nothing changed) before anything is ordered when the new + target is above the number of `accounts` (one node per account), `data_slots` + or `elastic.max_nodes`, or when the new nodes would pass `elastic.max_orders`, + `elastic.max_run_spend_ucr` (one node hour each) or `shard_limit` (the + trainer's `--rl-buffer-shards` / the tunnel port range; shard indexes are + never reused). The loop's own order gates still apply. +- **Scale down.** The loop picks the members to release: the `--retire` + tickets first, then members not yet ready, then the newest, cheapest and + shortest-lease ones. For each it fences the shard (its dispense claims go + back at once), stops the sampler, releases and deletes the ticket, checks it + reads `ended` and emits `nc_elastic_scaled_down`. A replacement in flight for + that member goes with it. Nothing is handed on and nothing is ordered for + it; the other members are not touched. `--target 0` releases every node and + keeps the loop running; a negative target is refused. `--retire` needs a + target below the current member count. + +The pool process must run code that has `pool-scale` (restart an older loop +once; H1 adopts its nodes). After that, no restart is needed to scale. + +### Stopping a pool + +``` +python -m yeto.launch.nc pool-stop --config pool.json [--wait 900] +``` + +- **The loop is running:** `pool-stop` writes a stop file. The loop picks it + up within one tick, drains every node and exits. `pool-stop` waits up to + `--wait` seconds for that. +- **No loop is alive** (it crashed, was killed, or the host rebooted): + `pool-stop` releases the nodes itself. It releases the members recorded in + the state file, plus any open ticket the ledger holds for this run's tag. + +Either way, `pool-stop` only deletes tickets that this tool created, are +tagged with this run, and are still the account's current ticket. It deletes +them by id and confirms each one reads `ended`. It never uses `--all` and +never touches another run's or another person's ticket. Exit code 0 means +every node is released. Exit code 1 prints the banner with the tickets left. + +### Retry after a configuration error: `pool-retry` + +``` +python -m yeto.launch.nc pool-retry --config pool.json +``` + +Fix `pool.json` or the sampler spec first. `pool-retry` checks the fixed +config the same way `--dry-run` does and refuses (exit 2) if it is still +wrong. If the config passes, it asks the loop to re-read the config and +restart the sampler on the same node(s) at its next tick. The shard, the data +and the model already on the node are kept. If no loop is alive, start +`pool` again; it picks up the request. + +To release a single ticket: + +``` +python -m yeto.launch.nc down --account --ticket +``` + +## What the pool does + +On each tick, the pool reads every member's ticket and probes its node with +one SSH round trip. Then it acts: + +- **Lease end.** A replacement is ordered on a free account `replace_lead_s` + before the end. It continues the old shard's prompts. Once the replacement + is ready, the old node is drained. A node is always drained `end_margin_s` + before `lease_until`. With no free account, there is a gap until the old + node is gone. +- **Eviction, lost node, overdue lease.** The node's shard is fenced, so a + node that comes back cannot add groups the trainer would take. The ticket + is deleted. The next order continues the shard. +- **Configuration error.** Before it orders anything, `pool` (and + `--dry-run`) renders the sampler spec and builds locally every part the + sampler builds at start: the island config (required keys such as + `max_backlog_groups`, sampling parameters), the engine adapter from the + `engine` section (unknown section keys, server fields and engine options + are refused; no engine process starts), the `rollout_fields`, and the reward + (`reward_kind` and the import of `reward`). A spec that fails is refused, so + you never wait for node setup to find out. The reward must be importable on + the control host too. If a sampler still exits with code 2 on a node (its spec or + arguments were refused), the pool treats it as a configuration error, not a + crash. It does not restart the sampler, fence its shard or replace the node. + It logs `CONFIGURATION ERROR`, emits `sampler_config_error`, and orders + nothing more. The node is kept (and billed) until you fix the config and run + `pool-retry`, or until its lease end, when it is drained as usual. +- **Crashed sampler.** The sampler is restarted in place, up to + `max_sampler_restarts` times. After that, the node is replaced. +- **Sampler exits 0: the data ended.** When the store dispenser has settled + every task, the sampler prints `data ended` and exits 0. The pool treats + this as the normal end of the run. It does not restart the sampler, does + not replace the node and places no new order. With `stop_on_finish` + (default), it drains every node and stops. Without it, the other samplers + finish their data or go at their lease end, and then the pool stops. +- **Stray tickets.** A patrol deletes any open ticket of this run that no + live member owns. + +Each decision is an `nc_elastic_*` event, for example `ordered`, +`member_ready`, `released`, `evicted`, `node_lost`, `lease_overdue`, +`sampler_restart`, `sampler_crash_loop`, `sampler_finished`, +`order_refused`, `breaker_open` and `pool_stopped`. + +Each sampler writes its event tape as `..jsonl` (from the +spec's `event_tape`), one file per incarnation. To read the tapes in the +dashboard, pass the directory: `yeto dashboard export --tapes `. + +The tapes stay on the nodes, under `/mnt/nvme/work/...` (the spec's `/work/` +path). Each node's local disk is destroyed when its lease is released, so +copy the tapes off while the node is up. The pool writes an SSH alias +`yeto-nc-` for each node. The alias goes through the workspace relay, +and `~/.ssh/config` includes it, so you can copy the tapes with plain `scp`: + +``` +scp 'yeto-nc-:/mnt/nvme/work/sampler/events*.jsonl' / +yeto dashboard export --tapes +``` + +## Billing + +- **A lease is not ended at `lease_until`.** The ticket stays `active` and + billed until it is deleted. The pool drains nodes before that point, and + its patrol deletes any ticket of the run found `active` past it. Both only + happen while the loop runs. +- A lease is granted for up to 4 hours (`lease_until`) and cannot be + renewed, but it does not end there by itself. Renewal is a replacement + order on another account. +- A node that drops off the network can stay `active` and billed. The pool + deletes it after `unreachable_grace_s`. +- The spend shown by `--dry-run` and tracked under `max_run_spend_ucr` is an + estimate: active time × the ticket's rate. Your account balance is the + real figure, and it is shared by everything on the organisation. +- Before you leave a run unattended: + 1. Set `elastic.max_run_spend_ucr`. + 2. Run the loop under tmux or systemd on an always-on host. + 3. Check `pool-status` now and then. + 4. When the run ends, confirm `pool-status` shows no `billed` members. diff --git a/docs/RL_ALGORITHMS.md b/docs/RL_ALGORITHMS.md index 857d8c6d5..3e6552b8e 100644 --- a/docs/RL_ALGORITHMS.md +++ b/docs/RL_ALGORITHMS.md @@ -486,10 +486,9 @@ fork. SAO math, the tied-embedding critic value head and `--critic-updates-per-step` exist only in a fork branch. `yeto/rl/algos/critic_fork.py` pins it as `CRITIC_FORK_PIN` (`FORK_COMMITS`) and the spec-level `*_not_at_pin` -rejections check it. The island image does not carry these flags; they reach -the island only through the setup-time overlay (`--rl-miles-overlay auto`, -patches in `yeto/rl/adapters/miles/overlays/`, see "Clouds, controller and -images" in [MILES_RL.md](MILES_RL.md#clouds-controller-and-images)). +rejections check it. The island image does not carry these flags, and no +setup-time source overlay exists any more: these algorithms run only once the +critic fork is part of the pinned Miles lineage. ### Spec fields and defaults diff --git a/docs/SANDBOX_HOSTS.md b/docs/SANDBOX_HOSTS.md new file mode 100644 index 000000000..ac3fde04c --- /dev/null +++ b/docs/SANDBOX_HOSTS.md @@ -0,0 +1,107 @@ +# Sandbox hosts: `yeto sandbox` and dispensing by readiness + +`yeto sandbox` prepares the task environments of an agentic RL run on one or more sandbox hosts +(docker daemons that Harbor's `remote-docker` provider drives), and records in the run store which +task is ready where. The store dispenser reads that record and gives a sampler only tasks that are +ready on its host, and only as many as its sandboxes can start now. + +## Task sets + +A task set is a directory of Harbor task directories: + +``` +/task.toml [environment] docker_image = "..." (optional when a Dockerfile is present) +/environment/Dockerfile build context of the task image (optional when docker_image is pullable) +/tests/ /solution/ +/yeto-sidecars.json optional: long-lived services this task needs on the host +``` + +A sidecar declaration (per task, or for the whole set in `sidecars_file` as +`{"tasks": {"": {...}}}`): + +```json +{"sidecars": [{"name": "task-server", "image": "ts:task-1", "build": "server/", "port": 9111, + "host_port": null, "container_name": null, "env": {"AUTH_TOKEN": "..."}, "health": "/health"}], + "prefetch": ["runner-image:tag"]} +``` + +- `host_port`: fixed host port. Without it a free port of the host's `port_range` is allocated + (ports already published on the host are skipped) and kept across re-runs. +- `container_name`: a container of that name that already runs `image` is **adopted**: recorded as + ready, never restarted, never removed. A container of that name with another image is an error + for that task, and the container is left alone. +- The published port binds the host's docker bridge gateway, probed per host. +- Text files of the task may contain `{{yeto.sidecar..url}}`, `{{yeto.sidecar..host}}`, + `{{yeto.sidecar..port}}` and `{{yeto.host.bridge_ip}}`. `prepare` writes a rendered copy of + each task per host into the stage directory (`--stage-dir`, default + `~/.yeto/sandbox//stage//`), and rsyncs it to `sync_to` when the host has `ssh`. + The sampler node of that host must use that copy as `HARBOR_TASKS_DIR`. + +## Configuration: `agentic.envs` + +The same object is read from the sampler spec (`agentic.envs`) or from a standalone JSON file. + +| Key | Meaning | Default | +|---|---|---| +| `run` | Label put on every container and image yeto starts or builds (`yeto.sandbox.run`). `teardown` removes only these. | required | +| `ready_key` | Manifest path in the run store. | `sandbox/ready.json` | +| `require_validation` | Dispense a task only after oracle and nop passed. | `true` | +| `affinity` | `strict`: a sampler takes only tasks ready on its own host. `prefer`: its own host first, then sidecar-free tasks ready on another host. | `strict` | +| `pull_parallel`, `build_parallel`, `validate_parallel` | Concurrency per host. | 4, 2, 4 | +| `hosts[]` | `name`, `docker_host` (`ssh://user@host`), `ssh` (optional, for disk, health checks and sync), `max_sandboxes` (host capacity, every owner's containers counted), `port_range`, `sync_to`. | | +| `tasksets[]` | `name`, `tasks_dir`, `ids`, `image_source` (`auto`: pull, else build; `pull`; `build`), `image_repo`, `agent_base` (`image`, optional build `context`), `bake_runtime` (`pi`), `sidecars_file`. | | + +Bridge address, docker data-root and used ports are probed on each host, never configured. + +`bake_runtime: "pi"` builds `FROM ` plus the pinned pi install, tagged +`:-pi-`, and points the rendered `task.toml` at it, so a trial finds pi +preinstalled and skips the per-trial apt/nvm/npm install. Task sets whose Dockerfile already starts +from a pi base image (`agent_base`) do not need it. + +## Commands + +``` +yeto sandbox prepare --envs spec.json --store [--host h] [--task id] [--stage-dir d] +yeto sandbox validate --envs spec.json --store [--out passed.txt] [--force] +yeto sandbox status --envs spec.json --store +yeto sandbox teardown --envs spec.json --store [--images] [--dry-run] +``` + +- `prepare` is idempotent: present images and running sidecars with the same declaration are left + alone. A task that fails is recorded with its error and the others go on; the exit code is 1 + when any task or host failed. Re-running finishes only what is missing. +- `validate` runs the oracle agent (must score 1) and the nop agent (must score 0) through yeto's + Harbor path on each host. A wrong score marks the task failed; an infrastructure error leaves it + unchecked, to be retried by the next `validate`. A task whose image digest changed is checked + again. +- `status` prints, per host: running containers against `max_sandboxes`, this run's containers, + free disk under the docker data-root, image readiness, sidecar health and how many tasks are + dispensable. +- `teardown` lists containers with `yeto.sandbox.run=`, checks each one's label again, and + removes those only. Adopted containers, Harbor trial containers and other people's containers + are never removed. + +## The readiness manifest + +`/sandbox/ready.json` (schema `yeto.sandbox-ready/v1`) holds host facts and, per task +and host: `prepared`, `image`, `digest`, `sidecars` (container, port, URL, adopted), `error`, +`prepared_at`, and `validation` (`passed` true / false / null, `oracle`, `nop`, `reason`, `at`). +Commands read, merge and write it; do not run two of them on the same run at once. + +## Dispensing + +With `agentic.envs` in the sampler spec and `prompts.dispense.backend: "store"`, each ledger scan: + +1. reads the manifest (one read per scan). The manifest is a runtime filter, not part of the + dispenser plan: every sampler keeps the same plan whatever the manifest says; +2. settles `dropped` every slot of a task whose validation failed, with the reason + (`validate_failed: :oracle_reward=0.0`); +3. claims only tasks that are ready (on the sampler's host under `strict`); a task that becomes ready + later is claimed at a later scan; +4. claims at most `free // n` groups (`n` = `sampling.n`), where `free` is the smaller of the + process cap (`HARBOR_MAX_SANDBOXES` minus held slots) and the host's `max_sandboxes` minus its + running containers (probed at most every 15 s). With no room it claims nothing and emits + `rl_dispense_wait_capacity`, so no lease is held by a task waiting for a sandbox. + +The sampler's host is `YETO_SANDBOX_HOST` (a host name), else the host whose `docker_host` equals +`HARBOR_ENV_KWARGS.docker_host` or `DOCKER_HOST`, else the only host. diff --git a/examples/README.md b/examples/README.md index 5bfeabc4a..fa0bc467c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -26,17 +26,45 @@ yeto launch --training-mode rl \ --seq-len 2048 --inner-lr 1e-5 --seed 17 --trust-remote-code --on-demand ``` -- Without `--rl-image` the island runs the pinned default ports image - (`MILES_NEXT_IMAGE` in `yeto/rl/adapters/miles/pins.py`). +- Without `--rl-image` the island runs the pinned default image + (`yeto launch ... --dry-run` prints it as `rl_image`). Default images are + pinned by digest on `ghcr.io/agentenv`; set `YETO_IMAGE_REGISTRY=/` + to pull the same digests from a mirror (see docs/CLOUDS.md). - `--rl-prompt-column` / `--rl-label-column` name the dataset columns; the RL data path otherwise expects `messages` / `prompt` / `input` and `label`, and never guesses. A named column that is missing is an error. -- `--reward-function` takes any `file.py:func` inside the repo; the file is - synced to the island and its sha256 is attested like any other reward. +- `--reward-function` takes any `file.py:func` inside the repo (the launcher + turns it into the module name, here `examples.math_reward:reward_func`) or a + `package.module:func`; the source is synced to the island and its sha256 is + attested like any other reward. No `PYTHONPATH` is needed after + `pip install -e .`. - `--gpu` can be any cloud entry; the syncer can run in any supported region. - Any Qwen3 checkpoint works; `enable_thinking: false` keeps rollouts short enough for `--rollout-max-response-len 1024`. +## Built-in rewards + +Each is passed as `--reward-function :` and ships with the +`yeto` package, so the island image already has it. + +| Reference | Grades | Needs | +|---|---|---| +| `yeto.rl.numeric_answer_reward:score` | one-number labels (`\boxed{N}` or `Answer: N`) | nothing beyond `yeto` (use it for vLLM / A-card samplers) | +| `yeto.rl.gsm8k_reward:score` | GSM8K `#### N` labels | nothing beyond `yeto` | +| `yeto.rl.math_reward:reward_func` | MATH-style `\boxed{...}` answers (mathd + sympy) | the Miles package (present in the Miles island image) | +| `yeto.rl.word_diversity_reward:score`, `yeto.rl.token_diversity_reward:score` | dense diversity signal for rehearsals | nothing beyond `yeto` (test rewards) | + +## Kimi-K3.1 (`k31/`) + +- `launch-trainer-modal-b300.sh`: the full-parameter trainer island (64 of 896 + routed experts per MoE layer trainable) on Modal B300; append `--dry-run` to + validate the selection and print a rough per-GPU memory estimate. +- `launch-convertible-modal.sh`: one node that samples, then trains. +- `sampler-spec.*.json`: engine sampler specs (CPU rehearsal, MI355X SGLang / + vLLM); `sampler-node-nc-mi355x.sh` starts one on a node. +- `algorithm-spec.async-age2*.json`: AlgorithmSpecs that tolerate a policy age + of 2 (the `-tis` one adds the TIS correction a synced run needs). + ## Other examples - [`custom_reward/`](custom_reward/README.md): a minimal framework-neutral diff --git a/examples/k31/README.md b/examples/k31/README.md new file mode 100644 index 000000000..803add042 --- /dev/null +++ b/examples/k31/README.md @@ -0,0 +1,18 @@ +# Kimi K3.1 examples + +Configs and scripts for Kimi K3.1 async GRPO: a trainer island on Modal B300, +sampler islands on National Compute, and the sample buffer in an object store. + +Start with the end-to-end guide: +[docs/KIMI_K31_ASYNC_RL_GUIDE.md](../../docs/KIMI_K31_ASYNC_RL_GUIDE.md). +It covers the run config, the pool config, the sampler spec, launch, +scaling, resume, the dashboard, and cost protection. + +| File | Purpose | +|---|---| +| `launch-trainer-modal-b300.sh` | Trainer flags for the Modal B300 island | +| `sampler-spec.nc-mi355x-vllm.json` | Sampler spec for an NC MI355X node (vLLM) | +| `sampler-spec.nc-mi355x.json` | Sampler spec for an NC MI355X node | +| `sampler-node-nc-mi355x.sh` | Sampler node script | +| `algorithm-spec.async-age2.json`, `algorithm-spec.async-age2-tis.json` | Algorithm specs for `--rl-max-policy-age 2` | +| `sampler-spec.cpu-fake.json`, `convertible-spec.cpu-fake.json`, `launch-convertible-modal.sh` | CPU rehearsal and convertible-island examples | diff --git a/examples/k31/algorithm-spec.async-age2-tis.json b/examples/k31/algorithm-spec.async-age2-tis.json new file mode 100644 index 000000000..054c82d7b --- /dev/null +++ b/examples/k31/algorithm-spec.async-age2-tis.json @@ -0,0 +1 @@ +{"advantage":{"estimator":"grpo","reward_binary":false,"reward_postprocess":null,"rewards_normalization":true,"std_normalization":true,"whiten":false},"correction":{"batch_normalize":false,"function":null,"level":"token","lower":0.5,"method":"tis","mismatch_metrics":false,"opsm_delta":null,"ratio_mask":"none","upper":2.0,"use_rollout_logprobs":false,"veto":null},"entropy_coef":0.0,"execution":{"group_required":true,"max_policy_staleness":2,"needs_critic":false,"needs_rollout_logprobs":false},"kl":{"coef":null,"estimator":null,"placement":"none","unbiased":false},"loss":{"aggregation":"default","custom_loss":null,"eps_clip":null,"eps_clip_c":null,"eps_clip_high":null,"reducer":null,"variant":"policy_loss"},"plugins":[],"sampling":{"filter":null,"max_replacements":null,"over_sampling_batch_size":null,"overlong_filter":false},"schema":"yeto-rl-algorithm-spec-v2"} \ No newline at end of file diff --git a/examples/k31/algorithm-spec.async-age2.json b/examples/k31/algorithm-spec.async-age2.json new file mode 100644 index 000000000..52c2de0ae --- /dev/null +++ b/examples/k31/algorithm-spec.async-age2.json @@ -0,0 +1 @@ +{"advantage":{"estimator":"grpo","reward_binary":false,"reward_postprocess":null,"rewards_normalization":true,"std_normalization":true,"whiten":false},"correction":{"batch_normalize":false,"function":null,"level":"token","lower":null,"method":"none","mismatch_metrics":false,"opsm_delta":null,"ratio_mask":"none","upper":null,"use_rollout_logprobs":false,"veto":null},"entropy_coef":0.0,"execution":{"group_required":true,"max_policy_staleness":2,"needs_critic":false,"needs_rollout_logprobs":false},"kl":{"coef":null,"estimator":null,"placement":"none","unbiased":false},"loss":{"aggregation":"default","custom_loss":null,"eps_clip":null,"eps_clip_c":null,"eps_clip_high":null,"reducer":null,"variant":"policy_loss"},"plugins":[],"sampling":{"filter":null,"max_replacements":null,"over_sampling_batch_size":null,"overlong_filter":false},"schema":"yeto-rl-algorithm-spec-v2"} diff --git a/examples/k31/convertible-spec.cpu-fake.json b/examples/k31/convertible-spec.cpu-fake.json new file mode 100644 index 000000000..c5456b56e --- /dev/null +++ b/examples/k31/convertible-spec.cpu-fake.json @@ -0,0 +1,42 @@ +{ + "_comment": [ + "CPU rehearsal of a convertible Kimi-K3.1 island (tests/test_rl_k31_convertible.py): the", + "island runs as an engine sampler on the fake engine until the role_switch trigger fires", + "(groups written, seconds elapsed, the local marker of YETO_ROLE_SWITCH_MARKER, or the", + "store marker convertible//switch.json), then runs the trainer phase, then comes back", + "as a sampler on the released weights. Paths in angle brackets are local scratch", + "directories. Run: python -m yeto.rl.engine.role_switch --spec THIS.json" + ], + "sampler_id": "n-island-0", + "platform": "rocm", + "rollout_fields": {"routed_experts": true, "top_logprobs_num": 128, "fp32_lm_head": true}, + "engine": { + "factory": "yeto.rl.adapters.sampler_engines.from_spec:build", + "adapter": "fake", + "kwargs": {"routed_experts": [896, 3, 16], "routed_experts_pad_rows": 1, "expert_file_update": true, + "max_model_len": 4096} + }, + "sampling": {"max_tokens": 64, "temperature": 1.0, "top_p": 1.0, "top_k": -1, "n": 4}, + "num_rollouts": 64, + "max_backlog_groups": 64, + "max_prompt_tokens": 512, + "store": "/store", + "prompts": {"path": "/prompts.jsonl", "input_key": "prompt", "label_key": "answer", + "groups_per_round": 2, "format": "text"}, + "reward": "yeto.rl.numeric_answer_reward:score", + "reward_kind": "miles", + "checks": {"require_fixed_answers": false}, + "incarnation_lease": false, + "event_tape": "/events.jsonl", + "role_switch": { + "node": "n-island-0", + "after_groups": 4, + "after_seconds": 1800, + "marker": true, + "store_marker": true, + "trainer": {"run": "test_rl_k31_convertible:fake_trainer_phase"}, + "then_sampler": true, + "max_cycles": 1, + "device_idle": false + } +} diff --git a/examples/k31/launch-convertible-modal.sh b/examples/k31/launch-convertible-modal.sh new file mode 100644 index 000000000..0c0159f87 --- /dev/null +++ b/examples/k31/launch-convertible-modal.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Kimi-K3.1 convertible island on one Modal B300 node (draft; CPU-rehearsed wiring, not GPU-run). +# +# Production flow: the N-card node (this script) and the A-card NC MI355X nodes +# (examples/k31/sampler-node-nc-mi355x.sh) all start as engine samplers writing their own shard +# of the same object-store buffer. After SAMPLE_MINUTES (or SAMPLE_GROUPS groups, or the store +# marker convertible//switch.json written by an operator) this node stops its SGLang server +# and runs the Miles trainer island on the same GPUs: it reads every island's shard from the +# newest cut's watermark, trains (score_centering + GRPO advantage, --rl-max-policy-age gating the +# staleness) and releases whole checkpoints with the selection sha into the store; the A-card +# samplers keep generating and pull each release. With THEN_SAMPLER=1 the node comes back as a +# sampler on the newest release when the trainer phase ends. +# +# The container runs `python -m yeto.rl.engine.role_switch --spec SPEC`, where SPEC is the +# engine sampler spec of this node (sampler-spec.nc-mi355x.json shaped, platform cuda) plus a +# role_switch section whose trainer.command is the trainer island's learner command. The learner +# command is what `yeto launch --training-mode rl ... --rl-sample-role trainer --dry-run` prints +# for the flags below (launch-trainer-modal-b300.sh): copy it into the spec unchanged. +# Every value below is a placeholder; nothing site-specific is written here. +set -euo pipefail + +: "${MODEL:?HF repo or path of the Kimi-K3.1 checkpoint (stage 1: a 4-layer cut from scripts/prune_layers.py)}" +: "${MODEL_REVISION:?pinned revision}" +: "${DATA:?prompt dataset}" +: "${DATA_REVISION:?pinned data revision}" +: "${STORE_URI:?s3:///}" +: "${STORE_ENDPOINT:?https://.r2.cloudflarestorage.com}" +: "${STORE_SECRET:?Modal Secret holding AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY}" +: "${EXPERT_SELECTION:?path of expert_selection.json (sha256 in every release manifest)}" +: "${RL_IMAGE:?docker:REPO@sha256:DIGEST of the k31 Miles image}" +: "${SAMPLER_SPEC:?engine sampler spec of this node (sglang-server, platform cuda, buffer shard = NODE)}" +NODE=${NODE:-n-island-0} +SAMPLE_MINUTES=${SAMPLE_MINUTES:-30} # sampler phase budget; 0 = groups / marker only +SAMPLE_GROUPS=${SAMPLE_GROUPS:-0} # or: switch after this many groups written by this node +THEN_SAMPLER=${THEN_SAMPLER:-0} # 1: back to sampler on the released weights +GPU=${GPU:-modal:1x8xb300} +HERE=$(cd "$(dirname "$0")" && pwd) +OUT=${OUT:-/tmp/k31-convertible} +mkdir -p "$OUT" + +# 1. The trainer island's learner command (dry run of the trainer launch; unchanged flags of +# launch-trainer-modal-b300.sh). The printed command becomes role_switch.trainer.command. +yeto launch --training-mode rl --dry-run \ + --gpu "$GPU" --cluster-prefix k31-conv \ + --rl-image "$RL_IMAGE" \ + --model "$MODEL" --model-revision "$MODEL_REVISION" \ + --data "$DATA" --data-revision "$DATA_REVISION" \ + --rl-prompt-column prompt --rl-label-column answer \ + --reward-function yeto.rl.numeric_answer_reward:score \ + --trust-remote-code --tuning full --rl-model-recipe kimi_k31 --rl-expert-selection "$EXPERT_SELECTION" \ + --expert-parallel 8 --tensor-parallel 1 --pipeline-parallel 1 \ + --rl-sample-role trainer --rl-single-island-no-sync --controller local \ + --rl-checkpoint-store "$STORE_URI" --rl-object-store-endpoint "$STORE_ENDPOINT" \ + --modal-store-secret "$STORE_SECRET" \ + --inner-lr "${INNER_LR:-1e-6}" --rl-max-policy-age 2 --rl-algorithm-spec "$HERE/algorithm-spec.async-age2.json" \ + --rl-loss score_centering --rl-score-centering-is tis --rl-rollout-top-logprobs-num 128 \ + --rl-allow-unverified-mechanism losses:score_centering \ + --rl-allow-unverified-mechanism features:rollout_logprobs_as_old \ + --rl-allow-unverified-mechanism features:no_grpo_std_normalization \ + --rl-allow-unverified-mechanism loss_aggregations:token \ + --rollout-batch-size 32 --n-samples-per-prompt 8 --rollout-max-response-len 8192 \ + --seq-len "${SEQ_LEN:-8192}" --total-steps "${TOTAL_STEPS:-8}" \ + --rl-release-delta-full-every 8 \ + "$@" > "$OUT/trainer-dry-run.txt" + +# 2. The convertible spec: the sampler spec plus the role_switch section. The trainer command is +# the learner line of the dry run (the line that runs yeto.rl.adapters.miles.island_entry). +python - "$SAMPLER_SPEC" "$OUT/trainer-dry-run.txt" "$OUT/convertible-spec.json" <<'PY' +import json, shlex, sys +spec = json.load(open(sys.argv[1])) +learner = next(l for l in open(sys.argv[2]) if "yeto.rl.adapters.miles.island_entry" in l) +spec["role_switch"] = { + "node": __import__("os").environ.get("NODE", "n-island-0"), + "after_seconds": (int(__import__("os").environ.get("SAMPLE_MINUTES", "30")) * 60) or None, + "after_groups": int(__import__("os").environ.get("SAMPLE_GROUPS", "0")) or None, + "marker": True, "store_marker": True, + "trainer": {"command": shlex.split(learner.strip())}, + "then_sampler": __import__("os").environ.get("THEN_SAMPLER", "0") == "1", + "max_cycles": 1, + "device_idle": {"used_mib": 1024, "timeout_s": 300}, +} +json.dump(spec, open(sys.argv[3], "w"), indent=2) +PY + +# 3. Run the island in the k31 image on one node: sampler (SGLang server on the 8 GPUs, buffer shard +# NODE) -> trainer (the learner command above) -> optional sampler. The operator can force the +# switch earlier with `yeto.rl.engine.role_switch.write_switch_marker` on the store +# (convertible//switch.json) or by touching $YETO_ROLE_SWITCH_MARKER in the container. +echo "convertible spec: $OUT/convertible-spec.json" +echo "run in the island container: python -m yeto.rl.engine.role_switch --spec $OUT/convertible-spec.json" diff --git a/examples/k31/launch-trainer-modal-b300.sh b/examples/k31/launch-trainer-modal-b300.sh new file mode 100755 index 000000000..9fdcadbad --- /dev/null +++ b/examples/k31/launch-trainer-modal-b300.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Kimi-K3.1 async GRPO, trainer island on Modal B300 (draft; CPU-rehearsed wiring, not GPU-run). +# +# Full-parameter form: the selected routed experts of every MoE layer (64 of 896 is the +# production setting; 128 also works) train in place next to the frozen MXFP4 experts +# (Miles --partial-experts-selection). LoRA is for infrastructure tests only. +# +# Stage 1 (default): a 4-layer smoke checkpoint you cut yourself +# (scripts/prune_layers.py --layers 4 of the K3.1-Pro checkpoint) on one B300 node, TP1 EP8 (DP 8). +# Stage 2 (STAGE=2): the full 93-layer Kimi-K3.1-Pro on a B300 cluster, 6 nodes x 8 = 48 GPUs as +# PP6 x TP8 x EP8 (DP 1; the synced forms need DP 1 on the k31 pin). 93 layers do not split +# evenly over PP6: yeto derives the uneven split first 17 / last 16 (4 middle stages x 15, +# --decoder-first/last-pipeline-num-layers) and translate_run_config refuses any split that +# does not add up before a GPU starts. +# +# Forms (FORM): +# no-sync (default): one trainer island, --rl-single-island-no-sync; it releases its own +# policy. +# synced-1: one trainer island behind the Rust syncer (head + syncer; learners 1 / quorum 1, +# the merge is the identity): the decoupled DiLoCo path with a single learner. +# synced-2: two trainer islands behind the syncer: each reads its own buffer lane, pushes its +# full-parameter pseudo-gradient (the selected experts' parameters) at every +# asynchronous round, the syncer merges them and learner 0 releases the merged policy. +# Both synced forms keep --tuning full (the K3.1 recipe is full-parameter only): the +# full-parameter preset with asynchronous rounds and a parameter-layout sha; they take no +# --rl-allow-unverified-mechanism (a run with outer sync uses declared mechanisms only), so +# the algorithm spec declares the staleness correction itself (algorithm-spec.async-age2-tis.json). +# +# The samplers are National Compute MI355X nodes (examples/k31/sampler-spec.nc-mi355x.json): +# this island reads their groups from the object store and releases every policy into it +# with the selection's sha256 in the release manifest. Every value below is a placeholder; +# the expert_selection.json comes from the sampler's routed-expert statistics, or, for a first +# run before any statistics exist, a uniform selection: +# python -m yeto.rl.adapters.miles.models.partial_experts.selection --model kimi_k31 \ +# --num-experts 896 --num-trainable 64 --ep-size 8 --moe-layer-offset 1 \ +# --mode uniform --num-moe-layers 92 --output expert_selection.json +# (a 4-layer cut has 3 MoE layers: --num-moe-layers 3 --skip-layer-check) +# The dry run (append --dry-run) validates the selection against --expert-parallel and prints a +# rough per-GPU memory estimate (kimi_k31_full.memory), with warnings for long sequences. +# 262k context: SEQ_LEN=262144; the response length follows SEQ_LEN - PROMPT_LEN. +set -euo pipefail + +STAGE=${STAGE:-1} +FORM=${FORM:-no-sync} +: "${MODEL:?HF repo or path of the Kimi-K3.1 checkpoint (stage 1: the 4-layer cut)}" +: "${MODEL_REVISION:?pinned revision}" +: "${DATA:?prompt dataset}" +: "${DATA_REVISION=}" # pinned data revision (HF datasets); a trainer island never reads --data rows +: "${STORE_URI:?s3:///}" +: "${STORE_ENDPOINT:?https://.r2.cloudflarestorage.com}" +: "${STORE_SECRET:?Modal Secret holding AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY}" +: "${EXPERT_SELECTION:?path of expert_selection.json (sha256 in every release manifest)}" +: "${RL_IMAGE:?docker:REPO@sha256:DIGEST of the k31 Miles image}" +HERE=$(cd "$(dirname "$0")" && pwd) + +if [ "$STAGE" = "1" ]; then + ISLAND=${ISLAND:-modal:1x8xb300} # one node: 8 x B300, TP1 EP8 + TP=${TP:-1}; PP=${PP:-1} + SEQ_LEN=${SEQ_LEN:-8192} + TOTAL_STEPS=${TOTAL_STEPS:-8} +else + ISLAND=${ISLAND:-modal:6x8xb300} # whole-node cluster, RDMA (yeto.modal_runner clustered) + TP=${TP:-8}; PP=${PP:-6} # PP6 x TP8 x EP8 = 48 GPUs, DP 1 (split 17 / 4 x 15 / 16) + SEQ_LEN=${SEQ_LEN:-16384} + TOTAL_STEPS=${TOTAL_STEPS:-64} +fi +PROMPT_LEN=${PROMPT_LEN:-4096} # prompt budget inside SEQ_LEN +RESPONSE_LEN=${RESPONSE_LEN:-$((SEQ_LEN - PROMPT_LEN))} + +case "$FORM" in + no-sync) + GPU=${GPU:-$ISLAND} + SYNC_FLAGS=(--rl-single-island-no-sync) + SPEC=${SPEC:-$HERE/algorithm-spec.async-age2.json} + # score_centering/tis is an unverified mechanism: allowed on a single island without outer sync + LOSS_FLAGS=(--rl-loss score_centering --rl-score-centering-is tis + --rl-allow-unverified-mechanism losses:score_centering + --rl-allow-unverified-mechanism features:rollout_logprobs_as_old + --rl-allow-unverified-mechanism features:no_grpo_std_normalization + --rl-allow-unverified-mechanism loss_aggregations:token) + ;; + synced-1|synced-2) + # One or two trainer islands behind the Rust syncer (the launcher starts the head + syncer). + # The syncer merges the islands' full-parameter pseudo-gradients each asynchronous round; + # with one island it runs learners=1/quorum=1 and the merge is the identity. + if [ "$FORM" = "synced-2" ]; then GPU=${GPU:-$ISLAND,$ISLAND}; else GPU=${GPU:-$ISLAND}; fi + : "${PARAMETER_LAYOUT_SHA256:?sha256 of the full-parameter layout (--rl-parameter-layout-sha256)}" + SYNC_FLAGS=(--rl-sync-preset full-parameter --rl-sync-full-parameter-rounds async + --rl-parameter-layout-sha256 "$PARAMETER_LAYOUT_SHA256" + --rl-sync-quorum 1 --rl-sync-final-cut ack) + SPEC=${SPEC:-$HERE/algorithm-spec.async-age2-tis.json} + LOSS_FLAGS=() # declared mechanisms only: the spec carries the tis correction + ;; + *) echo "FORM must be no-sync, synced-1 or synced-2 (got $FORM)" >&2; exit 2 ;; +esac + +yeto launch --training-mode rl \ + --gpu "$GPU" --cluster-prefix k31-stage"$STAGE"-"$FORM" \ + --rl-image "$RL_IMAGE" \ + --model "$MODEL" --model-revision "$MODEL_REVISION" \ + --data "$DATA" ${DATA_REVISION:+--data-revision "$DATA_REVISION"} \ + --rl-prompt-column prompt --rl-label-column answer \ + --reward-function yeto.rl.numeric_answer_reward:score \ + --trust-remote-code --tuning full --rl-model-recipe kimi_k31 --rl-expert-selection "$EXPERT_SELECTION" \ + --expert-parallel 8 --tensor-parallel "$TP" --pipeline-parallel "$PP" --context-parallel "${CP:-1}" \ + --rl-sample-role trainer "${SYNC_FLAGS[@]}" --controller local \ + --rl-checkpoint-store "$STORE_URI" --rl-object-store-endpoint "$STORE_ENDPOINT" \ + --modal-store-secret "$STORE_SECRET" \ + --inner-lr "${INNER_LR:-1e-6}" --rl-max-policy-age 2 --rl-algorithm-spec "$SPEC" \ + --rl-rollout-top-logprobs-num 128 ${LOSS_FLAGS[@]+"${LOSS_FLAGS[@]}"} \ + --rollout-batch-size 32 --n-samples-per-prompt 8 --rollout-max-response-len "$RESPONSE_LEN" \ + --seq-len "$SEQ_LEN" --total-steps "$TOTAL_STEPS" \ + --rl-release-delta-full-every 8 \ + "$@" diff --git a/examples/k31/sampler-node-nc-mi355x.sh b/examples/k31/sampler-node-nc-mi355x.sh new file mode 100755 index 000000000..1f0d36d26 --- /dev/null +++ b/examples/k31/sampler-node-nc-mi355x.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Kimi-K3.1 sampler node on National Compute (8 x MI355X, SGLang ROCm): the node scripts of +# scripts/sampler_node/ in order, with the K3.1 server contract on (draft; not run on a node). +# Every value is a placeholder; the store credentials stay in a mode-0600 env file. +set -euo pipefail + +: "${NC_ACCOUNT:?nc account number}" +: "${IMAGE:?SGLang ROCm image (sglang-miles-kimi-k31 build)}" +: "${MODEL:?HF repo of the Kimi-K3.1 MXFP4 checkpoint}" +: "${MODEL_DIR:?nvme path the checkpoint lands in}" +: "${R2_ENV_FILE:?0600 env file: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ENDPOINT_URL AWS_REGION R2_BUCKET}" +: "${RELEASE_STORE:?local release store directory}" +: "${RUN:?run prefix in the bucket}" +: "${SPEC:?engine sampler spec (examples/k31/sampler-spec.nc-mi355x.json filled in)}" +NODES=$(cd "$(dirname "$0")/../../scripts/sampler_node" && pwd) + +"$NODES/nc_lease.sh" --account "$NC_ACCOUNT" --out node.json +"$NODES/bootstrap.sh" --image "$IMAGE" --model "$MODEL" --r2-env-file "$R2_ENV_FILE" \ + --release-store "$RELEASE_STORE" --release-prefix "$RUN" \ + ${RELEASE_PEERS:+--release-peers "$RELEASE_PEERS"} --serve-port "${SERVE_PORT:-8431}" +"$NODES/run_sglang.sh" --image "$IMAGE" --model-dir "$MODEL_DIR" --tp 8 --r2-env-file "$R2_ENV_FILE" \ + -- --context-length 32768 --trust-remote-code +"$NODES/run_engine_sampler.sh" --spec "$SPEC" --r2-env-file "$R2_ENV_FILE" --pip-install diff --git a/examples/k31/sampler-spec.cpu-fake.json b/examples/k31/sampler-spec.cpu-fake.json new file mode 100644 index 000000000..15620102f --- /dev/null +++ b/examples/k31/sampler-spec.cpu-fake.json @@ -0,0 +1,30 @@ +{ + "_comment": [ + "The CPU rehearsal of the Kimi-K3.1 sampler node (tests/test_rl_k31_e2e_cpu.py): the fake", + "engine returns the K3-4layer shape of the K3.1 rollout fields (R3 routed experts", + "uint16[T, 3, 16] below 896 with 0xFFFF pads, 128 top-K log-probs, fp32 lm-head) and takes", + "a release's changed expert files through the k31_release plan. Paths in angle brackets", + "are local scratch directories. Run: python -m yeto.rl.engine.engine_sampler --spec THIS.json" + ], + "sampler_id": "cpu-fake-0", + "platform": "rocm", + "rollout_fields": {"routed_experts": true, "top_logprobs_num": 128, "fp32_lm_head": true}, + "engine": { + "factory": "yeto.rl.adapters.sampler_engines.from_spec:build", + "adapter": "fake", + "kwargs": {"routed_experts": [896, 3, 16], "routed_experts_pad_rows": 1, "expert_file_update": true, + "max_model_len": 4096} + }, + "sampling": {"max_tokens": 64, "temperature": 1.0, "top_p": 1.0, "top_k": -1, "n": 4}, + "num_rollouts": 8, + "max_backlog_groups": 64, + "max_prompt_tokens": 512, + "store": "/store", + "prompts": {"path": "/prompts.jsonl", "input_key": "prompt", "label_key": "answer", + "groups_per_round": 4, "format": "text"}, + "reward": "yeto.rl.numeric_answer_reward:score", + "reward_kind": "miles", + "checks": {"require_fixed_answers": false}, + "incarnation_lease": false, + "event_tape": "/events.jsonl" +} diff --git a/examples/k31/sampler-spec.nc-mi355x-vllm.json b/examples/k31/sampler-spec.nc-mi355x-vllm.json new file mode 100644 index 000000000..5b6640b28 --- /dev/null +++ b/examples/k31/sampler-spec.nc-mi355x-vllm.json @@ -0,0 +1,92 @@ +{ + "_comment": [ + "Engine sampler spec of one National Compute MI355X node (8 x MI355X, vLLM ROCm) for a", + "Kimi-K3.1 async GRPO run, in the dedicated image pins.VLLM_SAMPLER_IMAGE_ROCM", + "(vllm/vllm-openai-rocm:kimi-k31): python -m yeto.rl.engine.engine_sampler --spec THIS.json", + "(scripts/sampler_node/run_engine_sampler.sh --python /opt/runtime-venv/bin/python --pip-install", + "--spec THIS.json --r2-env-file ENV). The engine section is the vllm-server adapter on the", + "upstream line with the k31_preset amd-article (AMD's published TP8 + DCP8 layout without", + "speculative decoding and CPU KV offload); rl-precision is the same layout with BF16", + "all-reduce / MoE combine / KDA input projections and an FP32 KDA state. vLLM refuses R3", + "(routed_experts) with DCP / PCP > 1, so this R3 server runs the preset at DCP 1 (the MLA KV", + "cache whole on every rank, about 1/8 of the DCP8 KV token capacity; the load detail records", + "it as r3_cp_overrides): keep max_num_seqs x max_model_len inside it. Reload is by restart", + "(in-place reload_weights is not measured on this image). Every value in angle brackets is a", + "site value the launch layer fills in; the store and prompts sections are as in", + "sampler-spec.nc-mi355x.json." + ], + "sampler_id": "", + "platform": "rocm", + "rollout_fields": { + "routed_experts": true, + "top_logprobs_num": 128, + "fp32_lm_head": true + }, + "expert_selection_sha256": "", + "engine": { + "factory": "yeto.rl.adapters.sampler_engines.from_spec:build", + "adapter": "vllm-server", + "server": { + "base_dir": "", + "port": 30000, + "python": "/opt/runtime-venv/bin/python", + "card_model": "MI355X", + "device_family": "rocm", + "engine_version": "vllm-kimi-k31@0.0.0+mum.14d1ec2336", + "precision": "mxfp4", + "quant_scheme": "MXFP4_EXPERTS_RTN", + "topology": "tp8", + "max_model_len": 32768, + "in_place_reload": false, + "start_timeout_s": 7200.0, + "extra_args": [] + }, + "options": { + "tp": 8, + "max_num_seqs": 48, + "k31_preset": "amd-article", + "tool_call_parser": "kimi_k3", + "reasoning_parser": "kimi_k3", + "pin_version": "0.0.0+mum.14d1ec2336" + } + }, + "sampling": { + "max_tokens": 8192, + "temperature": 1.0, + "top_p": 1.0, + "top_k": -1, + "n": 8 + }, + "num_rollouts": 1000000, + "max_backlog_groups": 256, + "max_prompt_tokens": 4096, + "base_version": 0, + "poll_s": 5.0, + "min_round_s": null, + "store": "//", + "release_dir": "policy-releases", + "buffer_shards": [ + "" + ], + "prompts": { + "path": "", + "input_key": "prompt", + "label_key": "answer", + "groups_per_round": 32, + "format": "chat", + "tokenizer": "", + "trust_remote_code": true + }, + "reward": "yeto.rl.numeric_answer_reward:score", + "reward_kind": "miles", + "checks": { + "expected_expert_matrices": null, + "allow_unverified_count": true, + "require_fixed_answers": false + }, + "incarnation_lease": true, + "peers": [ + ":8431" + ], + "event_tape": "/events.jsonl" +} diff --git a/examples/k31/sampler-spec.nc-mi355x.json b/examples/k31/sampler-spec.nc-mi355x.json new file mode 100644 index 000000000..05b689098 --- /dev/null +++ b/examples/k31/sampler-spec.nc-mi355x.json @@ -0,0 +1,69 @@ +{ + "_comment": [ + "Engine sampler spec of one National Compute MI355X node (8 x MI355X, SGLang ROCm) for a", + "Kimi-K3.1 async GRPO run: python -m yeto.rl.engine.engine_sampler --spec THIS.json", + "(scripts/sampler_node/run_engine_sampler.sh --spec THIS.json --r2-env-file ENV).", + "Every value in angle brackets is a site value the launch layer fills in; nothing here", + "is specific to one account or bucket. The store is reached by object key (no mount):", + "the environment carries YETO_RL_STORE_IMPL=yeto.cloud.s3_store:store_from_env,", + "YETO_RL_STORE_S3_URI=s3:///, YETO_RL_STORE_S3_STAGE=,", + "YETO_RL_STORE_S3_ENDPOINT_URL=, YETO_RL_STORE_S3_REGION=auto and the", + "store credentials (yeto.cloud.s3_store); `store` below is the stage path of that URI.", + "rollout_fields and platform describe what the trainer needs; expert_selection_sha256 is", + "the sha256 of the run's expert_selection.json (the trainer's release manifest must name", + "the same selection or the release is refused and the served weights kept)." + ], + "sampler_id": "", + "platform": "rocm", + "rollout_fields": {"routed_experts": true, "top_logprobs_num": 128, "fp32_lm_head": true}, + "expert_selection_sha256": "", + "engine": { + "factory": "yeto.rl.adapters.sampler_engines.from_spec:build", + "adapter": "sglang-server", + "server": { + "base_dir": "", + "port": 30000, + "card_model": "MI355X", + "device_family": "rocm", + "engine_version": "sglang-miles-kimi-k31@35708f76", + "precision": "mxfp4", + "quant_scheme": "MXFP4_EXPERTS_RTN", + "topology": "tp8", + "max_model_len": 32768, + "in_place_reload": true, + "extra_args": ["--trust-remote-code", "--context-length", "32768"] + }, + "options": { + "tp": 8, + "dp_attention_flag": "attn-dp-size", + "expert_file_update": "auto", + "expert_update_probe": true, + "live_update": false + } + }, + "sampling": {"max_tokens": 8192, "temperature": 1.0, "top_p": 1.0, "top_k": -1, "n": 8}, + "num_rollouts": 1000000, + "max_backlog_groups": 256, + "max_prompt_tokens": 4096, + "base_version": 0, + "poll_s": 5.0, + "min_round_s": null, + "store": "//", + "release_dir": "policy-releases", + "buffer_shards": [""], + "prompts": { + "path": "", + "input_key": "prompt", + "label_key": "answer", + "groups_per_round": 32, + "format": "chat", + "tokenizer": "", + "trust_remote_code": true + }, + "reward": "yeto.rl.numeric_answer_reward:score", + "reward_kind": "miles", + "checks": {"expected_expert_matrices": null, "allow_unverified_count": true, "require_fixed_answers": false}, + "incarnation_lease": true, + "peers": [":8431"], + "event_tape": "/events.jsonl" +} diff --git a/examples/rl_algorithms/algo-eff-miles-overlong-filter.json b/examples/rl_algorithms/algo-eff-miles-overlong-filter.json index 46eff7fff..47f9f76a6 100644 --- a/examples/rl_algorithms/algo-eff-miles-overlong-filter.json +++ b/examples/rl_algorithms/algo-eff-miles-overlong-filter.json @@ -45,7 +45,7 @@ "plugins": [ { "path": "yeto.rl.algos.sample_filters.apply_sample_filters", - "sha256": "aa56dacdc9ee2dc1c16a3377676c5dabbef1560454c7fcc3580a9869a4206dfb" + "sha256": "9e16624407058ed66d1f66ecc0e0d0646358b5ce332399f546e4a042e89565cc" } ], "sampling": { diff --git a/examples/rl_algorithms/algo-eff-miles-overlong-pen2048.json b/examples/rl_algorithms/algo-eff-miles-overlong-pen2048.json index 0e4649493..606b8dbc2 100644 --- a/examples/rl_algorithms/algo-eff-miles-overlong-pen2048.json +++ b/examples/rl_algorithms/algo-eff-miles-overlong-pen2048.json @@ -4,7 +4,7 @@ "reward_binary": false, "reward_postprocess": { "path": "yeto.rl.algos.reward_pipeline.post_process", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, "reward_shapers": [ { @@ -55,11 +55,11 @@ "plugins": [ { "path": "yeto.rl.algos.reward_pipeline.grpo_default", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, { "path": "yeto.rl.algos.seq_adv.gdpo", - "sha256": "876e9450a553aea2bbe7ba7eb23d268cd4b8a7dbbb03d24fb3cb0cab042944e5" + "sha256": "e6ae120404ee1b0f785fc724af65245097206507d98b06ea7bf97f7b7040ed8a" } ], "sampling": { diff --git a/examples/rl_algorithms/algo-eff-miles-overlong-pen3072.json b/examples/rl_algorithms/algo-eff-miles-overlong-pen3072.json index 44fc2b3ef..7907d59df 100644 --- a/examples/rl_algorithms/algo-eff-miles-overlong-pen3072.json +++ b/examples/rl_algorithms/algo-eff-miles-overlong-pen3072.json @@ -4,7 +4,7 @@ "reward_binary": false, "reward_postprocess": { "path": "yeto.rl.algos.reward_pipeline.post_process", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, "reward_shapers": [ { @@ -55,11 +55,11 @@ "plugins": [ { "path": "yeto.rl.algos.reward_pipeline.grpo_default", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, { "path": "yeto.rl.algos.seq_adv.gdpo", - "sha256": "876e9450a553aea2bbe7ba7eb23d268cd4b8a7dbbb03d24fb3cb0cab042944e5" + "sha256": "e6ae120404ee1b0f785fc724af65245097206507d98b06ea7bf97f7b7040ed8a" } ], "sampling": { diff --git a/examples/rl_algorithms/algo-eff-miles-overlong-pen4096.json b/examples/rl_algorithms/algo-eff-miles-overlong-pen4096.json index 2669e6c24..e58adacad 100644 --- a/examples/rl_algorithms/algo-eff-miles-overlong-pen4096.json +++ b/examples/rl_algorithms/algo-eff-miles-overlong-pen4096.json @@ -4,7 +4,7 @@ "reward_binary": false, "reward_postprocess": { "path": "yeto.rl.algos.reward_pipeline.post_process", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, "reward_shapers": [ { @@ -55,11 +55,11 @@ "plugins": [ { "path": "yeto.rl.algos.reward_pipeline.grpo_default", - "sha256": "b6d0c1069f130a3bffcee65d27ba2d76bf94a7b3f5c44ef22688994ce75db2fa" + "sha256": "274dd0aede4b55285224db7103b9c122a2b1762f365ee334a2c144ef595b05b1" }, { "path": "yeto.rl.algos.seq_adv.gdpo", - "sha256": "876e9450a553aea2bbe7ba7eb23d268cd4b8a7dbbb03d24fb3cb0cab042944e5" + "sha256": "e6ae120404ee1b0f785fc724af65245097206507d98b06ea7bf97f7b7040ed8a" } ], "sampling": { diff --git a/pyproject.toml b/pyproject.toml index 40654bf7c..82195ac60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,4 +74,5 @@ include = ["yeto*"] [tool.setuptools.package-data] "yeto.dashboard" = ["*.json", "static/*"] "yeto.cloud" = ["*.json"] +"yeto.launch.nc" = ["*.mjs"] "yeto.rl.adapters.miles" = ["*.json", "overlays/*.patch"] diff --git a/scripts/build_miles_ports_image.sh b/scripts/build_miles_ports_image.sh index 348db081d..117c8b64b 100755 --- a/scripts/build_miles_ports_image.sh +++ b/scripts/build_miles_ports_image.sh @@ -33,7 +33,9 @@ BASE_AMD64_DIGEST=sha256:4d69750720eb1a4b99976a3fed45ed018e1fc4fff3d240667fa17d2 # What the base image itself has installed (read from its layers): BASE_MILES_COMMIT=9e4260de047a704208535c0e90c531929879ab40 # /root/miles BASE_SGLANG_COMMIT=e5bba1f9c433da5352e521a8bb826bab960e7836 # /sgl-workspace/sglang (sglang-miles head at build) -BASE_SGLANG_VERSION=0.5.21.dev64+ge5bba1f +# Megatron-Bridge as this base installs it (pip, not editable); its package +# __init__ is replaced by docker/miles-ports/megatron_bridge_init.py. +MEGATRON_BRIDGE_COMMIT=8cd3466d14d2337c8492827b3712482c2b3e4866 pins=$(python3 - <<'PY' import re @@ -63,19 +65,33 @@ MS=${MILES_NEXT_COMMIT:0:7}; SS=${SGLANG_NEXT_COMMIT:0:7} TAG="$DEST_REPO:$MS-$SS" MTIME=$(git -C "$MILES_SRC" log -1 --format=%ct "$MILES_NEXT_COMMIT") -# The base's sglang editable metadata, pulled from the one base layer that -# wrote it (streamed; only the metadata files are extracted). +# The base's sglang editable metadata, pulled from the base layer that wrote it. +# The layer is found by scanning (make_layer.py find-layer orders the candidates +# by the build history, check-listing recognises the metadata in a listing), so +# no layer number or version string is written here. META="$OUT/base-sglang-meta" if [ ! -d "$META" ]; then - layer=$("$CRANE" manifest "$BASE_REPO@$BASE_AMD64_DIGEST" | python3 -c ' -import json, sys; print(json.load(sys.stdin)["layers"][104]["digest"])') + "$CRANE" manifest "$BASE_REPO@$BASE_AMD64_DIGEST" > "$OUT/base-manifest.json" + "$CRANE" config "$BASE_REPO@$BASE_AMD64_DIGEST" > "$OUT/base-config.json" + layer="" + for digest in $(python3 docker/miles-ports/make_layer.py find-layer \ + --manifest "$OUT/base-manifest.json" --config "$OUT/base-config.json" \ + | python3 -c 'import json, sys; print("\n".join(l["digest"] for l in json.load(sys.stdin)))'); do + if "$CRANE" blob "$BASE_REPO@$digest" | tar -tzf - 2>/dev/null \ + | python3 docker/miles-ports/make_layer.py check-listing; then + layer=$digest; break + fi + done + [ -n "$layer" ] || { echo "no base layer holds the sglang editable-install metadata" >&2; exit 1; } mkdir -p "$OUT/meta-x" "$CRANE" blob "$BASE_REPO@$layer" | tar -xzf - -C "$OUT/meta-x" \ - --wildcards "opt/sglang/lib/python3.12/site-packages/*sglang_0_5_21_dev64*finder.py" \ - "opt/sglang/lib/python3.12/site-packages/__editable__.sglang-$BASE_SGLANG_VERSION.pth" \ - "opt/sglang/lib/python3.12/site-packages/sglang-$BASE_SGLANG_VERSION.dist-info/*" + --wildcards "opt/sglang/lib/python3.12/site-packages/__editable___sglang_*_finder.py" \ + "opt/sglang/lib/python3.12/site-packages/__editable__.sglang-*.pth" \ + "opt/sglang/lib/python3.12/site-packages/sglang-*.dist-info/*" mv "$OUT/meta-x/opt/sglang/lib/python3.12/site-packages" "$META" fi +BASE_SGLANG_VERSION=$(ls "$META" | sed -n 's/^sglang-\(.*\)\.dist-info$/\1/p' | head -1) +[ -n "$BASE_SGLANG_VERSION" ] || { echo "no sglang-*.dist-info in $META" >&2; exit 1; } # Megatron-LM files patched in the layer (docker/miles-ports/ # megatron_lm_lazy_modelopt.patch) start from the base's checkout commit. @@ -120,7 +136,7 @@ json.dump({ "wheel_sha256": "$PEFT_WHEEL_SHA", "install": "wheel unpacked into the base's site-packages (pip install --no-deps equivalent)"}, "pure_python_overlay": True, - "megatron_bridge": {"commit": "8cd3466d14d2337c8492827b3712482c2b3e4866", + "megatron_bridge": {"commit": "$MEGATRON_BRIDGE_COMMIT", "patches": ["lazy-modelopt (docker/miles-ports/megatron_bridge_init.py)"]}, "megatron_lm": {"repository": "$MEGATRON_LM_REPOSITORY", "commit": "$MEGATRON_LM_COMMIT", "path": "/root/Megatron-LM", "patches": ["lazy-modelopt (docker/miles-ports/megatron_lm_lazy_modelopt.patch)"]}, @@ -134,9 +150,10 @@ python3 docker/miles-ports/make_layer.py \ --miles-base-commit "$BASE_MILES_COMMIT" --miles-origin "$MILES_NEXT_REPOSITORY" \ --sglang-src "$SGLANG_SRC" --sglang-commit "$SGLANG_NEXT_COMMIT" \ --sglang-base-commit "$BASE_SGLANG_COMMIT" --sglang-origin "$SGLANG_NEXT_REPOSITORY" \ - --sglang-base-version "$BASE_SGLANG_VERSION" --sglang-version "$SGLANG_VERSION" \ + --sglang-version "$SGLANG_VERSION" \ --sglang-base-metadata "$META" --wheel "$PEFT_WHEEL" \ - --megatron-lm-src "$MEGATRON_LM_SRC" \ + --megatron-lm-src "$MEGATRON_LM_SRC" --megatron-lm-commit "$MEGATRON_LM_COMMIT" \ + --megatron-bridge-commit "$MEGATRON_BRIDGE_COMMIT" \ --mtime "$MTIME" --out "$OUT/layer.tar" LAYER_SHA=$(sha256sum "$OUT/layer.tar" | cut -d' ' -f1) echo "layer.tar sha256=$LAYER_SHA" diff --git a/scripts/prune_layers.py b/scripts/prune_layers.py new file mode 100755 index 000000000..e4e90e646 --- /dev/null +++ b/scripts/prune_layers.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +"""Cut the first N decoder layers out of a Hugging Face safetensors checkpoint. + + scripts/prune_layers.py SRC DEST --layers 4 [--keep-layer-tables] [--drop-vision] + [--revision REV] [--dry-run] + +``SRC`` is a local checkpoint directory or ``hf://ORG/NAME``; with ``hf://`` only +the shards that hold a kept tensor are downloaded (plus the small files), never +the whole checkpoint. ``DEST`` receives: + +* the shards that hold kept tensors. A shard whose tensors are all kept is + copied (or hard-linked) as it is; one that mixes kept and dropped tensors is + rewritten with the kept tensors only. Tensors are moved as bytes (the + safetensors header and byte ranges), so any dtype -- bf16, the MXFP4 packed + uint8 experts and their scales -- goes through unchanged, on CPU, without torch; +* ``model.safetensors.index.json`` over the kept tensors, with ``total_size``; +* ``config.json`` with ``num_hidden_layers`` = N and the layer-type tables cut + to the kept layers (Kimi K3.1: ``linear_attn_config.full_attn_layers`` / + ``kda_layers`` and ``dsa_config.dsa_layers``, 1-based entries; the general + ``layer_types`` list by position), ``num_nextn_predict_layers`` = 0 and no + MTP / draft tensors (``--keep-layer-tables`` leaves the tables as they are, + as some published cut-down checkpoints do); +* every other file of the checkpoint (tokenizer, processors, modeling code, + generation config) copied as it is. + +Kept: layers ``0..N-1`` with all their experts, the embeddings, the final norm +and head, the vision tower and projector (``--drop-vision`` removes those two). +Dropped: layers ``>= N`` and any next-token-prediction (``draft`` / ``mtp`` / +``nextn``) tensors. + +Kimi-K3.1-Pro, ``--layers 4``: layer 0 is the dense-MLP layer, layers 1-2 KDA +and layer 3 the first MLA/DSA layer, all three MoE layers with their 896 +experts; 7 of the 96 shards are needed (about 58 GB), as in the upstream +4-layer reference. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import struct +import sys +from pathlib import Path +from typing import Any, Iterable + +INDEX_FILE = "model.safetensors.index.json" +SINGLE_FILE = "model.safetensors" +LAYER_RE = re.compile(r"(?:^|\.)layers\.(\d+)\.") +# next-token-prediction / draft heads: never kept (num_nextn_predict_layers -> 0) +DRAFT_RE = re.compile(r"(?:^|\.)(draft|mtp|nextn|eh_proj|enorm|hnorm|shared_head)(?:$|\.)") +VISION_PREFIXES = ("vision_tower.", "mm_projector.", "vision_model.", "visual.", "multi_modal_projector.") +# config keys that list layer indices, 1-based (Kimi K3 / K3.1) +ONE_BASED_LAYER_LISTS = {"linear_attn_config": ("full_attn_layers", "kda_layers"), + "dsa_config": ("dsa_layers",)} +# config keys that list layer indices, 0-based (other families) +ZERO_BASED_LAYER_LISTS = ("moe_layers", "dense_layers", "sliding_window_layers", "full_attention_layers") +HEADER_LEN = 8 + + +class PruneError(RuntimeError): + pass + + +# --- tensor selection ---------------------------------------------------------- + +def layer_of(name: str) -> int | None: + """The decoder layer index of a tensor name, None for a non-layer tensor. + Vision encoder blocks say ``blocks.N``, not ``layers.N``, so they stay None.""" + match = LAYER_RE.search(name) + return int(match.group(1)) if match else None + + +def keep_tensor(name: str, layers: int, *, drop_vision: bool = False) -> bool: + if DRAFT_RE.search(name): + return False + if drop_vision and name.startswith(VISION_PREFIXES): + return False + index = layer_of(name) + return index is None or index < layers + + +# --- safetensors by bytes ----------------------------------------------------- + +def read_header(path: Path) -> tuple[dict[str, Any], int]: + """The safetensors header (tensor name -> dtype/shape/data_offsets, plus + ``__metadata__``) and the offset of the data section.""" + with open(path, "rb") as fh: + raw = fh.read(HEADER_LEN) + if len(raw) != HEADER_LEN: + raise PruneError(f"{path}: not a safetensors file") + (length,) = struct.unpack(" int: + """Write the tensors ``names`` of ``src`` into a new safetensors file ``dest``, + bytes copied range by range; returns the data bytes written.""" + header, data_start = read_header(src) + meta = header.get("__metadata__") + kept = [n for n in names if n in header] + missing = sorted(set(names) - set(kept)) + if missing: + raise PruneError(f"{src}: tensors not in the file: {missing[:5]}") + kept.sort(key=lambda n: header[n]["data_offsets"][0]) + new_header: dict[str, Any] = {} + if meta is not None: + new_header["__metadata__"] = meta + offset = 0 + for name in kept: + begin, end = header[name]["data_offsets"] + new_header[name] = {**header[name], "data_offsets": [offset, offset + (end - begin)]} + offset += end - begin + encoded = json.dumps(new_header, separators=(",", ":")).encode() + encoded += b" " * (-len(encoded) % 8) # safetensors pads the header to 8 bytes + tmp = dest.with_name(dest.name + ".tmp") + with open(src, "rb") as fin, open(tmp, "wb") as fout: + fout.write(struct.pack(" int: + begin, end = header[name]["data_offsets"] + return end - begin + + +# --- config ---------------------------------------------------------------------- + +def prune_config(config: dict[str, Any], layers: int, *, keep_tables: bool = False, + drop_vision: bool = False) -> dict[str, Any]: + """The config of the N-layer model: ``num_hidden_layers`` on the text config + (nested ``text_config`` or top level), the layer tables cut to the kept layers + unless ``keep_tables``, no next-token-prediction layers.""" + out = json.loads(json.dumps(config)) # deep copy + text = out.get("text_config") if isinstance(out.get("text_config"), dict) else out + if "num_hidden_layers" not in text: + raise PruneError("config has no num_hidden_layers (looked in text_config and at the top level)") + original = int(text["num_hidden_layers"]) + if not 1 <= layers <= original: + raise PruneError(f"--layers {layers} must be in 1..{original} (the checkpoint's num_hidden_layers)") + text["num_hidden_layers"] = layers + if "num_nextn_predict_layers" in text: + text["num_nextn_predict_layers"] = 0 + if not keep_tables: + for section, keys in ONE_BASED_LAYER_LISTS.items(): + block = text.get(section) + if isinstance(block, dict): + for key in keys: + if isinstance(block.get(key), list): + block[key] = [i for i in block[key] if 1 <= int(i) <= layers] + for key in ZERO_BASED_LAYER_LISTS: + if isinstance(text.get(key), list): + text[key] = [i for i in text[key] if 0 <= int(i) < layers] + if isinstance(text.get("layer_types"), list): + text["layer_types"] = text["layer_types"][:layers] + if isinstance(text.get("first_k_dense_replace"), int): + text["first_k_dense_replace"] = min(int(text["first_k_dense_replace"]), layers) + if drop_vision: + out.pop("vision_config", None) + return out + + +def kept_layer_kinds(config: dict[str, Any], layers: int) -> dict[int, str]: + """For the report: ``dense`` / ``kda`` / ``mla`` per kept layer of a Kimi K3 + config (empty for other families).""" + text = config.get("text_config") if isinstance(config.get("text_config"), dict) else config + linear = text.get("linear_attn_config") if isinstance(text.get("linear_attn_config"), dict) else {} + full = {int(i) for i in linear.get("full_attn_layers", [])} + dense_k = int(text.get("first_k_dense_replace", 0) or 0) + kinds = {} + if full: + for i in range(layers): + kinds[i] = "dense-mlp " + ("mla" if i + 1 in full else "kda") if i < dense_k else ("mla" if i + 1 in full else "kda") + return kinds + + +# --- sources ------------------------------------------------------------------- + +class LocalSource: + def __init__(self, root: Path) -> None: + self.root = root + + def file(self, name: str) -> Path: + path = self.root / name + if not path.exists(): + raise PruneError(f"{path} does not exist") + return path + + def exists(self, name: str) -> bool: + return (self.root / name).exists() + + def list(self) -> list[str]: + return sorted(str(p.relative_to(self.root)) for p in self.root.rglob("*") if p.is_file()) + + +class HubSource: + """``hf://ORG/NAME``: files fetched one by one (``huggingface_hub.hf_hub_download``).""" + + def __init__(self, repo: str, revision: str | None = None, cache_dir: str | None = None) -> None: + self.repo, self.revision, self.cache_dir = repo, revision, cache_dir + self._names: list[str] | None = None + + def list(self) -> list[str]: + if self._names is None: + from huggingface_hub import list_repo_files # noqa: PLC0415 + + self._names = sorted(list_repo_files(self.repo, revision=self.revision)) + return self._names + + def exists(self, name: str) -> bool: + return name in self.list() + + def file(self, name: str) -> Path: + from huggingface_hub import hf_hub_download # noqa: PLC0415 + + return Path(hf_hub_download(self.repo, name, revision=self.revision, cache_dir=self.cache_dir)) + + +def open_source(spec: str, *, revision: str | None = None, cache_dir: str | None = None): + if spec.startswith("hf://"): + return HubSource(spec[len("hf://"):].strip("/"), revision=revision, cache_dir=cache_dir) + root = Path(spec).expanduser() + if not root.is_dir(): + raise PruneError(f"{spec}: not a directory (use hf://ORG/NAME for a Hub checkpoint)") + return LocalSource(root) + + +# --- the cut ------------------------------------------------------------------- + +def weight_map_of(source) -> tuple[dict[str, str], dict[str, Any]]: + """``{tensor: shard file}`` of the checkpoint (an index, or the single file's header).""" + if source.exists(INDEX_FILE): + index = json.loads(source.file(INDEX_FILE).read_text(encoding="utf-8")) + return dict(index["weight_map"]), dict(index.get("metadata") or {}) + if source.exists(SINGLE_FILE): + header, _ = read_header(source.file(SINGLE_FILE)) + return {n: SINGLE_FILE for n in header if n != "__metadata__"}, {} + raise PruneError(f"no {INDEX_FILE} or {SINGLE_FILE} in the checkpoint") + + +def plan(weight_map: dict[str, str], layers: int, *, drop_vision: bool = False) -> dict[str, Any]: + """Which tensors stay, grouped by shard, and which shards are copied whole.""" + kept = {n for n in weight_map if keep_tensor(n, layers, drop_vision=drop_vision)} + by_shard: dict[str, list[str]] = {} + for name, shard in weight_map.items(): + by_shard.setdefault(shard, []).append(name) + shards_kept = {s: sorted(n for n in names if n in kept) for s, names in by_shard.items()} + shards_kept = {s: names for s, names in shards_kept.items() if names} + whole = sorted(s for s, names in shards_kept.items() if len(names) == len(by_shard[s])) + partial = sorted(s for s in shards_kept if s not in whole) + return {"kept": kept, "shards": shards_kept, "whole": whole, "partial": partial, + "dropped": len(weight_map) - len(kept), + "kept_layers": sorted({layer_of(n) for n in kept if layer_of(n) is not None})} + + +def copy_or_link(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + dest.unlink() + try: + os.link(src, dest) + except OSError: + shutil.copyfile(src, dest) + + +def prune(source, dest: Path, layers: int, *, keep_tables: bool = False, drop_vision: bool = False, + dry_run: bool = False, log=print) -> dict[str, Any]: + weight_map, _old_meta = weight_map_of(source) + config = json.loads(source.file("config.json").read_text(encoding="utf-8")) + new_config = prune_config(config, layers, keep_tables=keep_tables, drop_vision=drop_vision) + p = plan(weight_map, layers, drop_vision=drop_vision) + if p["kept_layers"] != list(range(layers)): + raise PruneError(f"the checkpoint has no tensors for layers {sorted(set(range(layers)) - set(p['kept_layers']))}") + kinds = kept_layer_kinds(config, layers) + log(f"[prune] {len(p['kept'])} tensors kept, {p['dropped']} dropped; shards: {len(p['whole'])} whole, " + f"{len(p['partial'])} rewritten, of {len(set(weight_map.values()))}") + for i in range(layers): + log(f"[prune] layer {i}: {kinds.get(i, '?')}") + if dry_run: + return {"kept": len(p["kept"]), "dropped": p["dropped"], "shards": sorted(p["shards"]), + "config": new_config, "layer_kinds": kinds} + dest.mkdir(parents=True, exist_ok=True) + new_map: dict[str, str] = {} + total = 0 + for shard in sorted(p["shards"]): + src = source.file(shard) + header, _ = read_header(src) + names = p["shards"][shard] + if shard in p["whole"]: + copy_or_link(src, dest / shard) + log(f"[prune] {shard}: kept whole ({len(names)} tensors)") + else: + write_subset(src, dest / shard, names) + log(f"[prune] {shard}: rewritten with {len(names)} of {len([n for n in header if n != '__metadata__'])} tensors") + for name in names: + new_map[name] = shard + total += tensor_bytes(header, name) + (dest / INDEX_FILE).write_text(json.dumps({"metadata": {"total_size": total}, + "weight_map": dict(sorted(new_map.items()))}, indent=2) + "\n", + encoding="utf-8") + (dest / "config.json").write_text(json.dumps(new_config, indent=2) + "\n", encoding="utf-8") + for name in source.list(): + if name in (INDEX_FILE, "config.json") or name.endswith(".safetensors") or name.startswith("."): + continue + if "/" in name and not name.startswith(("assets/",)): + continue # nested directories other than assets are not part of the checkpoint proper + copy_or_link(source.file(name), dest / name) + (dest / "PRUNE.json").write_text(json.dumps({ + "layers": layers, "kept_tensors": len(p["kept"]), "dropped_tensors": p["dropped"], + "shards_whole": p["whole"], "shards_rewritten": p["partial"], "total_size": total, + "layer_kinds": {str(k): v for k, v in kinds.items()}, "keep_layer_tables": keep_tables, + "drop_vision": drop_vision, + "source": getattr(source, "repo", None) or str(getattr(source, "root", "")), + }, indent=2) + "\n", encoding="utf-8") + log(f"[prune] wrote {dest} ({total / 1e9:.1f} GB of tensors)") + return {"kept": len(p["kept"]), "dropped": p["dropped"], "shards": sorted(p["shards"]), + "total_size": total, "config": new_config, "layer_kinds": kinds} + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0], + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__) + ap.add_argument("src", help="checkpoint directory, or hf://ORG/NAME") + ap.add_argument("dest", type=Path) + ap.add_argument("--layers", type=int, required=True, help="keep decoder layers 0..N-1") + ap.add_argument("--keep-layer-tables", action="store_true", + help="leave the layer-type tables as they are") + ap.add_argument("--drop-vision", action="store_true", help="drop the vision tower and projector") + ap.add_argument("--revision", default=None, help="hf://: the revision to read") + ap.add_argument("--cache-dir", default=None, help="hf://: huggingface_hub cache directory") + ap.add_argument("--dry-run", action="store_true", help="report the plan and the new config, write nothing") + args = ap.parse_args(argv) + try: + source = open_source(args.src, revision=args.revision, cache_dir=args.cache_dir) + result = prune(source, args.dest, args.layers, keep_tables=args.keep_layer_tables, + drop_vision=args.drop_vision, dry_run=args.dry_run) + except PruneError as error: + print(f"prune_layers: {error}", file=sys.stderr) + return 1 + if args.dry_run: + print(json.dumps({"kept": result["kept"], "dropped": result["dropped"], "shards": result["shards"], + "layer_kinds": {str(k): v for k, v in result["layer_kinds"].items()}}, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/sampler_node/bootstrap.sh b/scripts/sampler_node/bootstrap.sh new file mode 100755 index 000000000..687726754 --- /dev/null +++ b/scripts/sampler_node/bootstrap.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# First minutes on a fresh ROCm sampler node (runs on the node as the login +# user): host facts, docker, the NVMe directories, the SGLang image pull and +# the base model in the background. +# +# scripts/sampler_node/bootstrap.sh --image IMAGE --model MODEL [--model-dir DIR] +# [--nvme-root /mnt/nvme] [--peer user@host:/path/to/model] [--hf-token-file FILE] +# [--r2-env-file FILE] [--no-pull] +# [--release-store DIR --release-prefix RUN [--release-peers host:port,...] [--serve-port P] +# [--yeto-dir DIR] [--python PY]] +# +# MODEL an HF repo id (downloaded with `hf download`), or an HF bucket +# path `hf://buckets//` (synced with `hf buckets sync`) +# --peer copy the model from another node that already has it (rsync +# over ssh) instead of from the Hub; the peer's ssh key must be +# set up already +# --r2-env-file a file with the object store credentials (AWS_ACCESS_KEY_ID, +# AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT_URL, AWS_REGION, R2_BUCKET); +# only its permissions are fixed here (0600), nothing is read +# +# --release-store start the policy release puller (`python -m yeto.rl.spot.s3_delivery +# pull-loop`) in the background: releases come from the peers of +# --release-peers first (near-region fan-out, yeto.rl.spot.peer_fanout) and +# from the object store only when no peer has them; --serve-port serves this +# node's placed releases to the other nodes. The store settings come from +# --r2-env-file (S3_BUCKET / R2_BUCKET, AWS_ENDPOINT_URL, AWS_REGION) and +# --release-prefix (the run's prefix in the bucket); the file is read only for +# this (0600 enforced) and nothing from it is printed. +# +# Background jobs log to $NVME_ROOT/logs/{pull,model,releases}.log; `tail -f` them, +# while they run. Idempotent: re-running resumes. +# amd-smi (ROCm 6.1+) is required on the node: the launcher's card preflight runs +# `amd-smi static --json` (rocm-smi is the fallback) and bootstrap refuses a node +# without either tool. +set -euo pipefail +IMAGE=""; MODEL=""; MODEL_DIR=""; NVME_ROOT=/mnt/nvme; PEER=""; HF_TOKEN_FILE=""; R2_ENV_FILE=""; PULL=1 +HERE=$(cd "$(dirname "$0")" && pwd) +RELEASE_STORE=""; RELEASE_PREFIX=""; RELEASE_PEERS=""; SERVE_PORT=""; YETO_DIR=$(cd "$HERE/../.." && pwd); PY="" +while [ $# -gt 0 ]; do + case "$1" in + --image) IMAGE=$2; shift 2;; + --model) MODEL=$2; shift 2;; + --model-dir) MODEL_DIR=$2; shift 2;; + --nvme-root) NVME_ROOT=$2; shift 2;; + --peer) PEER=$2; shift 2;; + --hf-token-file) HF_TOKEN_FILE=$2; shift 2;; + --r2-env-file) R2_ENV_FILE=$2; shift 2;; + --no-pull) PULL=0; shift;; + --release-store) RELEASE_STORE=$2; shift 2;; + --release-prefix) RELEASE_PREFIX=$2; shift 2;; + --release-peers) RELEASE_PEERS=$2; shift 2;; + --serve-port) SERVE_PORT=$2; shift 2;; + --yeto-dir) YETO_DIR=$2; shift 2;; + --python) PY=$2; shift 2;; + -h|--help) sed -n '2,36p' "$0"; exit 0;; + *) echo "unknown option $1" >&2; exit 64;; + esac +done +[ -n "$IMAGE" ] && [ -n "$MODEL" ] || { echo "usage: $0 --image IMAGE --model MODEL [...]" >&2; exit 64; } +model_name=$(basename "${MODEL%/}") +MODEL_DIR=${MODEL_DIR:-$NVME_ROOT/models/$model_name} +SUDO=""; [ "$(id -u)" = 0 ] || SUDO=sudo + +echo "== host" +hostname; uname -r; nproc; free -g | head -2 +cat /opt/rocm/.info/version 2>/dev/null || echo "no /opt/rocm/.info/version" +if command -v amd-smi >/dev/null; then + echo "amd-smi cards: $(amd-smi static --json 2>/dev/null | grep -c market_name || true)" +elif command -v rocm-smi >/dev/null; then + echo "warning: no amd-smi (the launcher preflight falls back to rocm-smi)" >&2 + rocm-smi --showproductname 2>/dev/null | grep -i "card series" || true +else + echo "neither amd-smi nor rocm-smi on PATH: the launcher cannot check the cards on this node" >&2 + exit 69 +fi +command -v docker >/dev/null || { echo "docker is not installed" >&2; exit 69; } +df -h "$NVME_ROOT" 2>/dev/null | tail -1 || echo "no $NVME_ROOT mount" + +echo "== directories" +$SUDO mkdir -p "$NVME_ROOT/models" "$NVME_ROOT/cache" "$NVME_ROOT/logs" "$NVME_ROOT/yeto" +$SUDO chown -R "$(id -u):$(id -g)" "$NVME_ROOT/models" "$NVME_ROOT/cache" "$NVME_ROOT/logs" "$NVME_ROOT/yeto" +if [ -n "$R2_ENV_FILE" ]; then + [ -f "$R2_ENV_FILE" ] || { echo "$R2_ENV_FILE does not exist" >&2; exit 66; } + chmod 600 "$R2_ENV_FILE" + echo "store env file $R2_ENV_FILE: mode $(stat -c %a "$R2_ENV_FILE" 2>/dev/null || stat -f %Lp "$R2_ENV_FILE")" +fi + +if [ "$PULL" = 1 ]; then + echo "== image pull (background): $IMAGE" + nohup $SUDO docker pull "$IMAGE" > "$NVME_ROOT/logs/pull.log" 2>&1 & +fi + +echo "== model: $MODEL -> $MODEL_DIR (background)" +mkdir -p "$MODEL_DIR" +if [ -n "$PEER" ]; then + nohup rsync -a --partial --info=progress2 -e "ssh -o StrictHostKeyChecking=accept-new" \ + "$PEER/" "$MODEL_DIR/" > "$NVME_ROOT/logs/model.log" 2>&1 & +else + if [ ! -x "$HOME/hfenv/bin/hf" ]; then + python3 -m venv "$HOME/hfenv" 2>/dev/null \ + || { $SUDO apt-get update -qq && $SUDO apt-get install -y -qq python3-venv && python3 -m venv "$HOME/hfenv"; } + "$HOME/hfenv/bin/pip" install -q 'huggingface_hub[hf_xet]>=1.0' 2>&1 | tail -1 + fi + "$HOME/hfenv/bin/hf" version 2>&1 | head -1 + token_env="" + if [ -n "$HF_TOKEN_FILE" ]; then + [ -f "$HF_TOKEN_FILE" ] || { echo "$HF_TOKEN_FILE does not exist" >&2; exit 66; } + chmod 600 "$HF_TOKEN_FILE" + token_env="HF_TOKEN=$(tr -d '\n' < "$HF_TOKEN_FILE")" + fi + case "$MODEL" in + hf://buckets/*) + nohup env HF_XET_HIGH_PERFORMANCE=1 $token_env "$HOME/hfenv/bin/hf" buckets sync "$MODEL" "$MODEL_DIR" \ + > "$NVME_ROOT/logs/model.log" 2>&1 &;; + *) + nohup env HF_XET_HIGH_PERFORMANCE=1 $token_env "$HOME/hfenv/bin/hf" download "$MODEL" --local-dir "$MODEL_DIR" \ + > "$NVME_ROOT/logs/model.log" 2>&1 &;; + esac +fi + +if [ -n "$RELEASE_STORE" ]; then + echo "== policy releases: $RELEASE_STORE (background)" + [ -n "$RELEASE_PREFIX" ] || { echo "--release-store needs --release-prefix RUN" >&2; exit 64; } + [ -n "$R2_ENV_FILE" ] || { echo "--release-store needs --r2-env-file (bucket, endpoint, credentials)" >&2; exit 64; } + [ -d "$YETO_DIR/yeto" ] || { echo "$YETO_DIR is not a yeto checkout (--yeto-dir)" >&2; exit 66; } + if [ -z "$PY" ]; then + if [ -x "$YETO_DIR/.venv/bin/python" ]; then PY="$YETO_DIR/.venv/bin/python"; else PY=python3; fi + fi + command -v rclone >/dev/null || echo "warning: rclone not on PATH; the release puller needs it" >&2 + mkdir -p "$RELEASE_STORE" + peers_args=(); [ -n "$RELEASE_PEERS" ] && peers_args=(--peers "$RELEASE_PEERS") + serve_args=(); [ -n "$SERVE_PORT" ] && serve_args=(--serve-port "$SERVE_PORT") + ( + set -a + # shellcheck disable=SC1090 + . "$R2_ENV_FILE" + set +a + export S3_BUCKET="${S3_BUCKET:-${R2_BUCKET:-}}" S3_PREFIX="$RELEASE_PREFIX" S3_REGION="${S3_REGION:-${AWS_REGION:-auto}}" + # s3_delivery builds an env-only rclone remote named "s3w" (RCLONE_CONFIG_S3W_*); + # an S3-compatible endpoint (R2) reaches it through these variables + if [ -n "${AWS_ENDPOINT_URL:-}" ]; then + export RCLONE_CONFIG_S3W_ENDPOINT="${RCLONE_CONFIG_S3W_ENDPOINT:-$AWS_ENDPOINT_URL}" + export RCLONE_S3_ENDPOINT="${RCLONE_S3_ENDPOINT:-$AWS_ENDPOINT_URL}" + fi + [ -n "$S3_BUCKET" ] || { echo "the env file sets neither S3_BUCKET nor R2_BUCKET" >&2; exit 64; } + export PYTHONPATH="$YETO_DIR${PYTHONPATH:+:$PYTHONPATH}" PYTHONUNBUFFERED=1 + nohup "$PY" -m yeto.rl.spot.s3_delivery pull-loop --store "$RELEASE_STORE" "${peers_args[@]}" "${serve_args[@]}" > "$NVME_ROOT/logs/releases.log" 2>&1 & + echo $! > "$NVME_ROOT/logs/releases.pid" + ) + echo "release puller pid $(cat "$NVME_ROOT/logs/releases.pid"), log $NVME_ROOT/logs/releases.log" +fi +echo "started: image pull -> $NVME_ROOT/logs/pull.log, model -> $NVME_ROOT/logs/model.log" diff --git a/scripts/sampler_node/nc_lease.sh b/scripts/sampler_node/nc_lease.sh new file mode 100755 index 000000000..3017bb231 --- /dev/null +++ b/scripts/sampler_node/nc_lease.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Poll a National Compute Public Research ticket until it is active, then write +# the node record and pin the node's SSH host key. +# +# scripts/sampler_node/nc_lease.sh --account N [--out node.json] [--interval 60] [--max-polls 240] +# [--known-hosts ~/.ssh/known_hosts] [--nc-query CMD] +# +# `--nc-query` is the status command of the National Compute tool (default +# `nc-query`, i.e. `nc-query status --account N`), which prints JSON with a +# `ticket` object (state, position, estimated_grant_at, node.public_ip, +# node.ssh_user, node.host_key, granted_at, lease_until, rate_ucr_per_gpu_hour, +# gpus). No account, token or site value lives here: the tool reads its own +# account file; this script only passes the account index through. +# +# Exit: 0 active (node.json written, host key pinned), 1 the ticket ended or +# there is none, 2 still waiting after --max-polls. Requires jq. +set -uo pipefail +ACCOUNT=""; OUT="node.json"; INTERVAL=60; MAX_POLLS=240; KNOWN_HOSTS="${HOME}/.ssh/known_hosts"; NC_QUERY="${NC_QUERY:-nc-query}" +while [ $# -gt 0 ]; do + case "$1" in + --account) ACCOUNT=$2; shift 2;; + --out) OUT=$2; shift 2;; + --interval) INTERVAL=$2; shift 2;; + --max-polls) MAX_POLLS=$2; shift 2;; + --known-hosts) KNOWN_HOSTS=$2; shift 2;; + --nc-query) NC_QUERY=$2; shift 2;; + -h|--help) sed -n '2,20p' "$0"; exit 0;; + *) echo "unknown option $1" >&2; exit 64;; + esac +done +[ -n "$ACCOUNT" ] || { echo "usage: $0 --account N [--out node.json]" >&2; exit 64; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 69; } + +state=none +for _ in $(seq 1 "$MAX_POLLS"); do + st=$($NC_QUERY status --account "$ACCOUNT" 2>&1) || true + state=$(jq -r '.ticket.state // "none"' <<<"$st" 2>/dev/null || echo parse-error) + printf '%s state=%s pos=%s eta=%s\n' "$(date -u +%FT%TZ)" "$state" \ + "$(jq -r '.ticket.position // "-"' <<<"$st" 2>/dev/null)" \ + "$(jq -r '.ticket.estimated_grant_at // "-"' <<<"$st" 2>/dev/null)" + if [ "$state" = active ]; then + jq '{id: .ticket.id, state: .ticket.state, site: .ticket.site_name, ip: .ticket.node.public_ip, + user: .ticket.node.ssh_user, host_key: .ticket.node.host_key, granted_at: .ticket.granted_at, + lease_until: .ticket.lease_until, rate_ucr_per_gpu_hour: .ticket.rate_ucr_per_gpu_hour, + gpus: .ticket.gpus}' <<<"$st" > "$OUT" + ip=$(jq -r .ip "$OUT") + key=$(jq -r '.host_key // empty' "$OUT") + if [ -n "$ip" ] && [ -n "$key" ]; then + mkdir -p "$(dirname "$KNOWN_HOSTS")" + ssh-keygen -R "$ip" -f "$KNOWN_HOSTS" >/dev/null 2>&1 || true + echo "$ip $key" >> "$KNOWN_HOSTS" + fi + cat "$OUT"; echo ACTIVE; exit 0 + fi + case "$state" in + ended|releasing|none|parse-error) echo "$st" | head -40; exit 1;; + esac + sleep "$INTERVAL" +done +echo "TIMEOUT still $state"; exit 2 diff --git a/scripts/sampler_node/run_engine_sampler.sh b/scripts/sampler_node/run_engine_sampler.sh new file mode 100755 index 000000000..a16cb1d70 --- /dev/null +++ b/scripts/sampler_node/run_engine_sampler.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Start the yeto engine sampler (one rollout island) on a sampler node, driving +# the SGLang server run_sglang.sh started. +# +# scripts/sampler_node/run_engine_sampler.sh --spec SPEC.json [--yeto-dir DIR] [--python PY] +# [--r2-env-file FILE] [--log FILE] [--foreground] [-- EXTRA ARGS...] +# +# --spec the sampler spec JSON (`python -m yeto.rl.engine.engine_sampler +# --spec`): engine endpoint, store, buffer, release follower, +# sampler id. Everything the island does comes from the spec. +# --yeto-dir the yeto checkout (default: this script's repository) +# --python interpreter with yeto's dependencies (default: $YETO_DIR/.venv/bin/python +# when it exists, else python3) +# --r2-env-file object store credentials exported into the sampler's environment +# (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT_URL, AWS_REGION, +# R2_BUCKET); mode 0600 required; the values are never printed +# --pip-install install the Python packages the sampler needs and the interpreter +# lacks (safetensors, boto3) before starting; default: refuse to start +# +# Detached by default (nohup, pid in $LOG.pid); exit code 75 from the sampler is a +# reclaim exit (run it again on the next lease). +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +SPEC=""; YETO_DIR=$(cd "$HERE/../.." && pwd); PY=""; R2_ENV_FILE=""; LOG=""; DETACH=1; PIP_INSTALL=0 +REQUIRED_MODULES="safetensors boto3" # the release follower reads safetensors, the store client is boto3 +while [ $# -gt 0 ]; do + case "$1" in + --spec) SPEC=$2; shift 2;; + --yeto-dir) YETO_DIR=$2; shift 2;; + --python) PY=$2; shift 2;; + --r2-env-file) R2_ENV_FILE=$2; shift 2;; + --log) LOG=$2; shift 2;; + --foreground) DETACH=0; shift;; + --pip-install) PIP_INSTALL=1; shift;; + --) shift; break;; + -h|--help) sed -n '2,22p' "$0"; exit 0;; + *) echo "unknown option $1" >&2; exit 64;; + esac +done +[ -n "$SPEC" ] || { echo "usage: $0 --spec SPEC.json [...]" >&2; exit 64; } +[ -f "$SPEC" ] || { echo "spec $SPEC does not exist" >&2; exit 66; } +[ -d "$YETO_DIR/yeto" ] || { echo "$YETO_DIR is not a yeto checkout" >&2; exit 66; } +if [ -z "$PY" ]; then + if [ -x "$YETO_DIR/.venv/bin/python" ]; then PY="$YETO_DIR/.venv/bin/python"; else PY=python3; fi +fi +if [ -n "$R2_ENV_FILE" ]; then + [ -f "$R2_ENV_FILE" ] || { echo "$R2_ENV_FILE does not exist" >&2; exit 66; } + mode=$(stat -c %a "$R2_ENV_FILE" 2>/dev/null || stat -f %Lp "$R2_ENV_FILE") + [ "$mode" = 600 ] || { echo "$R2_ENV_FILE must be mode 600 (is $mode)" >&2; exit 77; } + set -a + # shellcheck disable=SC1090 + . "$R2_ENV_FILE" + set +a +fi +export PYTHONPATH="$YETO_DIR${PYTHONPATH:+:$PYTHONPATH}" PYTHONUNBUFFERED=1 +missing=$("$PY" - $REQUIRED_MODULES <<'PY' +import importlib.util, sys +print(" ".join(m for m in sys.argv[1:] if importlib.util.find_spec(m) is None)) +PY +) +if [ -n "$missing" ]; then + if [ "$PIP_INSTALL" = 1 ]; then + echo "installing into $PY: $missing" + # shellcheck disable=SC2086 + "$PY" -m pip install -q $missing + else + echo "$PY lacks: $missing (pass --pip-install, or install them into the interpreter)" >&2; exit 69 + fi +fi +"$PY" -c 'import yeto.rl.engine.engine_sampler' || { echo "$PY cannot import yeto from $YETO_DIR" >&2; exit 69; } +cmd=("$PY" -m yeto.rl.engine.engine_sampler --spec "$SPEC" "$@") +if [ "$DETACH" = 1 ]; then + LOG=${LOG:-$(dirname "$SPEC")/engine-sampler.log} + nohup "${cmd[@]}" >> "$LOG" 2>&1 & + echo $! > "$LOG.pid" + echo "started engine sampler (pid $(cat "$LOG.pid")), spec $SPEC, log $LOG" +else + exec "${cmd[@]}" +fi diff --git a/scripts/sampler_node/run_sglang.sh b/scripts/sampler_node/run_sglang.sh new file mode 100755 index 000000000..c149e2bff --- /dev/null +++ b/scripts/sampler_node/run_sglang.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start one SGLang server (ROCm image) on a sampler node, in docker. +# +# scripts/sampler_node/run_sglang.sh --image IMAGE --model-dir DIR [--served-name NAME] +# [--tp 8] [--gpus 0,1,...,7] [--port 30000] [--host 0.0.0.0] [--name sglang-0] +# [--cache-dir /mnt/nvme/cache/sglang-0] [--r2-env-file FILE] [--no-routed-experts] +# [--no-fp32-lm-head] [--attn-dp-size N] [--moe-a2a-backend deepep|none] +# [--mem-fraction 0.85] [--shm-size 64g] [--foreground] [-- EXTRA SGLANG ARGS...] +# +# Defaults follow the Kimi K3.1 RL sampler contract (INTERFACES 2.6 / 2.8, S1): +# --enable-return-routed-experts the server returns the routed expert ids +# --enable-fp32-lm-head fp32 logits for score-centering top-k logprobs +# --attn-dp-size TP attention data parallel over the TP group +# (--attn-dp-size 0 turns it off) +# --moe-a2a-backend deepep MoE all-to-all through DeepEP (none: off) +# ROCm environment: ROCM_QUICK_REDUCE_QUANTIZATION=NONE SGLANG_ROCM_RUNTIME_RESERVE_GB=10 +# AITER_SITUV2_A8W4=1 HSA_NO_SCRATCH_RECLAIM=1 (no MX_PATCH) +# Every value is a parameter; nothing here names a site, a host or a model. +# --r2-env-file passes the object store credentials into the container as its +# environment (docker --env-file); the file must be mode 0600 and is never +# printed. The model directory is mounted read-only. +set -euo pipefail +IMAGE=""; MODEL_DIR=""; SERVED=""; TP=8; GPUS=""; PORT=30000; HOST=0.0.0.0; NAME="sglang-0" +CACHE_DIR=""; R2_ENV_FILE=""; ROUTED=1; FP32_HEAD=1; ATTN_DP=""; MOE_A2A=deepep; MEM_FRACTION=""; SHM=64g; DETACH=1 +while [ $# -gt 0 ]; do + case "$1" in + --image) IMAGE=$2; shift 2;; + --model-dir) MODEL_DIR=$2; shift 2;; + --served-name) SERVED=$2; shift 2;; + --tp) TP=$2; shift 2;; + --gpus) GPUS=$2; shift 2;; + --port) PORT=$2; shift 2;; + --host) HOST=$2; shift 2;; + --name) NAME=$2; shift 2;; + --cache-dir) CACHE_DIR=$2; shift 2;; + --r2-env-file) R2_ENV_FILE=$2; shift 2;; + --no-routed-experts) ROUTED=0; shift;; + --no-fp32-lm-head) FP32_HEAD=0; shift;; + --attn-dp-size) ATTN_DP=$2; shift 2;; + --moe-a2a-backend) MOE_A2A=$2; shift 2;; + --mem-fraction) MEM_FRACTION=$2; shift 2;; + --shm-size) SHM=$2; shift 2;; + --foreground) DETACH=0; shift;; + --) shift; break;; + -h|--help) sed -n '2,20p' "$0"; exit 0;; + *) echo "unknown option $1" >&2; exit 64;; + esac +done +[ -n "$IMAGE" ] && [ -n "$MODEL_DIR" ] || { echo "usage: $0 --image IMAGE --model-dir DIR [...]" >&2; exit 64; } +[ -d "$MODEL_DIR" ] || { echo "model dir $MODEL_DIR does not exist" >&2; exit 66; } +SERVED=${SERVED:-$(basename "${MODEL_DIR%/}")} +if [ -z "$GPUS" ]; then GPUS=$(seq -s, 0 $((TP - 1))); fi +CACHE_DIR=${CACHE_DIR:-$(dirname "$MODEL_DIR")/../cache/$NAME} +mkdir -p "$CACHE_DIR" +SUDO=""; [ "$(id -u)" = 0 ] || docker info >/dev/null 2>&1 || SUDO=sudo + +env_file_args=() +if [ -n "$R2_ENV_FILE" ]; then + [ -f "$R2_ENV_FILE" ] || { echo "$R2_ENV_FILE does not exist" >&2; exit 66; } + mode=$(stat -c %a "$R2_ENV_FILE" 2>/dev/null || stat -f %Lp "$R2_ENV_FILE") + [ "$mode" = 600 ] || { echo "$R2_ENV_FILE must be mode 600 (is $mode)" >&2; exit 77; } + env_file_args=(--env-file "$R2_ENV_FILE") +fi + +sglang_args=(--model-path /models/"$SERVED" --served-model-name "$SERVED" + --host "$HOST" --port "$PORT" --tp "$TP" --trust-remote-code) +[ "$ROUTED" = 1 ] && sglang_args+=(--enable-return-routed-experts) +[ "$FP32_HEAD" = 1 ] && sglang_args+=(--enable-fp32-lm-head) +ATTN_DP=${ATTN_DP:-$TP} +[ "$ATTN_DP" != 0 ] && sglang_args+=(--attn-dp-size "$ATTN_DP") +[ "$MOE_A2A" != none ] && sglang_args+=(--moe-a2a-backend "$MOE_A2A") +[ -n "$MEM_FRACTION" ] && sglang_args+=(--mem-fraction-static "$MEM_FRACTION") +sglang_args+=("$@") + +run_mode=(-d --restart unless-stopped); [ "$DETACH" = 1 ] || run_mode=(--rm -it) +$SUDO docker rm -f "$NAME" >/dev/null 2>&1 || true +$SUDO docker run "${run_mode[@]}" --name "$NAME" --network host --ipc host --shm-size "$SHM" \ + --device /dev/kfd --device /dev/dri --group-add video --security-opt seccomp=unconfined \ + --cap-add SYS_PTRACE --ulimit memlock=-1 \ + -v "$MODEL_DIR":/models/"$SERVED":ro -v "$CACHE_DIR":/root/.cache \ + "${env_file_args[@]}" \ + -e HIP_VISIBLE_DEVICES="$GPUS" \ + -e ROCM_QUICK_REDUCE_QUANTIZATION=NONE -e SGLANG_ROCM_RUNTIME_RESERVE_GB=10 \ + -e AITER_SITUV2_A8W4=1 -e HSA_NO_SCRATCH_RECLAIM=1 \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" python3 -m sglang.launch_server "${sglang_args[@]}" +echo "started $NAME: GPUs $GPUS, TP $TP, http://$HOST:$PORT (model $SERVED)" diff --git a/sitecustomize.py b/sitecustomize.py index 5f45e2cd1..51a539ce3 100644 --- a/sitecustomize.py +++ b/sitecustomize.py @@ -24,6 +24,13 @@ install() +if os.environ.get("YETO_SGLANG_MXFP4_INPLACE") == "1": + # In-place reload of MXFP4 routed experts on SGLang's SM100 trtllm-gen path + # (every SGLang process, including the spawned schedulers). + from yeto.rl.adapters.sampler_engines.sglang_mxfp4_reload import install as _install_mxfp4_inplace + + _install_mxfp4_inplace() + def _install_tms_post_pause_hold() -> None: """Hold a Miles trainer actor after TMS pause for crash attribution. diff --git a/tests/conftest.py b/tests/conftest.py index e7a278a62..8406a68f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,40 @@ def pytest_configure(config): import pytest as _pytest +# YETO_REQUIRE_MILES=1 (the Miles image venv): a test that would skip because a +# ``miles.*`` module is not importable fails instead. Without it the loss_hub +# -> loss/hub move silently skipped every upstream-parity test. +if _os.environ.get("YETO_REQUIRE_MILES") == "1": + _importorskip = _pytest.importorskip + + def _importorskip_requiring_miles(modname, *args, **kwargs): + if modname == "miles" or modname.startswith(("miles.", "miles_plugins")): + import importlib + + importlib.import_module(modname) # ImportError: the path moved or Miles is missing + return _importorskip(modname, *args, **kwargs) + + _pytest.importorskip = _importorskip_requiring_miles + +@_pytest.fixture +def ports_v1_pin(monkeypatch): + """Pin the run to the frozen ports-v1 lineage head (the DiLoCo fork/image). + + Feature gates (DP replicas, sharded masters, distopt reader, loss variants) + and the pin-bound GPU declarations hold on that lineage; on the k31 lineage + they are off until the batch that brings the feature back (B2/B8) or the + GPU re-verification lands (yeto-adapter.md 4.2/4.5). Tests of the gate and + declaration machinery use this fixture; test_rl_engine_pins checks the k31 + pin's own (empty) state. + """ + from yeto.rl.adapters.miles import config as miles_config + from yeto.rl.adapters.miles import pins + + monkeypatch.setattr(pins, "MILES_NEXT_COMMIT", pins.MILES_PORTS_V1_COMMIT) + monkeypatch.setattr(miles_config, "MILES_NEXT_COMMIT", pins.MILES_PORTS_V1_COMMIT) + return pins.MILES_PORTS_V1_COMMIT + + # decoupling 7.7a: islands must declare a compat_group (no default in code). # Unit tests that build identities at import time run as one H100 island; # tests of the missing-value error delete it with monkeypatch. @@ -223,3 +257,12 @@ def ppu_plan(tmp_path_factory): pytest_sessionfinish, pytest_sessionstart, ) + + +# Most launch tests build RL argv without --inner-lr; yeto refuses that +# (yeto.launch.rl_args.refuse_unset_inner_lr). Before any test module builds a +# parser (some at import), the CLI default reads as an explicit value here; +# tests/test_rl_inner_lr_guard.py puts the marked default back. +import yeto.cli as _yeto_cli # noqa: E402 + +_yeto_cli.DEFAULT_INNER_LR = 3e-4 diff --git a/tests/data/r0_driver_tape_a50e9d2.json b/tests/data/r0_driver_tape_a50e9d2.json index 1d0f52dc1..37ba6decc 100644 --- a/tests/data/r0_driver_tape_a50e9d2.json +++ b/tests/data/r0_driver_tape_a50e9d2.json @@ -71,7 +71,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -152,7 +152,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -233,7 +233,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -336,7 +336,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -405,7 +405,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -474,7 +474,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, diff --git a/tests/fixtures/dashboard/async_grpo_replay/nc/events-sampler.jsonl b/tests/fixtures/dashboard/async_grpo_replay/nc/events-sampler.jsonl new file mode 100644 index 000000000..54c5f77bf --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/nc/events-sampler.jsonl @@ -0,0 +1,15 @@ +{"available":false,"event":"rl_resource_sample","sampler_id":"nc-s1","t":9927.223791509,"time_unix":1790053763.788} +{"allow_penalties":false,"event":"rl_sampler_start","loop_guard":null,"max_backlog_groups":32,"max_prompt_tokens":2048,"num_rollouts":1000000,"provenance":{},"sampler_id":"nc-s1","sampling":{"allow_penalties":false,"frequency_penalty":0.0,"max_tokens":65536,"n":8,"presence_penalty":0.0,"repetition_penalty":1.0,"seed":null,"stop":[],"temperature":1.0,"top_k":-1,"top_p":1.0},"time_unix":1790053763.789} +{"checks":[{"passed":true},{"answers":[{"answered":true,"ok":true,"problems":[]},{"answered":true,"ok":true,"problems":[]},{"answered":true,"ok":true,"problems":[]},{"answered":true,"ok":true,"problems":[]}],"answers_required":false,"passed":true},{"against":null,"max_probe_jump":null,"means":[-0.331333,-1.663408,-0.829035,-2.813843],"passed":true},{"expected":0,"passed":true,"served":0}],"event":"rl_sampler_load_checked","load_and_check_s":300.597,"pause_s":256.029,"policy_version":0,"r3_cp_overrides":[{"now":1,"size":8}],"sampler_id":"nc-s1","server_info":{"vllm_pin_match":true},"time_unix":1790054064.386} +{"data_cursor":null,"event":"rl_sampler_join","groups":0,"incarnations":[],"latest_release":null,"next_rollout_id":0,"partial_rounds":[],"sampler_id":"nc-s1","time_unix":1790054064.618,"written_rounds":0} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"stop":64},"groups":8,"policy_version":0,"response_tokens":180985,"rollout_id":0,"sampler_id":"nc-s1","samples":64,"time_unix":1790054506.999,"zero_variance_groups":0} +{"buffer_end":8,"event":"rl_sampler_round","generate_s":440.847,"groups":8,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":0,"sampler_id":"nc-s1","time_unix":1790054507.222,"version_lag":0} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"stop":64},"groups":8,"policy_version":0,"response_tokens":370130,"rollout_id":1,"sampler_id":"nc-s1","samples":64,"time_unix":1790055132.062,"zero_variance_groups":0} +{"buffer_end":16,"event":"rl_sampler_round","generate_s":623.947,"groups":8,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":1,"sampler_id":"nc-s1","time_unix":1790055132.284,"version_lag":0} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"stop":64},"groups":6,"policy_version":0,"response_tokens":620253,"rollout_id":2,"sampler_id":"nc-s1","samples":64,"time_unix":1790056636.763,"zero_variance_groups":2} +{"buffer_end":22,"event":"rl_sampler_round","generate_s":1503.695,"groups":6,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":2,"sampler_id":"nc-s1","time_unix":1790056636.992,"version_lag":0} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"length":2,"stop":62},"groups":6,"policy_version":0,"response_tokens":667047,"rollout_id":3,"sampler_id":"nc-s1","samples":64,"time_unix":1790058511.311,"zero_variance_groups":2} +{"buffer_end":28,"event":"rl_sampler_round","generate_s":1873.59,"groups":6,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":3,"sampler_id":"nc-s1","time_unix":1790058511.556,"version_lag":0} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"stop":64},"groups":7,"policy_version":0,"response_tokens":243599,"rollout_id":4,"sampler_id":"nc-s1","samples":64,"time_unix":1790059094.482,"zero_variance_groups":1} +{"buffer_end":35,"event":"rl_sampler_round","generate_s":582.169,"groups":7,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":4,"sampler_id":"nc-s1","time_unix":1790059094.702,"version_lag":0} +{"backlog_groups":35,"event":"rl_sampler_backlog_full","max_backlog_groups":32,"policy_version":0,"rollout_id":5,"sampler_id":"nc-s1","time_unix":1790059095.712} diff --git a/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l0-modal.jsonl b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l0-modal.jsonl new file mode 100644 index 000000000..ebc491cd5 --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l0-modal.jsonl @@ -0,0 +1,300 @@ +{"event":"rl_driver_start","island_id":0,"time_unix":1790000367.727} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":0,"time_unix":1790000393.304} +{"event":"rl_publication","island_id":0,"policy_version":0,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000396.344} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790000396.345} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":0,"stale_groups":0,"time_unix":1790000438.95} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":0,"time_unix":1790000438.95} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":0,"time_unix":1790000438.951} +{"buffer_selected":15,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":0,"origin":{"kind":"merged","learner_id":0,"local_version":0,"num_learners":2},"policy_version":0,"publish_s":0.575,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":1.335,"prune_s":0.248,"read_latest_s":0.234,"stage_export_s":1.78,"upload_s":18.504,"wait_previous_s":0.0},"released_at":1790000441.75,"seq":1,"sha256":null,"staged":true,"time_unix":1790000443.335,"visible_at":1794245443.334} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0000163316726685,"tis_abs":0.012633257545530796,"tis_clipfrac":0.0,"train_rollout_kl":0.0006074385601095855},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":0,"step_seconds":21.030653820999987,"submitted_groups":null,"time_unix":1790000459.984,"tok_per_s":342.6426996199475,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0004753563553094864,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.0004753563553094864,"ppo_kl":0.0,"tis":1.0000163316726685,"tis_abs":0.012633257545530796,"tis_clipfrac":0.0,"train_rollout_kl":0.0006074385601095855,"train_rollout_logprob_abs_diff":0.012638493441045284},"train_step":1,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":0,"rollout_id":0,"t":314.949694313,"time_unix":1790000459.987,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_tokens":0,"cross_version_truncated_fraction":null,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":0,"resubmitted_groups":0,"rollout_id":0,"t":314.949990031,"time_unix":1790000459.987,"trained_groups_with_older_tokens":0,"trained_response_tokens":7206,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":0,"time_unix":1790000459.988} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":1,"time_unix":1790000461.112} +{"event":"rl_publication","island_id":0,"policy_version":1,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000462.272} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":42.606,"publish":4.202,"sync":1.124,"train":21.037},"rollout_id":0,"time_unix":1790000462.272,"train_fraction":0.3049,"train_seconds":21.031,"trained_tokens":7206,"wall_seconds":68.969} +{"event":"rl_round_wall","island_id":0,"rollout_id":0,"rollout_seconds":42.605820255000026,"round_seconds":65.92750832000002,"time_unix":1790000462.272} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790000462.273} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":1,"stale_groups":0,"time_unix":1790000486.194} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":1,"time_unix":1790000486.194} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":1,"time_unix":1790000486.195} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0001987218856812,"tis_abs":0.012189678847789764,"tis_clipfrac":0.0,"train_rollout_kl":0.0006161475903354585},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":1,"step_seconds":11.11442780599998,"submitted_groups":null,"time_unix":1790000497.31,"tok_per_s":616.8558669569005,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0007572919130325317,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0007572919130325317,"ppo_kl":0.0,"tis":1.0001987218856812,"tis_abs":0.012189678847789764,"tis_clipfrac":0.0,"train_rollout_kl":0.0006161475903354585,"train_rollout_logprob_abs_diff":0.01218851562589407},"train_step":2,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":1,"rollout_id":1,"t":352.272950296,"time_unix":1790000497.31,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6856,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":1,"resubmitted_groups":0,"rollout_id":1,"t":352.273285825,"time_unix":1790000497.311,"trained_groups_with_older_tokens":8,"trained_response_tokens":6856,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":1,"time_unix":1790000497.311} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":2,"time_unix":1790000497.935} +{"event":"rl_publication","island_id":0,"policy_version":2,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000499.015} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":23.922,"publish":1.082,"sync":0.623,"train":11.116},"rollout_id":1,"time_unix":1790000499.016,"train_fraction":0.3025,"train_seconds":11.114,"trained_tokens":6856,"wall_seconds":36.744} +{"event":"rl_round_wall","island_id":0,"rollout_id":1,"rollout_seconds":23.92200638899999,"round_seconds":36.743267163999974,"time_unix":1790000499.016} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790000499.016} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":2,"stale_groups":0,"time_unix":1790000533.813} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":2,"time_unix":1790000533.813} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":2,"time_unix":1790000533.813} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0008580684661865,"tis_abs":0.014256625436246395,"tis_clipfrac":0.0,"train_rollout_kl":0.0007820039172656834},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":2,"step_seconds":11.245556385000043,"submitted_groups":null,"time_unix":1790000545.059,"tok_per_s":705.5231176096202,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.00029392167925834656,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.00029392167925834656,"ppo_kl":0.0,"tis":1.0008580684661865,"tis_abs":0.014256625436246395,"tis_clipfrac":0.0,"train_rollout_kl":0.0007820039172656834,"train_rollout_logprob_abs_diff":0.014187541790306568},"train_step":3,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":2,"rollout_id":2,"t":400.02294787,"time_unix":1790000545.061,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7934,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":2,"resubmitted_groups":0,"rollout_id":2,"t":400.023883656,"time_unix":1790000545.061,"trained_groups_with_older_tokens":8,"trained_response_tokens":7934,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":2,"time_unix":1790000545.062} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":3,"time_unix":1790000545.672} +{"event":"rl_publication","island_id":0,"policy_version":3,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000546.687} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":34.797,"publish":1.016,"sync":0.61,"train":11.248},"rollout_id":2,"time_unix":1790000546.687,"train_fraction":0.2359,"train_seconds":11.246,"trained_tokens":7934,"wall_seconds":47.672} +{"event":"rl_round_wall","island_id":0,"rollout_id":2,"rollout_seconds":34.79696821299996,"round_seconds":47.67142055599999,"time_unix":1790000546.688} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790000546.688} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":3,"stale_groups":0,"time_unix":1790000570.231} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":3,"time_unix":1790000570.231} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":3,"time_unix":1790000570.231} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.000259280204773,"tis_abs":0.013165702112019062,"tis_clipfrac":0.0,"train_rollout_kl":0.0006902799941599369},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":3,"step_seconds":11.309105617,"submitted_groups":null,"time_unix":1790000581.541,"tok_per_s":630.4654179975195,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-6.258860230445862e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-6.258860230445862e-05,"ppo_kl":0.0,"tis":1.000259280204773,"tis_abs":0.013165702112019062,"tis_clipfrac":0.0,"train_rollout_kl":0.0006902799941599369,"train_rollout_logprob_abs_diff":0.013203444890677929},"train_step":4,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":3,"rollout_id":3,"t":436.504289602,"time_unix":1790000581.542,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7130,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":3,"resubmitted_groups":0,"rollout_id":3,"t":436.504561699,"time_unix":1790000581.542,"trained_groups_with_older_tokens":8,"trained_response_tokens":7130,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":3,"time_unix":1790000581.542} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":4,"time_unix":1790000583.257} +{"event":"rl_publication","island_id":0,"policy_version":4,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000584.268} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":23.543,"publish":1.013,"sync":1.713,"train":11.311},"rollout_id":3,"time_unix":1790000584.268,"train_fraction":0.3009,"train_seconds":11.309,"trained_tokens":7130,"wall_seconds":37.581} +{"event":"rl_round_wall","island_id":0,"rollout_id":3,"rollout_seconds":23.543313177000016,"round_seconds":37.58060400599999,"time_unix":1790000584.269} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1790000584.269} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":4,"stale_groups":0,"time_unix":1790000624.547} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":4,"time_unix":1790000624.548} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":4,"time_unix":1790000624.548} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0002202987670898,"tis_abs":0.012768377549946308,"tis_clipfrac":0.0,"train_rollout_kl":0.000587461341638118},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":4,"step_seconds":11.005705316999979,"submitted_groups":null,"time_unix":1790000635.555,"tok_per_s":610.3198120001065,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-3.9789825677871704e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-3.9789825677871704e-05,"ppo_kl":0.0,"tis":1.0002202987670898,"tis_abs":0.012768377549946308,"tis_clipfrac":0.0,"train_rollout_kl":0.000587461341638118,"train_rollout_logprob_abs_diff":0.012741588987410069},"train_step":5,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":4,"rollout_id":4,"t":490.517652787,"time_unix":1790000635.555,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6717,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":4,"resubmitted_groups":0,"rollout_id":4,"t":490.517973786,"time_unix":1790000635.555,"trained_groups_with_older_tokens":8,"trained_response_tokens":6717,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":4,"time_unix":1790000635.556} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":5,"time_unix":1790000636.194} +{"event":"rl_publication","island_id":0,"policy_version":5,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000639.136} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":40.279,"publish":2.944,"sync":0.638,"train":11.008},"rollout_id":4,"time_unix":1790000639.137,"train_fraction":0.2006,"train_seconds":11.006,"trained_tokens":6717,"wall_seconds":54.869} +{"event":"rl_round_wall","island_id":0,"rollout_id":4,"rollout_seconds":40.279171417999976,"round_seconds":54.868577657,"time_unix":1790000639.137} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1790000639.138} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":5,"stale_groups":0,"time_unix":1790000678.786} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":5,"time_unix":1790000678.786} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":5,"time_unix":1790000678.786} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9993799924850464,"tis_abs":0.016260061413049698,"tis_clipfrac":0.0,"train_rollout_kl":0.0008823196403682232},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":5,"step_seconds":11.36364422500003,"submitted_groups":null,"time_unix":1790000690.151,"tok_per_s":695.1995190609708,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.00024359486997127533,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.00024359486997127533,"ppo_kl":0.0,"tis":0.9993799924850464,"tis_abs":0.016260061413049698,"tis_clipfrac":0.0,"train_rollout_kl":0.0008823196403682232,"train_rollout_logprob_abs_diff":0.01640203967690468},"train_step":6,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":5,"rollout_id":5,"t":545.113920314,"time_unix":1790000690.151,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7900,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":5,"resubmitted_groups":0,"rollout_id":5,"t":545.114222983,"time_unix":1790000690.152,"trained_groups_with_older_tokens":8,"trained_response_tokens":7900,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":5,"time_unix":1790000690.152} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":6,"time_unix":1790000690.748} +{"buffer_selected":95,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":5,"origin":{"kind":"merged","learner_id":0,"local_version":5,"num_learners":2},"policy_version":1,"publish_s":0.777,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":1.293,"prune_s":0.451,"read_latest_s":0.245,"stage_export_s":1.912,"upload_s":17.471,"wait_previous_s":0.0},"released_at":1790000705.215,"seq":2,"sha256":null,"staged":true,"time_unix":1790000706.96,"visible_at":1794245706.96} +{"event":"rl_publication","island_id":0,"policy_version":6,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000707.395} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":39.648,"publish":16.65,"sync":0.594,"train":11.365},"rollout_id":5,"time_unix":1790000707.395,"train_fraction":0.1665,"train_seconds":11.364,"trained_tokens":7900,"wall_seconds":68.258} +{"event":"rl_round_wall","island_id":0,"rollout_id":5,"rollout_seconds":39.64844161599996,"round_seconds":68.25715309699996,"time_unix":1790000707.395} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":6,"rollout_id":6,"time_unix":1790000707.395} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":6,"stale_groups":0,"time_unix":1790000736.568} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":6,"time_unix":1790000736.568} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":6,"time_unix":1790000736.569} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.999879002571106,"tis_abs":0.01926540397107601,"tis_clipfrac":0.0001516990305390209,"train_rollout_kl":0.0010987819405272603},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":6,"step_seconds":11.59507136100001,"submitted_groups":null,"time_unix":1790000748.164,"tok_per_s":772.1384130599998,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":8.296780288219452e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":8.296780288219452e-05,"ppo_kl":0.0,"tis":0.999879002571106,"tis_abs":0.01926540397107601,"tis_clipfrac":0.0001516990305390209,"train_rollout_kl":0.0010987819405272603,"train_rollout_logprob_abs_diff":0.019325198605656624},"train_step":7,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":6,"rollout_id":6,"t":603.127541778,"time_unix":1790000748.165,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":8953,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":6,"resubmitted_groups":0,"rollout_id":6,"t":603.12797929,"time_unix":1790000748.166,"trained_groups_with_older_tokens":8,"trained_response_tokens":8953,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":6,"time_unix":1790000748.166} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":7,"time_unix":1790000748.765} +{"event":"rl_publication","island_id":0,"policy_version":7,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000750.002} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":29.173,"publish":1.239,"sync":0.598,"train":11.597},"rollout_id":6,"time_unix":1790000750.002,"train_fraction":0.2721,"train_seconds":11.595,"trained_tokens":8953,"wall_seconds":42.607} +{"event":"rl_round_wall","island_id":0,"rollout_id":6,"rollout_seconds":29.17327150099993,"round_seconds":42.60721604899993,"time_unix":1790000750.003} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":7,"rollout_id":7,"time_unix":1790000750.003} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":7,"stale_groups":0,"time_unix":1790000781.397} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":7,"time_unix":1790000781.398} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":7,"time_unix":1790000781.398} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9992583990097046,"tis_abs":0.01947125978767872,"tis_clipfrac":0.0002244925417471677,"train_rollout_kl":0.0013732132501900196},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":7,"step_seconds":11.321472863000054,"submitted_groups":null,"time_unix":1790000792.72,"tok_per_s":650.5337325914293,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0005882736295461655,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0005882736295461655,"ppo_kl":0.0,"tis":0.9992583990097046,"tis_abs":0.01947125978767872,"tis_clipfrac":0.0002244925417471677,"train_rollout_kl":0.0013732132501900196,"train_rollout_logprob_abs_diff":0.01973971165716648},"train_step":8,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":7,"rollout_id":7,"t":647.683072893,"time_unix":1790000792.721,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7365,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":7,"resubmitted_groups":0,"rollout_id":7,"t":647.683394152,"time_unix":1790000792.721,"trained_groups_with_older_tokens":8,"trained_response_tokens":7365,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":7,"time_unix":1790000792.721} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":8,"time_unix":1790000794.759} +{"event":"rl_publication","island_id":0,"policy_version":8,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000795.935} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":31.395,"publish":1.178,"sync":2.037,"train":11.323},"rollout_id":7,"time_unix":1790000795.936,"train_fraction":0.2465,"train_seconds":11.321,"trained_tokens":7365,"wall_seconds":45.933} +{"event":"rl_round_wall","island_id":0,"rollout_id":7,"rollout_seconds":31.39504420100002,"round_seconds":45.93303566999998,"time_unix":1790000795.936} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":8,"rollout_id":8,"time_unix":1790000795.936} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":8,"stale_groups":0,"time_unix":1790000836.778} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":8,"time_unix":1790000836.778} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":8,"time_unix":1790000836.778} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9997923970222473,"tis_abs":0.0168465506285429,"tis_clipfrac":0.0,"train_rollout_kl":0.001141420449130237},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":8,"step_seconds":11.118321986000069,"submitted_groups":null,"time_unix":1790000847.897,"tok_per_s":632.4695407143748,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.000722663477063179,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.000722663477063179,"ppo_kl":0.0,"tis":0.9997923970222473,"tis_abs":0.0168465506285429,"tis_clipfrac":0.0,"train_rollout_kl":0.001141420449130237,"train_rollout_logprob_abs_diff":0.0170314721763134},"train_step":9,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":8,"rollout_id":8,"t":702.860265302,"time_unix":1790000847.898,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7032,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":8,"resubmitted_groups":0,"rollout_id":8,"t":702.86053252,"time_unix":1790000847.898,"trained_groups_with_older_tokens":8,"trained_response_tokens":7032,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":8,"time_unix":1790000847.898} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":9,"time_unix":1790000848.622} +{"event":"rl_publication","island_id":0,"policy_version":9,"sync/outer_version":2,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000850.956} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":40.842,"publish":2.334,"sync":0.724,"train":11.12},"rollout_id":8,"time_unix":1790000850.956,"train_fraction":0.2021,"train_seconds":11.118,"trained_tokens":7032,"wall_seconds":55.02} +{"event":"rl_round_wall","island_id":0,"rollout_id":8,"rollout_seconds":40.84201149800003,"round_seconds":55.019984978000025,"time_unix":1790000850.956} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":9,"rollout_id":9,"time_unix":1790000850.956} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":9,"stale_groups":0,"time_unix":1790000906.084} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":2,"rollout_id":9,"time_unix":1790000906.085} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":9,"time_unix":1790000906.085} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0008835792541504,"tis_abs":0.019941220059990883,"tis_clipfrac":0.0010490071726962924,"train_rollout_kl":0.0019983150996267796},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":9,"step_seconds":11.222138901999983,"submitted_groups":null,"time_unix":1790000917.308,"tok_per_s":605.8559833712536,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":4.712305963039398e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":4.712305963039398e-05,"ppo_kl":0.0,"tis":1.0008835792541504,"tis_abs":0.019941220059990883,"tis_clipfrac":0.0010490071726962924,"train_rollout_kl":0.0019983150996267796,"train_rollout_logprob_abs_diff":0.019938170909881592},"train_step":10,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":9,"rollout_id":9,"t":772.270742893,"time_unix":1790000917.308,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6799,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":9,"resubmitted_groups":0,"rollout_id":9,"t":772.271024331,"time_unix":1790000917.308,"trained_groups_with_older_tokens":8,"trained_response_tokens":6799,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":9,"time_unix":1790000917.309} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":10,"time_unix":1790000917.9} +{"buffer_selected":159,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":9,"origin":{"kind":"merged","learner_id":0,"local_version":9,"num_learners":2},"policy_version":2,"publish_s":0.963,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":1.292,"prune_s":2.614,"read_latest_s":0.243,"stage_export_s":1.098,"upload_s":23.659,"wait_previous_s":0.0},"released_at":1790000940.46,"seq":3,"sha256":null,"staged":true,"time_unix":1790000944.368,"visible_at":1794245944.367} +{"event":"rl_publication","island_id":0,"policy_version":10,"sync/outer_version":2,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000944.793} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":55.128,"publish":26.895,"sync":0.59,"train":11.224},"rollout_id":9,"time_unix":1790000944.794,"train_fraction":0.1196,"train_seconds":11.222,"trained_tokens":6799,"wall_seconds":93.838} +{"event":"rl_round_wall","island_id":0,"rollout_id":9,"rollout_seconds":55.128471458000035,"round_seconds":93.83741734900002,"time_unix":1790000944.794} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":10,"rollout_id":10,"time_unix":1790000944.794} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":10,"stale_groups":0,"time_unix":1790000985.87} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":2,"rollout_id":10,"time_unix":1790000985.87} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":10,"time_unix":1790000985.87} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9989138841629028,"tis_abs":0.017696009948849678,"tis_clipfrac":0.00022007041843608022,"train_rollout_kl":0.0012196797179058194},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":10,"step_seconds":11.126529840999979,"submitted_groups":null,"time_unix":1790000996.998,"tok_per_s":623.0154503748671,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":9.704381227493286e-06,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":9.704381227493286e-06,"ppo_kl":0.0,"tis":0.9989138841629028,"tis_abs":0.017696009948849678,"tis_clipfrac":0.00022007041843608022,"train_rollout_kl":0.0012196797179058194,"train_rollout_logprob_abs_diff":0.017974290996789932},"train_step":11,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":10,"rollout_id":10,"t":851.960870419,"time_unix":1790000996.998,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6932,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":10,"resubmitted_groups":0,"rollout_id":10,"t":851.961176498,"time_unix":1790000996.999,"trained_groups_with_older_tokens":8,"trained_response_tokens":6932,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":10,"time_unix":1790000996.999} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":11,"time_unix":1790000997.621} +{"event":"rl_publication","island_id":0,"policy_version":11,"sync/outer_version":2,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000998.925} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":41.076,"publish":1.305,"sync":0.622,"train":11.128},"rollout_id":10,"time_unix":1790000998.925,"train_fraction":0.2055,"train_seconds":11.127,"trained_tokens":6932,"wall_seconds":54.131} +{"event":"rl_round_wall","island_id":0,"rollout_id":10,"rollout_seconds":41.07624659199996,"round_seconds":54.13092049800002,"time_unix":1790000998.925} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":11,"rollout_id":11,"time_unix":1790000998.925} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":11,"stale_groups":0,"time_unix":1790001038.023} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":2,"rollout_id":11,"time_unix":1790001038.024} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":11,"time_unix":1790001038.024} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9992305040359497,"tis_abs":0.02002197876572609,"tis_clipfrac":0.0015803563874214888,"train_rollout_kl":0.0018548418302088976},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":11,"step_seconds":10.707567701000016,"submitted_groups":null,"time_unix":1790001048.732,"tok_per_s":551.4791187776951,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.00042511336505413055,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.00042511336505413055,"ppo_kl":0.0,"tis":0.9992305040359497,"tis_abs":0.02002197876572609,"tis_clipfrac":0.0015803563874214888,"train_rollout_kl":0.0018548418302088976,"train_rollout_logprob_abs_diff":0.02050277590751648},"train_step":12,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":11,"rollout_id":11,"t":903.69539422,"time_unix":1790001048.733,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":5905,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":11,"resubmitted_groups":0,"rollout_id":11,"t":903.695684248,"time_unix":1790001048.733,"trained_groups_with_older_tokens":8,"trained_response_tokens":5905,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":11,"time_unix":1790001048.733} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":12,"time_unix":1790001050.453} +{"event":"rl_publication","island_id":0,"policy_version":12,"sync/outer_version":2,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001051.881} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":39.099,"publish":1.429,"sync":1.72,"train":10.709},"rollout_id":11,"time_unix":1790001051.882,"train_fraction":0.2022,"train_seconds":10.708,"trained_tokens":5905,"wall_seconds":52.957} +{"event":"rl_round_wall","island_id":0,"rollout_id":11,"rollout_seconds":39.098569906999955,"round_seconds":52.956770976999906,"time_unix":1790001051.882} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":12,"rollout_id":12,"time_unix":1790001051.882} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":12,"stale_groups":0,"time_unix":1790001095.141} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":2,"rollout_id":12,"time_unix":1790001095.142} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":12,"time_unix":1790001095.142} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9984570145606995,"tis_abs":0.020099855959415436,"tis_clipfrac":0.0007812500698491931,"train_rollout_kl":0.001680627465248108},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":12,"step_seconds":11.115001907999954,"submitted_groups":null,"time_unix":1790001106.258,"tok_per_s":576.0682771805447,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":9.37432050704956e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":9.37432050704956e-05,"ppo_kl":0.0,"tis":0.9984570145606995,"tis_abs":0.020099855959415436,"tis_clipfrac":0.0007812500698491931,"train_rollout_kl":0.001680627465248108,"train_rollout_logprob_abs_diff":0.02032221294939518},"train_step":13,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":12,"rollout_id":12,"t":961.220864068,"time_unix":1790001106.258,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6403,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":12,"resubmitted_groups":0,"rollout_id":12,"t":961.221183357,"time_unix":1790001106.259,"trained_groups_with_older_tokens":8,"trained_response_tokens":6403,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":12,"time_unix":1790001106.259} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":13,"time_unix":1790001107.105} +{"event":"rl_publication","island_id":0,"policy_version":13,"sync/outer_version":3,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001110.133} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":43.26,"publish":3.028,"sync":0.846,"train":11.117},"rollout_id":12,"time_unix":1790001110.133,"train_fraction":0.1908,"train_seconds":11.115,"trained_tokens":6403,"wall_seconds":58.251} +{"event":"rl_round_wall","island_id":0,"rollout_id":12,"rollout_seconds":43.25956996499997,"round_seconds":58.25098772399997,"time_unix":1790001110.133} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":13,"rollout_id":13,"time_unix":1790001110.134} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":13,"stale_groups":0,"time_unix":1790001149.78} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":3,"rollout_id":13,"time_unix":1790001149.781} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":13,"time_unix":1790001149.781} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9994939565658569,"tis_abs":0.021909082308411598,"tis_clipfrac":0.0009176499443128705,"train_rollout_kl":0.0021876166574656963},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":13,"step_seconds":11.045672402000037,"submitted_groups":null,"time_unix":1790001160.827,"tok_per_s":644.8679392945096,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":2.3245811462402344e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":2.3245811462402344e-05,"ppo_kl":0.0,"tis":0.9994939565658569,"tis_abs":0.021909082308411598,"tis_clipfrac":0.0009176499443128705,"train_rollout_kl":0.0021876166574656963,"train_rollout_logprob_abs_diff":0.02232322096824646},"train_step":14,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":13,"rollout_id":13,"t":1015.790597043,"time_unix":1790001160.828,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7123,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":13,"resubmitted_groups":0,"rollout_id":13,"t":1015.790882931,"time_unix":1790001160.828,"trained_groups_with_older_tokens":8,"trained_response_tokens":7123,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":13,"time_unix":1790001160.829} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":14,"time_unix":1790001161.476} +{"buffer_selected":223,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":13,"origin":{"kind":"merged","learner_id":0,"local_version":13,"num_learners":2},"policy_version":3,"publish_s":0.981,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":2.233,"prune_s":2.697,"read_latest_s":0.23,"stage_export_s":1.782,"upload_s":31.733,"wait_previous_s":0.0},"released_at":1790001180.429,"seq":4,"sha256":null,"staged":true,"time_unix":1790001185.359,"visible_at":1794246185.359} +{"event":"rl_publication","island_id":0,"policy_version":14,"sync/outer_version":3,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001185.603} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":39.647,"publish":24.129,"sync":0.646,"train":11.048},"rollout_id":13,"time_unix":1790001185.604,"train_fraction":0.1464,"train_seconds":11.046,"trained_tokens":7123,"wall_seconds":75.47} +{"event":"rl_round_wall","island_id":0,"rollout_id":13,"rollout_seconds":39.64728179100007,"round_seconds":75.47018826700003,"time_unix":1790001185.604} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":14,"rollout_id":14,"time_unix":1790001185.604} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":14,"stale_groups":0,"time_unix":1790001235.031} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":3,"rollout_id":14,"time_unix":1790001235.031} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":14,"time_unix":1790001235.032} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9980000853538513,"tis_abs":0.026657728478312492,"tis_clipfrac":0.0022659662645310163,"train_rollout_kl":0.00307315937243402},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":14,"step_seconds":10.668352167999956,"submitted_groups":null,"time_unix":1790001245.701,"tok_per_s":492.4846796639814,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":3.166124224662781e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":3.166124224662781e-05,"ppo_kl":0.0,"tis":0.9980000853538513,"tis_abs":0.026657728478312492,"tis_clipfrac":0.0022659662645310163,"train_rollout_kl":0.00307315937243402,"train_rollout_logprob_abs_diff":0.027076687663793564},"train_step":15,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":14,"rollout_id":14,"t":1100.663815331,"time_unix":1790001245.701,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":5254,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":14,"resubmitted_groups":0,"rollout_id":14,"t":1100.66413689,"time_unix":1790001245.702,"trained_groups_with_older_tokens":8,"trained_response_tokens":5254,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":14,"time_unix":1790001245.702} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":15,"time_unix":1790001246.339} +{"event":"rl_publication","island_id":0,"policy_version":15,"sync/outer_version":3,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001247.493} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":49.428,"publish":1.155,"sync":0.637,"train":10.67},"rollout_id":14,"time_unix":1790001247.494,"train_fraction":0.1724,"train_seconds":10.668,"trained_tokens":5254,"wall_seconds":61.89} +{"event":"rl_round_wall","island_id":0,"rollout_id":14,"rollout_seconds":49.42758945500009,"round_seconds":61.88980480400005,"time_unix":1790001247.494} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":15,"rollout_id":15,"time_unix":1790001247.494} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":15,"stale_groups":0,"time_unix":1790001281.758} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":3,"rollout_id":15,"time_unix":1790001281.759} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":15,"time_unix":1790001281.759} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9988275170326233,"tis_abs":0.02504134736955166,"tis_clipfrac":0.0006329235038720071,"train_rollout_kl":0.0023012007586658},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":15,"step_seconds":11.263476155000035,"submitted_groups":null,"time_unix":1790001293.023,"tok_per_s":642.4304451328576,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0001550782471895218,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0001550782471895218,"ppo_kl":0.0,"tis":0.9988275170326233,"tis_abs":0.02504134736955166,"tis_clipfrac":0.0006329235038720071,"train_rollout_kl":0.0023012007586658,"train_rollout_logprob_abs_diff":0.02535233274102211},"train_step":16,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":15,"rollout_id":15,"t":1147.9866163,"time_unix":1790001293.024,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7236,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":15,"resubmitted_groups":0,"rollout_id":15,"t":1147.986959089,"time_unix":1790001293.024,"trained_groups_with_older_tokens":8,"trained_response_tokens":7236,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":15,"time_unix":1790001293.025} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":16,"time_unix":1790001295.957} +{"event":"rl_publication","island_id":0,"policy_version":16,"sync/outer_version":3,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001297.183} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":34.265,"publish":1.231,"sync":2.929,"train":11.266},"rollout_id":15,"time_unix":1790001297.184,"train_fraction":0.2267,"train_seconds":11.263,"trained_tokens":7236,"wall_seconds":49.69} +{"event":"rl_round_wall","island_id":0,"rollout_id":15,"rollout_seconds":34.26487917700001,"round_seconds":49.68980307200013,"time_unix":1790001297.184} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":16,"rollout_id":16,"time_unix":1790001297.184} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":16,"stale_groups":0,"time_unix":1790001319.935} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]],[[1]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":3,"rollout_id":16,"time_unix":1790001319.936} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":16,"time_unix":1790001319.937} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9996217489242554,"tis_abs":0.02852676808834076,"tis_clipfrac":0.0019838265143334866,"train_rollout_kl":0.0038230344653129578},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":16,"step_seconds":11.071945272999983,"submitted_groups":null,"time_unix":1790001331.009,"tok_per_s":650.2922316241845,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0007974710315465927,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0007974710315465927,"ppo_kl":0.0,"tis":0.9996217489242554,"tis_abs":0.02852676808834076,"tis_clipfrac":0.0019838265143334866,"train_rollout_kl":0.0038230344653129578,"train_rollout_logprob_abs_diff":0.02935338020324707},"train_step":17,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":16,"rollout_id":16,"t":1185.972291085,"time_unix":1790001331.01,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7200,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":16,"resubmitted_groups":0,"rollout_id":16,"t":1185.972572922,"time_unix":1790001331.01,"trained_groups_with_older_tokens":8,"trained_response_tokens":7200,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":16,"time_unix":1790001331.01} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":17,"time_unix":1790001331.623} +{"event":"rl_publication","island_id":0,"policy_version":17,"sync/outer_version":4,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001334.624} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":22.752,"publish":3.003,"sync":0.612,"train":11.074},"rollout_id":16,"time_unix":1790001334.624,"train_fraction":0.2957,"train_seconds":11.072,"trained_tokens":7200,"wall_seconds":37.441} +{"event":"rl_round_wall","island_id":0,"rollout_id":16,"rollout_seconds":22.752187879000076,"round_seconds":37.440369765000014,"time_unix":1790001334.625} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":17,"rollout_id":17,"time_unix":1790001334.625} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":17,"stale_groups":0,"time_unix":1790001353.927} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":4,"rollout_id":17,"time_unix":1790001353.927} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":17,"time_unix":1790001353.928} +{"buffer_selected":287,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":17,"origin":{"kind":"merged","learner_id":0,"local_version":17,"num_learners":2},"policy_version":4,"publish_s":0.999,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":1.309,"prune_s":2.72,"read_latest_s":0.246,"stage_export_s":1.708,"upload_s":24.013,"wait_previous_s":0.0},"released_at":1790001360.273,"seq":5,"sha256":null,"staged":true,"time_unix":1790001364.302,"visible_at":1794246364.302} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0012112855911255,"tis_abs":0.023364249616861343,"tis_clipfrac":0.00032237317645922303,"train_rollout_kl":0.001978767104446888},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":17,"step_seconds":10.99458940799991,"submitted_groups":null,"time_unix":1790001364.923,"tok_per_s":594.383269578488,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.00014959461987018585,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.00014959461987018585,"ppo_kl":0.0,"tis":1.0012112855911255,"tis_abs":0.023364249616861343,"tis_clipfrac":0.00032237317645922303,"train_rollout_kl":0.001978767104446888,"train_rollout_logprob_abs_diff":0.023461565375328064},"train_step":18,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":17,"rollout_id":17,"t":1219.885900049,"time_unix":1790001364.923,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6535,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":17,"resubmitted_groups":0,"rollout_id":17,"t":1219.886214638,"time_unix":1790001364.924,"trained_groups_with_older_tokens":8,"trained_response_tokens":6535,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":17,"time_unix":1790001364.924} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":18,"time_unix":1790001365.55} +{"event":"rl_publication","island_id":0,"policy_version":18,"sync/outer_version":4,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001366.823} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":19.303,"publish":1.275,"sync":0.626,"train":10.996},"rollout_id":17,"time_unix":1790001366.824,"train_fraction":0.3415,"train_seconds":10.995,"trained_tokens":6535,"wall_seconds":32.199} +{"event":"rl_round_wall","island_id":0,"rollout_id":17,"rollout_seconds":19.302553406000015,"round_seconds":32.19946897999989,"time_unix":1790001366.825} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":18,"rollout_id":18,"time_unix":1790001366.825} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":18,"stale_groups":0,"time_unix":1790001369.202} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":4,"rollout_id":18,"time_unix":1790001369.203} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":18,"time_unix":1790001369.203} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9979467391967773,"tis_abs":0.020613469183444977,"tis_clipfrac":0.000202922077733092,"train_rollout_kl":0.0016542543889954686},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":18,"step_seconds":11.455258424000021,"submitted_groups":null,"time_unix":1790001380.659,"tok_per_s":705.3529218573991,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.000173225998878479,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.000173225998878479,"ppo_kl":0.0,"tis":0.9979467391967773,"tis_abs":0.020613469183444977,"tis_clipfrac":0.000202922077733092,"train_rollout_kl":0.0016542543889954686,"train_rollout_logprob_abs_diff":0.021066883578896523},"train_step":19,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":18,"rollout_id":18,"t":1235.621888623,"time_unix":1790001380.659,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":8080,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":18,"resubmitted_groups":0,"rollout_id":18,"t":1235.622183081,"time_unix":1790001380.66,"trained_groups_with_older_tokens":8,"trained_response_tokens":8080,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":18,"time_unix":1790001380.66} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":19,"time_unix":1790001381.251} +{"event":"rl_publication","island_id":0,"policy_version":19,"sync/outer_version":4,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001382.509} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":2.378,"publish":1.26,"sync":0.591,"train":11.457},"rollout_id":18,"time_unix":1790001382.51,"train_fraction":0.7303,"train_seconds":11.455,"trained_tokens":8080,"wall_seconds":15.686} +{"event":"rl_round_wall","island_id":0,"rollout_id":18,"rollout_seconds":2.377803558000096,"round_seconds":15.68472966000013,"time_unix":1790001382.51} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":19,"rollout_id":19,"time_unix":1790001382.51} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":19,"stale_groups":0,"time_unix":1790001384.781} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":4,"rollout_id":19,"time_unix":1790001384.782} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":19,"time_unix":1790001384.782} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9992741942405701,"tis_abs":0.02314995974302292,"tis_clipfrac":0.001264921622350812,"train_rollout_kl":0.0023578666150569916},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":19,"step_seconds":11.311609020999867,"submitted_groups":null,"time_unix":1790001396.094,"tok_per_s":601.2407242306576,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0006402879953384399,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0006402879953384399,"ppo_kl":0.0,"tis":0.9992741942405701,"tis_abs":0.02314995974302292,"tis_clipfrac":0.001264921622350812,"train_rollout_kl":0.0023578666150569916,"train_rollout_logprob_abs_diff":0.02343938685953617},"train_step":20,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":19,"rollout_id":19,"t":1251.057423191,"time_unix":1790001396.095,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6801,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":19,"resubmitted_groups":0,"rollout_id":19,"t":1251.057887244,"time_unix":1790001396.095,"trained_groups_with_older_tokens":8,"trained_response_tokens":6801,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":19,"time_unix":1790001396.096} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":20,"time_unix":1790001397.827} +{"event":"rl_publication","island_id":0,"policy_version":20,"sync/outer_version":4,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001398.968} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":2.272,"publish":1.141,"sync":1.732,"train":11.313},"rollout_id":19,"time_unix":1790001398.968,"train_fraction":0.6873,"train_seconds":11.312,"trained_tokens":6801,"wall_seconds":16.459} +{"event":"rl_round_wall","island_id":0,"rollout_id":19,"rollout_seconds":2.2719878719999542,"round_seconds":16.45830801599982,"time_unix":1790001398.968} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":20,"rollout_id":20,"time_unix":1790001398.969} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":20,"stale_groups":0,"time_unix":1790001421.311} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]],[[2]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":4,"rollout_id":20,"time_unix":1790001421.312} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":20,"time_unix":1790001421.312} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0002822875976562,"tis_abs":0.030636589974164963,"tis_clipfrac":0.0023364617954939604,"train_rollout_kl":0.0037764315493404865},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":20,"step_seconds":10.961943236000025,"submitted_groups":null,"time_unix":1790001432.275,"tok_per_s":554.8286347648796,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.0002549216151237488,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.0002549216151237488,"ppo_kl":0.0,"tis":1.0002822875976562,"tis_abs":0.030636589974164963,"tis_clipfrac":0.0023364617954939604,"train_rollout_kl":0.0037764315493404865,"train_rollout_logprob_abs_diff":0.031431008130311966},"train_step":21,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":20,"rollout_id":20,"t":1287.237754423,"time_unix":1790001432.275,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6082,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":20,"resubmitted_groups":0,"rollout_id":20,"t":1287.23802796,"time_unix":1790001432.275,"trained_groups_with_older_tokens":8,"trained_response_tokens":6082,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":20,"time_unix":1790001432.276} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":21,"time_unix":1790001433.162} +{"event":"rl_publication","island_id":0,"policy_version":21,"sync/outer_version":5,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001435.505} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":22.343,"publish":2.344,"sync":0.886,"train":10.964},"rollout_id":20,"time_unix":1790001435.505,"train_fraction":0.3,"train_seconds":10.962,"trained_tokens":6082,"wall_seconds":36.537} +{"event":"rl_round_wall","island_id":0,"rollout_id":20,"rollout_seconds":22.34336089400017,"round_seconds":36.536890000000085,"time_unix":1790001435.506} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":21,"rollout_id":21,"time_unix":1790001435.506} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":21,"stale_groups":25,"time_unix":1790001519.708} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":5,"rollout_id":21,"time_unix":1790001519.708} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":21,"time_unix":1790001519.709} +{"buffer_selected":401,"delta_base_seq":null,"delta_ratio":null,"engine_checksum":null,"event":"rl_policy_released","island_id":0,"local_version":21,"origin":{"kind":"merged","learner_id":0,"local_version":21,"num_learners":2},"policy_version":5,"publish_s":1.083,"release_bytes":1514729467,"release_timings":{"digest_s":0.001,"latest_s":1.309,"prune_s":2.785,"read_latest_s":0.222,"stage_export_s":1.025,"upload_s":19.322,"wait_previous_s":0.0},"released_at":1790001520.506,"seq":6,"sha256":null,"staged":true,"time_unix":1790001524.6,"visible_at":1794246524.6} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0000529289245605,"tis_abs":0.019688282161951065,"tis_clipfrac":0.0007457027095369995,"train_rollout_kl":0.0017585131572559476},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":21,"step_seconds":11.454600320000054,"submitted_groups":null,"time_unix":1790001531.164,"tok_per_s":670.7348825244707,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0004105791449546814,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0004105791449546814,"ppo_kl":0.0,"tis":1.0000529289245605,"tis_abs":0.019688282161951065,"tis_clipfrac":0.0007457027095369995,"train_rollout_kl":0.0017585131572559476,"train_rollout_logprob_abs_diff":0.019799578934907913},"train_step":22,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":21,"rollout_id":21,"t":1386.127179006,"time_unix":1790001531.165,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7683,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":21,"resubmitted_groups":0,"rollout_id":21,"t":1386.127479014,"time_unix":1790001531.165,"trained_groups_with_older_tokens":8,"trained_response_tokens":7683,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":21,"time_unix":1790001531.165} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":22,"time_unix":1790001531.796} +{"event":"rl_publication","island_id":0,"policy_version":22,"sync/outer_version":5,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001532.977} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":84.203,"publish":1.184,"sync":0.629,"train":11.457},"rollout_id":21,"time_unix":1790001532.978,"train_fraction":0.1175,"train_seconds":11.455,"trained_tokens":7683,"wall_seconds":97.472} +{"event":"rl_round_wall","island_id":0,"rollout_id":21,"rollout_seconds":84.20265254700007,"round_seconds":97.4719813690001,"time_unix":1790001532.978} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":22,"rollout_id":22,"time_unix":1790001532.978} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":22,"stale_groups":0,"time_unix":1790001536.11} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":5,"rollout_id":22,"time_unix":1790001536.11} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":22,"time_unix":1790001536.11} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9992305636405945,"tis_abs":0.02393278293311596,"tis_clipfrac":0.0012378062820062041,"train_rollout_kl":0.002530856756493449},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":22,"step_seconds":11.622222701000055,"submitted_groups":null,"time_unix":1790001547.733,"tok_per_s":692.6385947936898,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.000982837751507759,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.000982837751507759,"ppo_kl":0.0,"tis":0.9992305636405945,"tis_abs":0.02393278293311596,"tis_clipfrac":0.0012378062820062041,"train_rollout_kl":0.002530856756493449,"train_rollout_logprob_abs_diff":0.024484820663928986},"train_step":23,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":22,"rollout_id":22,"t":1402.696431693,"time_unix":1790001547.734,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":8050,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":22,"resubmitted_groups":0,"rollout_id":22,"t":1402.696775013,"time_unix":1790001547.734,"trained_groups_with_older_tokens":8,"trained_response_tokens":8050,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":22,"time_unix":1790001547.734} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":23,"time_unix":1790001548.358} +{"event":"rl_publication","island_id":0,"policy_version":23,"sync/outer_version":5,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001549.567} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":3.132,"publish":1.21,"sync":0.623,"train":11.624},"rollout_id":22,"time_unix":1790001549.567,"train_fraction":0.7006,"train_seconds":11.622,"trained_tokens":8050,"wall_seconds":16.59} +{"event":"rl_round_wall","island_id":0,"rollout_id":22,"rollout_seconds":3.1320589979998203,"round_seconds":16.5895507539999,"time_unix":1790001549.568} +{"event":"rl_driver_phase","island_id":0,"phase":"generate","policy_version":23,"rollout_id":23,"time_unix":1790001549.568} +{"event":"rl_buffer_stale_skipped","island_id":0,"max_policy_age":2,"rollout_id":23,"stale_groups":0,"time_unix":1790001559.806} +{"ages":{"2":8},"event":"rl_group_policies","groups":[[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]],[[3]]],"island_id":0,"kind":"external-merged-cut","max_policy_age":2,"merged_version":5,"rollout_id":23,"time_unix":1790001559.806} +{"event":"rl_driver_phase","island_id":0,"phase":"train","rollout_id":23,"time_unix":1790001559.806} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":0,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0004881620407104,"tis_abs":0.026130303740501404,"tis_clipfrac":0.0012666478287428617,"train_rollout_kl":0.0027471627108752728},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":23,"step_seconds":12.557113374999972,"submitted_groups":null,"time_unix":1790001572.364,"tok_per_s":570.5132848655207,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0002743992954492569,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0002743992954492569,"ppo_kl":0.0,"tis":1.0004881620407104,"tis_abs":0.026130303740501404,"tis_clipfrac":0.0012666478287428617,"train_rollout_kl":0.0027471627108752728,"train_rollout_logprob_abs_diff":0.026202265173196793},"train_step":24,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":0,"policy_version":23,"rollout_id":23,"t":1427.327054657,"time_unix":1790001572.364,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7164,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":0,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":23,"resubmitted_groups":0,"rollout_id":23,"t":1427.327354225,"time_unix":1790001572.365,"trained_groups_with_older_tokens":8,"trained_response_tokens":7164,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":0,"phase":"sync","rollout_id":23,"time_unix":1790001572.365} +{"event":"rl_driver_phase","island_id":0,"phase":"publish","policy_version":24,"time_unix":1790001582.206} +{"event":"rl_publication","island_id":0,"policy_version":24,"sync/outer_version":5,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790001583.436} +{"event":"rl_round_phases","island_id":0,"phase_seconds":{"generate":10.238,"publish":1.231,"sync":9.841,"train":12.559},"rollout_id":23,"time_unix":1790001583.437,"train_fraction":0.3708,"train_seconds":12.557,"trained_tokens":7164,"wall_seconds":33.869} +{"event":"rl_round_wall","island_id":0,"rollout_id":23,"rollout_seconds":10.238126640000019,"round_seconds":33.86878006899997,"time_unix":1790001583.437} +{"event":"rl_driver_phase","island_id":0,"phase":"finish","rollout_id":24,"time_unix":1790001583.437} +{"event":"rl_sync_closed","island_id":0,"sync/finished":true,"sync/leave_reason":0,"time_unix":1790001583.453} +{"event":"rl_learner_finalized","island_id":0,"time_unix":1790001584.451} diff --git a/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l1-modal.jsonl b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l1-modal.jsonl new file mode 100644 index 000000000..5ec241fd4 --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l1-modal.jsonl @@ -0,0 +1,126 @@ +{"event":"rl_driver_start","island_id":1,"time_unix":1790000337.347} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":0,"time_unix":1790000393.325} +{"event":"rl_publication","island_id":1,"policy_version":0,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000393.935} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":0,"rollout_id":0,"time_unix":1790000393.936} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":0,"stale_groups":0,"time_unix":1790000443.468} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":0,"time_unix":1790000443.469} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":0,"time_unix":1790000443.469} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9996271729469299,"tis_abs":0.012924110516905785,"tis_clipfrac":0.0,"train_rollout_kl":0.0006594143924303353},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":0,"step_seconds":14.92935143599999,"submitted_groups":null,"time_unix":1790000458.4,"tok_per_s":440.7425217503604,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":1.0468997061252594e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":1.0468997061252594e-05,"ppo_kl":0.0,"tis":0.9996271729469299,"tis_abs":0.012924110516905785,"tis_clipfrac":0.0,"train_rollout_kl":0.0006594143924303353,"train_rollout_logprob_abs_diff":0.012953429482877254},"train_step":1,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":0,"rollout_id":0,"t":315.605752538,"time_unix":1790000458.402,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_tokens":0,"cross_version_truncated_fraction":null,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":0,"resubmitted_groups":0,"rollout_id":0,"t":315.606036434,"time_unix":1790000458.403,"trained_groups_with_older_tokens":0,"trained_response_tokens":6580,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":0,"time_unix":1790000458.403} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":1,"time_unix":1790000459.38} +{"event":"rl_publication","island_id":1,"policy_version":1,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000460.606} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":49.534,"publish":1.838,"sync":0.977,"train":14.934},"rollout_id":0,"time_unix":1790000460.607,"train_fraction":0.2219,"train_seconds":14.929,"trained_tokens":6580,"wall_seconds":67.282} +{"event":"rl_round_wall","island_id":1,"rollout_id":0,"rollout_seconds":49.53351009899998,"round_seconds":66.671405147,"time_unix":1790000460.607} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":1,"rollout_id":1,"time_unix":1790000460.607} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":1,"stale_groups":0,"time_unix":1790000487.84} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":1,"time_unix":1790000487.84} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":1,"time_unix":1790000487.84} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.000192403793335,"tis_abs":0.013526510447263718,"tis_clipfrac":0.0001140510939876549,"train_rollout_kl":0.0007231530616991222},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":1,"step_seconds":10.154640083000004,"submitted_groups":null,"time_unix":1790000497.996,"tok_per_s":738.5786141801159,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.00037271343171596527,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.00037271343171596527,"ppo_kl":0.0,"tis":1.000192403793335,"tis_abs":0.013526510447263718,"tis_clipfrac":0.0001140510939876549,"train_rollout_kl":0.0007231530616991222,"train_rollout_logprob_abs_diff":0.013545182533562183},"train_step":2,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":1,"rollout_id":1,"t":355.199784024,"time_unix":1790000497.996,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7500,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":1,"resubmitted_groups":0,"rollout_id":1,"t":355.200456157,"time_unix":1790000497.997,"trained_groups_with_older_tokens":8,"trained_response_tokens":7500,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":1,"time_unix":1790000497.998} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":2,"time_unix":1790000498.57} +{"event":"rl_publication","island_id":1,"policy_version":2,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000499.222} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":27.233,"publish":0.654,"sync":0.571,"train":10.157},"rollout_id":1,"time_unix":1790000499.222,"train_fraction":0.263,"train_seconds":10.155,"trained_tokens":7500,"wall_seconds":38.616} +{"event":"rl_round_wall","island_id":1,"rollout_id":1,"rollout_seconds":27.233239484999956,"round_seconds":38.615320763,"time_unix":1790000499.223} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":2,"rollout_id":2,"time_unix":1790000499.223} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":2,"stale_groups":0,"time_unix":1790000535.091} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":2,"time_unix":1790000535.092} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":2,"time_unix":1790000535.092} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.999638020992279,"tis_abs":0.01387706771492958,"tis_clipfrac":0.0,"train_rollout_kl":0.0007004652870818973},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":2,"step_seconds":10.184072807000007,"submitted_groups":null,"time_unix":1790000545.277,"tok_per_s":748.3253649523909,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.00023960694670677185,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.00023960694670677185,"ppo_kl":0.0,"tis":0.999638020992279,"tis_abs":0.01387706771492958,"tis_clipfrac":0.0,"train_rollout_kl":0.0007004652870818973,"train_rollout_logprob_abs_diff":0.013955311849713326},"train_step":3,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":2,"rollout_id":2,"t":402.480540254,"time_unix":1790000545.277,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7621,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":2,"resubmitted_groups":0,"rollout_id":2,"t":402.480796709,"time_unix":1790000545.277,"trained_groups_with_older_tokens":8,"trained_response_tokens":7621,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":2,"time_unix":1790000545.278} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":3,"time_unix":1790000545.828} +{"event":"rl_publication","island_id":1,"policy_version":3,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000546.466} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":35.869,"publish":0.64,"sync":0.55,"train":10.186},"rollout_id":2,"time_unix":1790000546.467,"train_fraction":0.2156,"train_seconds":10.184,"trained_tokens":7621,"wall_seconds":47.245} +{"event":"rl_round_wall","island_id":1,"rollout_id":2,"rollout_seconds":35.86920627699999,"round_seconds":47.24427599699999,"time_unix":1790000546.467} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":3,"rollout_id":3,"time_unix":1790000546.467} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":3,"stale_groups":0,"time_unix":1790000570.732} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":3,"time_unix":1790000570.732} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":3,"time_unix":1790000570.733} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0000673532485962,"tis_abs":0.012150060385465622,"tis_clipfrac":0.00012600806076079607,"train_rollout_kl":0.0009349965257570148},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":3,"step_seconds":10.078627583000014,"submitted_groups":null,"time_unix":1790000580.813,"tok_per_s":687.1967381434289,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.000334128737449646,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.000334128737449646,"ppo_kl":0.0,"tis":1.0000673532485962,"tis_abs":0.012150060385465622,"tis_clipfrac":0.00012600806076079607,"train_rollout_kl":0.0009349965257570148,"train_rollout_logprob_abs_diff":0.012544515542685986},"train_step":4,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":3,"rollout_id":3,"t":438.017767848,"time_unix":1790000580.815,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6926,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":3,"resubmitted_groups":0,"rollout_id":3,"t":438.019088813,"time_unix":1790000580.817,"trained_groups_with_older_tokens":8,"trained_response_tokens":6926,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":3,"time_unix":1790000580.819} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":4,"time_unix":1790000582.722} +{"event":"rl_publication","island_id":1,"policy_version":4,"sync/outer_version":0,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000583.357} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":24.265,"publish":0.636,"sync":1.904,"train":10.086},"rollout_id":3,"time_unix":1790000583.357,"train_fraction":0.2732,"train_seconds":10.079,"trained_tokens":6926,"wall_seconds":36.891} +{"event":"rl_round_wall","island_id":1,"rollout_id":3,"rollout_seconds":24.26519715400002,"round_seconds":36.89041670500001,"time_unix":1790000583.358} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":4,"rollout_id":4,"time_unix":1790000583.358} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":4,"stale_groups":0,"time_unix":1790000609.035} +{"ages":{"0":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":0,"rollout_id":4,"time_unix":1790000609.035} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":4,"time_unix":1790000609.036} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9997570514678955,"tis_abs":0.012732021510601044,"tis_clipfrac":0.0001201923078042455,"train_rollout_kl":0.0006828251061961055},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":4,"step_seconds":10.02119946700003,"submitted_groups":null,"time_unix":1790000619.057,"tok_per_s":589.0512427617745,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.00027659162878990173,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.00027659162878990173,"ppo_kl":0.0,"tis":0.9997570514678955,"tis_abs":0.012732021510601044,"tis_clipfrac":0.0001201923078042455,"train_rollout_kl":0.0006828251061961055,"train_rollout_logprob_abs_diff":0.01285580825060606},"train_step":5,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":4,"rollout_id":4,"t":476.261243386,"time_unix":1790000619.058,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":5903,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":4,"resubmitted_groups":0,"rollout_id":4,"t":476.261546661,"time_unix":1790000619.058,"trained_groups_with_older_tokens":8,"trained_response_tokens":5903,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":4,"time_unix":1790000619.058} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":5,"time_unix":1790000619.649} +{"event":"rl_publication","island_id":1,"policy_version":5,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000620.292} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":25.678,"publish":0.644,"sync":0.59,"train":10.023},"rollout_id":4,"time_unix":1790000620.292,"train_fraction":0.2713,"train_seconds":10.021,"trained_tokens":5903,"wall_seconds":36.935} +{"event":"rl_round_wall","island_id":1,"rollout_id":4,"rollout_seconds":25.677611368000044,"round_seconds":36.93449190900003,"time_unix":1790000620.292} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":5,"rollout_id":5,"time_unix":1790000620.293} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":5,"stale_groups":0,"time_unix":1790000642.482} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":5,"time_unix":1790000642.482} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":5,"time_unix":1790000642.483} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0008078813552856,"tis_abs":0.016116918995976448,"tis_clipfrac":0.0,"train_rollout_kl":0.0008473492925986648},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":5,"step_seconds":10.440823053999964,"submitted_groups":null,"time_unix":1790000652.924,"tok_per_s":807.6949447744208,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":0.00014013051986694336,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":0.00014013051986694336,"ppo_kl":0.0,"tis":1.0008078813552856,"tis_abs":0.016116918995976448,"tis_clipfrac":0.0,"train_rollout_kl":0.0008473492925986648,"train_rollout_logprob_abs_diff":0.016113491728901863},"train_step":6,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":5,"rollout_id":5,"t":510.127963615,"time_unix":1790000652.925,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":8433,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":5,"resubmitted_groups":0,"rollout_id":5,"t":510.12822935,"time_unix":1790000652.925,"trained_groups_with_older_tokens":8,"trained_response_tokens":8433,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":5,"time_unix":1790000652.925} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":6,"time_unix":1790000653.473} +{"event":"rl_publication","island_id":1,"policy_version":6,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000654.162} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":22.19,"publish":0.69,"sync":0.548,"train":10.442},"rollout_id":5,"time_unix":1790000654.163,"train_fraction":0.3083,"train_seconds":10.441,"trained_tokens":8433,"wall_seconds":33.87} +{"event":"rl_round_wall","island_id":1,"rollout_id":5,"rollout_seconds":22.18992746699996,"round_seconds":33.87009435199997,"time_unix":1790000654.163} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":6,"rollout_id":6,"time_unix":1790000654.163} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":6,"stale_groups":0,"time_unix":1790000674.578} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":6,"time_unix":1790000674.579} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":6,"time_unix":1790000674.579} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.000024437904358,"tis_abs":0.017042960971593857,"tis_clipfrac":0.00012112403055652976,"train_rollout_kl":0.0009666622499935329},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":6,"step_seconds":10.672416408000004,"submitted_groups":null,"time_unix":1790000685.252,"tok_per_s":688.8786680483126,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":1.9019469618797302e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":1.9019469618797302e-05,"ppo_kl":0.0,"tis":1.000024437904358,"tis_abs":0.017042960971593857,"tis_clipfrac":0.00012112403055652976,"train_rollout_kl":0.0009666622499935329,"train_rollout_logprob_abs_diff":0.017143378034234047},"train_step":7,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":6,"rollout_id":6,"t":542.456228969,"time_unix":1790000685.253,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7352,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":6,"resubmitted_groups":0,"rollout_id":6,"t":542.456520795,"time_unix":1790000685.253,"trained_groups_with_older_tokens":8,"trained_response_tokens":7352,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":6,"time_unix":1790000685.253} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":7,"time_unix":1790000685.284} +{"event":"rl_publication","island_id":1,"policy_version":7,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000685.914} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":20.416,"publish":0.631,"sync":0.031,"train":10.674},"rollout_id":6,"time_unix":1790000685.914,"train_fraction":0.3361,"train_seconds":10.672,"trained_tokens":7352,"wall_seconds":31.752} +{"event":"rl_round_wall","island_id":1,"rollout_id":6,"rollout_seconds":20.41620686899995,"round_seconds":31.751586361999955,"time_unix":1790000685.915} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":7,"rollout_id":7,"time_unix":1790000685.915} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":7,"stale_groups":0,"time_unix":1790000704.736} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":7,"time_unix":1790000704.736} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":7,"time_unix":1790000704.737} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0001304149627686,"tis_abs":0.01615240052342415,"tis_clipfrac":0.0006633272278122604,"train_rollout_kl":0.001174743054434657},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":7,"step_seconds":10.23812003699993,"submitted_groups":null,"time_unix":1790000714.975,"tok_per_s":629.8031256419467,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-4.8685818910598755e-05,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-4.8685818910598755e-05,"ppo_kl":0.0,"tis":1.0001304149627686,"tis_abs":0.01615240052342415,"tis_clipfrac":0.0006633272278122604,"train_rollout_kl":0.001174743054434657,"train_rollout_logprob_abs_diff":0.0161486454308033},"train_step":8,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":7,"rollout_id":7,"t":572.179241925,"time_unix":1790000714.976,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6448,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":7,"resubmitted_groups":0,"rollout_id":7,"t":572.17953623,"time_unix":1790000714.976,"trained_groups_with_older_tokens":8,"trained_response_tokens":6448,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":7,"time_unix":1790000714.976} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":8,"time_unix":1790000715.527} +{"event":"rl_publication","island_id":1,"policy_version":8,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000716.574} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":18.822,"publish":1.048,"sync":0.551,"train":10.24},"rollout_id":7,"time_unix":1790000716.575,"train_fraction":0.3339,"train_seconds":10.238,"trained_tokens":6448,"wall_seconds":30.66} +{"event":"rl_round_wall","island_id":1,"rollout_id":7,"rollout_seconds":18.821808201000067,"round_seconds":30.660152054000037,"time_unix":1790000716.575} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":8,"rollout_id":8,"time_unix":1790000716.575} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":8,"stale_groups":0,"time_unix":1790000737.314} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":8,"time_unix":1790000737.315} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":8,"time_unix":1790000737.315} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":1.0000771284103394,"tis_abs":0.019478876143693924,"tis_clipfrac":0.0012831216445192695,"train_rollout_kl":0.0017102581914514303},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":8,"step_seconds":10.424414142999922,"submitted_groups":null,"time_unix":1790000747.74,"tok_per_s":659.6054133763756,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.0002064388245344162,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.0002064388245344162,"ppo_kl":0.0,"tis":1.0000771284103394,"tis_abs":0.019478876143693924,"tis_clipfrac":0.0012831216445192695,"train_rollout_kl":0.0017102581914514303,"train_rollout_logprob_abs_diff":0.019820991903543472},"train_step":9,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":8,"rollout_id":8,"t":604.943603525,"time_unix":1790000747.74,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":6876,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":8,"resubmitted_groups":0,"rollout_id":8,"t":604.94384868,"time_unix":1790000747.74,"trained_groups_with_older_tokens":8,"trained_response_tokens":6876,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":8,"time_unix":1790000747.741} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":9,"time_unix":1790000747.771} +{"event":"rl_publication","island_id":1,"policy_version":9,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000748.651} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":20.739,"publish":0.881,"sync":0.03,"train":10.426},"rollout_id":8,"time_unix":1790000748.652,"train_fraction":0.325,"train_seconds":10.424,"trained_tokens":6876,"wall_seconds":32.077} +{"event":"rl_round_wall","island_id":1,"rollout_id":8,"rollout_seconds":20.739488746999996,"round_seconds":32.07650234100004,"time_unix":1790000748.652} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":9,"rollout_id":9,"time_unix":1790000748.652} +{"event":"rl_buffer_stale_skipped","island_id":1,"max_policy_age":2,"rollout_id":9,"stale_groups":0,"time_unix":1790000766.982} +{"ages":{"1":8},"event":"rl_group_policies","groups":[[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]],[[0]]],"island_id":1,"kind":"external-merged-cut","max_policy_age":2,"merged_version":1,"rollout_id":9,"time_unix":1790000766.983} +{"event":"rl_driver_phase","island_id":1,"phase":"train","rollout_id":9,"time_unix":1790000766.983} +{"aborted_in_flight_groups":null,"adv_mean":null,"adv_std":null,"applied_lrs":[2e-06],"clip_fraction":0.0,"dynamic_filter_source":{},"event":"rl_round_trained","filtered_samples":null,"island_id":1,"label/config_epoch":0,"masked_fraction":null,"mismatch":{"ois":1.0,"tis":0.9978910684585571,"tis_abs":0.02130867727100849,"tis_clipfrac":0.0008141435100696981,"train_rollout_kl":0.0017039889935404062},"nonzero_advantages":null,"resp_len_mean":null,"resp_len_p95":null,"reward_p10":null,"reward_p50":null,"reward_p90":null,"rollout_id":9,"step_seconds":10.333032317000061,"submitted_groups":null,"time_unix":1790000777.316,"tok_per_s":729.8922299499429,"tool_wait_seconds":null,"train_metrics":{"entropy_loss":0.0,"ess_ratio":1.0,"loss":-0.00036441534757614136,"ois":1.0,"pg_clipfrac":0.0,"pg_loss":-0.00036441534757614136,"ppo_kl":0.0,"tis":0.9978910684585571,"tis_abs":0.02130867727100849,"tis_clipfrac":0.0008141435100696981,"train_rollout_kl":0.0017039889935404062,"train_rollout_logprob_abs_diff":0.02201204001903534},"train_step":10,"trained_groups":8,"trained_samples":32,"truncated_frac":null} +{"config_epoch":0,"counts":{"group_dropped":0,"kept":8,"unknown_version_dropped":0},"dropped_length_edges":[256,1024,4096,16384],"dropped_length_histogram":{"group_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0},"unknown_version_dropped":{"0-255":0,"1024-4095":0,"16384+":0,"256-1023":0,"4096-16383":0}},"event":"offpolicy_gate","island_id":1,"policy_version":9,"rollout_id":9,"t":634.520135007,"time_unix":1790000777.317,"truncation_by_turn":null} +{"carried_in_groups":0,"carried_in_tokens":0,"carried_in_trajectories":0,"config_epoch":0,"cross_version_scored_tokens":0,"cross_version_tokens":7542,"cross_version_truncated_fraction":null,"cross_version_unscored_reasons":{"InvalidURL: nonnumeric port: 'None'":32},"cross_version_unscored_samples":32,"event":"rl_rollout_carry_over","island_id":1,"max_policy_age":2,"over_age_discarded_groups":0,"over_age_discarded_tokens":0,"over_age_discarded_trajectories":0,"policy_version":9,"resubmitted_groups":0,"rollout_id":9,"t":634.520416192,"time_unix":1790000777.317,"trained_groups_with_older_tokens":8,"trained_response_tokens":7542,"unknown_version_discarded_groups":0} +{"event":"rl_driver_phase","island_id":1,"phase":"sync","rollout_id":9,"time_unix":1790000777.317} +{"event":"rl_driver_phase","island_id":1,"phase":"publish","policy_version":10,"time_unix":1790000779.227} +{"event":"rl_publication","island_id":1,"policy_version":10,"sync/outer_version":1,"sync/publication_members":[],"sync/publication_payload_bytes":2384199680,"time_unix":1790000780.043} +{"event":"rl_round_phases","island_id":1,"phase_seconds":{"generate":18.331,"publish":0.817,"sync":1.91,"train":10.334},"rollout_id":9,"time_unix":1790000780.044,"train_fraction":0.3292,"train_seconds":10.333,"trained_tokens":7542,"wall_seconds":31.392} +{"event":"rl_round_wall","island_id":1,"rollout_id":9,"rollout_seconds":18.330657520000045,"round_seconds":31.39194901799999,"time_unix":1790000780.044} +{"event":"rl_driver_phase","island_id":1,"phase":"generate","policy_version":10,"rollout_id":10,"time_unix":1790000780.044} +{"error_type":"KeyboardInterrupt","event":"island_leave","island_id":1,"reason":2,"reason_name":"STOPPED","time_unix":1790000795.35} +{"event":"rl_sync_closed","island_id":1,"sync/finished":false,"sync/leave_reason":2,"time_unix":1790000795.356} diff --git a/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l2-modal.jsonl b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l2-modal.jsonl new file mode 100644 index 000000000..2e9899e83 --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l2-modal.jsonl @@ -0,0 +1,53 @@ +{"data_cursor":null,"event":"rl_sampler_join","groups":0,"incarnations":[],"island_id":0,"latest_release":null,"next_rollout_id":0,"partial_rounds":[],"time_unix":1790000362.618,"written_rounds":0} +{"event":"rl_sampler_start","island_id":0,"max_backlog_groups":112,"num_rollouts":null,"start_rollout_id":0,"time_unix":1790000363.36,"until_trainers_end":2} +{"buffer_end":8,"event":"rl_sampler_round","generate_s":41.953,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":0,"time_unix":1790000406.138,"version_lag":0} +{"buffer_end":16,"event":"rl_sampler_round","generate_s":32.278,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":1,"time_unix":1790000439.258,"version_lag":0} +{"buffer_end":24,"event":"rl_sampler_round","generate_s":29.206,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":2,"time_unix":1790000469.2,"version_lag":0} +{"buffer_selected":15,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.252,"read_latest_s":0.252,"resolve_s":14.361,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.238,"lock_s":0.003,"stamp_s":0.006,"unlock_s":0.003},"policy_version":0,"probe":null,"released_at":1790000441.75,"seen_at":1790000469.729,"seq":1,"served_checksum":null,"sha256":null,"time_unix":1790000484.342} +{"buffer_end":32,"event":"rl_sampler_round","generate_s":31.138,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":3,"time_unix":1790000515.693,"version_lag":0} +{"buffer_end":40,"event":"rl_sampler_round","generate_s":30.995,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":4,"time_unix":1790000547.411,"version_lag":0} +{"buffer_end":48,"event":"rl_sampler_round","generate_s":28.911,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":5,"time_unix":1790000577.082,"version_lag":0} +{"buffer_end":56,"event":"rl_sampler_round","generate_s":31.178,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":6,"time_unix":1790000608.991,"version_lag":0} +{"buffer_end":64,"event":"rl_sampler_round","generate_s":30.678,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":7,"time_unix":1790000640.388,"version_lag":0} +{"buffer_end":72,"event":"rl_sampler_round","generate_s":30.019,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":8,"time_unix":1790000671.116,"version_lag":0} +{"buffer_end":80,"event":"rl_sampler_round","generate_s":29.488,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":9,"time_unix":1790000701.28,"version_lag":0} +{"buffer_end":88,"event":"rl_sampler_round","generate_s":29.855,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":10,"time_unix":1790000731.844,"version_lag":0} +{"buffer_selected":95,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.229,"read_latest_s":0.316,"resolve_s":17.06,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.217,"lock_s":0.003,"stamp_s":0.005,"unlock_s":0.003},"policy_version":1,"probe":null,"released_at":1790000705.215,"seen_at":1790000732.388,"seq":2,"served_checksum":null,"sha256":null,"time_unix":1790000749.677} +{"buffer_end":96,"event":"rl_sampler_round","generate_s":31.294,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":11,"time_unix":1790000781.185,"version_lag":0} +{"buffer_end":104,"event":"rl_sampler_round","generate_s":27.936,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":12,"time_unix":1790000809.883,"version_lag":0} +{"buffer_end":112,"event":"rl_sampler_round","generate_s":30.037,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":13,"time_unix":1790000840.629,"version_lag":0} +{"buffer_end":120,"event":"rl_sampler_round","generate_s":28.257,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":14,"time_unix":1790000869.583,"version_lag":0} +{"buffer_end":128,"event":"rl_sampler_round","generate_s":29.339,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":15,"time_unix":1790000899.706,"version_lag":0} +{"buffer_end":136,"event":"rl_sampler_round","generate_s":28.118,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":16,"time_unix":1790000928.569,"version_lag":0} +{"buffer_end":144,"event":"rl_sampler_round","generate_s":31.768,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":17,"time_unix":1790000961.028,"version_lag":0} +{"buffer_selected":159,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.228,"read_latest_s":0.223,"resolve_s":18.93,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.218,"lock_s":0.002,"stamp_s":0.005,"unlock_s":0.002},"policy_version":2,"probe":null,"released_at":1790000940.46,"seen_at":1790000961.481,"seq":3,"served_checksum":null,"sha256":null,"time_unix":1790000980.64} +{"buffer_end":152,"event":"rl_sampler_round","generate_s":32.526,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":18,"time_unix":1790001013.443,"version_lag":0} +{"buffer_end":160,"event":"rl_sampler_round","generate_s":33.922,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":19,"time_unix":1790001048.143,"version_lag":0} +{"buffer_end":168,"event":"rl_sampler_round","generate_s":34.118,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":20,"time_unix":1790001083.048,"version_lag":0} +{"buffer_end":176,"event":"rl_sampler_round","generate_s":31.279,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":21,"time_unix":1790001115.151,"version_lag":0} +{"buffer_end":184,"event":"rl_sampler_round","generate_s":33.059,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":22,"time_unix":1790001148.898,"version_lag":0} +{"buffer_end":192,"event":"rl_sampler_round","generate_s":31.946,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":23,"time_unix":1790001181.509,"version_lag":0} +{"buffer_end":200,"event":"rl_sampler_round","generate_s":31.315,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":24,"time_unix":1790001213.469,"version_lag":0} +{"buffer_selected":223,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.23,"read_latest_s":0.31,"resolve_s":16.24,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.217,"lock_s":0.003,"stamp_s":0.006,"unlock_s":0.003},"policy_version":3,"probe":null,"released_at":1790001180.429,"seen_at":1790001214.014,"seq":4,"served_checksum":null,"sha256":null,"time_unix":1790001230.485} +{"buffer_end":208,"event":"rl_sampler_round","generate_s":30.55,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":25,"time_unix":1790001261.269,"version_lag":0} +{"buffer_end":216,"event":"rl_sampler_round","generate_s":30.035,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":26,"time_unix":1790001292.088,"version_lag":0} +{"buffer_end":224,"event":"rl_sampler_round","generate_s":32.159,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":27,"time_unix":1790001324.934,"version_lag":0} +{"buffer_end":232,"event":"rl_sampler_round","generate_s":32.291,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":28,"time_unix":1790001357.9,"version_lag":0} +{"backlog_groups":112,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":3,"rollout_id":29,"time_unix":1790001358.611} +{"buffer_selected":287,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.215,"read_latest_s":0.245,"resolve_s":14.524,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.205,"lock_s":0.002,"stamp_s":0.004,"unlock_s":0.002},"policy_version":4,"probe":null,"released_at":1790001360.273,"seen_at":1790001364.085,"seq":5,"served_checksum":null,"sha256":null,"time_unix":1790001378.823} +{"backlog_groups":81,"event":"rl_sampler_admitted","island_id":0,"policy_version":4,"rollout_id":29,"time_unix":1790001379.041,"waited_s":20.43} +{"buffer_end":240,"event":"rl_sampler_round","generate_s":30.719,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":29,"time_unix":1790001409.759,"version_lag":0} +{"buffer_end":248,"event":"rl_sampler_round","generate_s":29.248,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":30,"time_unix":1790001439.704,"version_lag":0} +{"buffer_end":256,"event":"rl_sampler_round","generate_s":30.687,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":31,"time_unix":1790001471.123,"version_lag":0} +{"buffer_end":264,"event":"rl_sampler_round","generate_s":29.124,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":32,"time_unix":1790001500.915,"version_lag":0} +{"backlog_groups":113,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":4,"rollout_id":33,"time_unix":1790001501.58} +{"buffer_selected":401,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.21,"read_latest_s":0.46,"resolve_s":16.879,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.198,"lock_s":0.002,"stamp_s":0.004,"unlock_s":0.003},"policy_version":5,"probe":null,"released_at":1790001520.506,"seen_at":1790001524.378,"seq":6,"served_checksum":null,"sha256":null,"time_unix":1790001541.466} +{"backlog_groups":59,"event":"rl_sampler_admitted","island_id":0,"policy_version":5,"rollout_id":33,"time_unix":1790001541.845,"waited_s":40.265} +{"buffer_end":272,"event":"rl_sampler_round","generate_s":28.11,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":33,"time_unix":1790001569.955,"version_lag":0} +{"buffer_end":280,"event":"rl_sampler_round","generate_s":31.721,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":34,"time_unix":1790001602.388,"version_lag":0} +{"buffer_end":288,"event":"rl_sampler_round","generate_s":31.573,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":35,"time_unix":1790001634.682,"version_lag":0} +{"buffer_end":296,"event":"rl_sampler_round","generate_s":28.715,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":36,"time_unix":1790001664.087,"version_lag":0} +{"buffer_end":304,"event":"rl_sampler_round","generate_s":29.899,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":37,"time_unix":1790001694.627,"version_lag":0} +{"buffer_end":312,"event":"rl_sampler_round","generate_s":28.703,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":38,"time_unix":1790001724.021,"version_lag":0} +{"buffer_end":320,"event":"rl_sampler_round","generate_s":28.671,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":39,"time_unix":1790001753.363,"version_lag":0} +{"backlog_groups":115,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":5,"rollout_id":40,"time_unix":1790001754.078} diff --git a/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l3-modal.jsonl b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l3-modal.jsonl new file mode 100644 index 000000000..a5cce18ae --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/p7/events/run-l3-modal.jsonl @@ -0,0 +1,52 @@ +{"data_cursor":null,"event":"rl_sampler_join","groups":0,"incarnations":[],"island_id":0,"latest_release":null,"next_rollout_id":0,"partial_rounds":[],"time_unix":1790000420.737,"written_rounds":0} +{"event":"rl_sampler_start","island_id":0,"max_backlog_groups":112,"num_rollouts":null,"start_rollout_id":0,"time_unix":1790000421.673,"until_trainers_end":2} +{"buffer_end":8,"event":"rl_sampler_round","generate_s":43.71,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":0,"time_unix":1790000466.232,"version_lag":0} +{"buffer_selected":15,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.193,"read_latest_s":0.252,"resolve_s":19.736,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.181,"lock_s":0.002,"stamp_s":0.005,"unlock_s":0.002},"policy_version":0,"probe":null,"released_at":1790000441.75,"seen_at":1790000466.804,"seq":1,"served_checksum":null,"sha256":null,"time_unix":1790000486.733} +{"buffer_end":16,"event":"rl_sampler_round","generate_s":29.84,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":1,"time_unix":1790000516.839,"version_lag":0} +{"buffer_end":24,"event":"rl_sampler_round","generate_s":32.356,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":2,"time_unix":1790000549.889,"version_lag":0} +{"buffer_end":32,"event":"rl_sampler_round","generate_s":32.047,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":3,"time_unix":1790000582.65,"version_lag":0} +{"buffer_end":40,"event":"rl_sampler_round","generate_s":29.498,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":4,"time_unix":1790000612.831,"version_lag":0} +{"buffer_end":48,"event":"rl_sampler_round","generate_s":28.612,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":5,"time_unix":1790000642.135,"version_lag":0} +{"buffer_end":56,"event":"rl_sampler_round","generate_s":29.57,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":6,"time_unix":1790000672.411,"version_lag":0} +{"buffer_end":64,"event":"rl_sampler_round","generate_s":28.247,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":7,"time_unix":1790000701.362,"version_lag":0} +{"buffer_end":72,"event":"rl_sampler_round","generate_s":31.349,"groups":8,"island_id":0,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":1,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":8,"time_unix":1790000733.445,"version_lag":0} +{"buffer_selected":95,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.216,"read_latest_s":0.217,"resolve_s":15.564,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.204,"lock_s":0.002,"stamp_s":0.005,"unlock_s":0.002},"policy_version":1,"probe":null,"released_at":1790000705.215,"seen_at":1790000733.885,"seq":2,"served_checksum":null,"sha256":null,"time_unix":1790000749.666} +{"buffer_end":80,"event":"rl_sampler_round","generate_s":30.594,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":9,"time_unix":1790000780.477,"version_lag":0} +{"buffer_end":88,"event":"rl_sampler_round","generate_s":30.218,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":10,"time_unix":1790000811.368,"version_lag":0} +{"buffer_end":96,"event":"rl_sampler_round","generate_s":29.102,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":11,"time_unix":1790000841.155,"version_lag":0} +{"buffer_end":104,"event":"rl_sampler_round","generate_s":27.869,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":12,"time_unix":1790000869.771,"version_lag":0} +{"buffer_end":112,"event":"rl_sampler_round","generate_s":29.733,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":13,"time_unix":1790000900.178,"version_lag":0} +{"buffer_end":120,"event":"rl_sampler_round","generate_s":30.778,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":14,"time_unix":1790000931.664,"version_lag":0} +{"buffer_end":128,"event":"rl_sampler_round","generate_s":27.122,"groups":8,"island_id":0,"policy_version":1,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":2,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":15,"time_unix":1790000959.484,"version_lag":0} +{"buffer_selected":159,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.196,"read_latest_s":0.235,"resolve_s":16.097,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.183,"lock_s":0.002,"stamp_s":0.006,"unlock_s":0.003},"policy_version":2,"probe":null,"released_at":1790000940.46,"seen_at":1790000960.015,"seq":3,"served_checksum":null,"sha256":null,"time_unix":1790000976.308} +{"buffer_end":136,"event":"rl_sampler_round","generate_s":29.277,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":16,"time_unix":1790001005.799,"version_lag":0} +{"buffer_end":144,"event":"rl_sampler_round","generate_s":30.761,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":17,"time_unix":1790001037.295,"version_lag":0} +{"buffer_end":152,"event":"rl_sampler_round","generate_s":30.846,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":18,"time_unix":1790001068.898,"version_lag":0} +{"buffer_end":160,"event":"rl_sampler_round","generate_s":27.457,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":19,"time_unix":1790001097.019,"version_lag":0} +{"buffer_end":168,"event":"rl_sampler_round","generate_s":28.667,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":20,"time_unix":1790001126.358,"version_lag":0} +{"buffer_end":176,"event":"rl_sampler_round","generate_s":27.77,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":21,"time_unix":1790001154.772,"version_lag":0} +{"buffer_end":184,"event":"rl_sampler_round","generate_s":29.494,"groups":8,"island_id":0,"policy_version":2,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":3,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":22,"time_unix":1790001184.941,"version_lag":0} +{"buffer_selected":223,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.198,"read_latest_s":0.439,"resolve_s":15.491,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.186,"lock_s":0.003,"stamp_s":0.005,"unlock_s":0.002},"policy_version":3,"probe":null,"released_at":1790001180.429,"seen_at":1790001185.601,"seq":4,"served_checksum":null,"sha256":null,"time_unix":1790001201.29} +{"buffer_end":192,"event":"rl_sampler_round","generate_s":29.745,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":23,"time_unix":1790001231.248,"version_lag":0} +{"buffer_end":200,"event":"rl_sampler_round","generate_s":32.378,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":24,"time_unix":1790001264.341,"version_lag":0} +{"buffer_end":208,"event":"rl_sampler_round","generate_s":27.933,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":25,"time_unix":1790001292.955,"version_lag":0} +{"buffer_end":216,"event":"rl_sampler_round","generate_s":28.381,"groups":8,"island_id":0,"policy_version":3,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":4,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":26,"time_unix":1790001322.048,"version_lag":0} +{"backlog_groups":113,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":3,"rollout_id":27,"time_unix":1790001322.722} +{"buffer_selected":287,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.286,"read_latest_s":0.479,"resolve_s":16.077,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.267,"lock_s":0.003,"stamp_s":0.008,"unlock_s":0.005},"policy_version":4,"probe":null,"released_at":1790001360.273,"seen_at":1790001362.538,"seq":5,"served_checksum":null,"sha256":null,"time_unix":1790001378.901} +{"backlog_groups":80,"event":"rl_sampler_admitted","island_id":0,"policy_version":4,"rollout_id":27,"time_unix":1790001379.118,"waited_s":56.396} +{"buffer_end":224,"event":"rl_sampler_round","generate_s":29.097,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":27,"time_unix":1790001408.215,"version_lag":0} +{"buffer_end":232,"event":"rl_sampler_round","generate_s":30.772,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":28,"time_unix":1790001439.706,"version_lag":0} +{"buffer_end":240,"event":"rl_sampler_round","generate_s":27.821,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":29,"time_unix":1790001468.169,"version_lag":0} +{"buffer_end":248,"event":"rl_sampler_round","generate_s":27.977,"groups":8,"island_id":0,"policy_version":4,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":5,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":30,"time_unix":1790001496.814,"version_lag":0} +{"backlog_groups":112,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":4,"rollout_id":31,"time_unix":1790001497.531} +{"buffer_selected":401,"event":"rl_sampler_policy_loaded","follower_timings":{"load_s":0.197,"read_latest_s":0.308,"resolve_s":15.514,"verify_s":0.0},"fused_stage":false,"island_id":0,"load_timings":{"engine_load_s":0.184,"lock_s":0.002,"stamp_s":0.005,"unlock_s":0.002},"policy_version":5,"probe":null,"released_at":1790001520.506,"seen_at":1790001525.735,"seq":6,"served_checksum":null,"sha256":null,"time_unix":1790001541.446} +{"backlog_groups":52,"event":"rl_sampler_admitted","island_id":0,"policy_version":5,"rollout_id":31,"time_unix":1790001541.67,"waited_s":44.138} +{"buffer_end":256,"event":"rl_sampler_round","generate_s":27.154,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":31,"time_unix":1790001568.824,"version_lag":0} +{"buffer_end":264,"event":"rl_sampler_round","generate_s":29.188,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":32,"time_unix":1790001598.734,"version_lag":0} +{"buffer_end":272,"event":"rl_sampler_round","generate_s":30.149,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":33,"time_unix":1790001629.531,"version_lag":0} +{"buffer_end":280,"event":"rl_sampler_round","generate_s":26.312,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":34,"time_unix":1790001656.543,"version_lag":0} +{"buffer_end":288,"event":"rl_sampler_round","generate_s":28.809,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":35,"time_unix":1790001686.114,"version_lag":0} +{"buffer_end":296,"event":"rl_sampler_round","generate_s":27.696,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":36,"time_unix":1790001714.504,"version_lag":0} +{"buffer_end":304,"event":"rl_sampler_round","generate_s":29.274,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":37,"time_unix":1790001744.471,"version_lag":0} +{"buffer_end":312,"event":"rl_sampler_round","generate_s":26.554,"groups":8,"island_id":0,"policy_version":5,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":6,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":38,"time_unix":1790001771.741,"version_lag":0} +{"backlog_groups":116,"event":"rl_sampler_backlog_full","island_id":0,"max_backlog_groups":112,"policy_version":5,"rollout_id":39,"time_unix":1790001772.408} diff --git a/tests/fixtures/dashboard/async_grpo_replay/p7/fleet.jsonl b/tests/fixtures/dashboard/async_grpo_replay/p7/fleet.jsonl new file mode 100644 index 000000000..adac83590 --- /dev/null +++ b/tests/fixtures/dashboard/async_grpo_replay/p7/fleet.jsonl @@ -0,0 +1,11 @@ +{"cpus":4,"event":"island_ready","gpus":1,"initial":true,"island":"run-l0-modal","island_id":0,"memory_gib":32.0,"price_key":null,"region":null,"time_unix":1790000141.64} +{"cpus":4,"event":"island_ready","gpus":1,"initial":true,"island":"run-l1-modal","island_id":1,"memory_gib":32.0,"price_key":null,"region":null,"time_unix":1790000141.641} +{"cpus":4,"event":"island_ready","gpus":1,"initial":true,"island":"run-l2-modal","island_id":2,"memory_gib":32.0,"price_key":null,"region":null,"time_unix":1790000141.642} +{"cpus":4,"event":"island_ready","gpus":1,"initial":true,"island":"run-l3-modal","island_id":3,"memory_gib":32.0,"price_key":null,"region":null,"time_unix":1790000141.642} +{"event":"cost_tick","time_unix":1790000141.643} +{"event":"cost_tick","time_unix":1790000468.748} +{"event":"cost_tick","time_unix":1790000795.793} +{"event":"cost_tick","time_unix":1790001124.38} +{"cpus":4,"event":"island_done","gpus":1,"island":"run-l1-modal","island_id":1,"memory_gib":32.0,"price_key":null,"region":null,"time_unix":1790001847.828} +{"event":"cost_tick","time_unix":1790001880.889} +{"event":"cost_tick","time_unix":1790002187.221} diff --git a/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl b/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl new file mode 100644 index 000000000..521d2b88d --- /dev/null +++ b/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl @@ -0,0 +1,70 @@ +{"base_dir":null,"base_version":0,"event":"rl_sampler_start_on_latest","sampler_id":"run-1","time_unix":1790001912.955} +{"event":"rl_dispense_recover","recovered":0,"released":0,"sampler_id":"run-1","time_unix":1790001915.764} +{"available":false,"event":"rl_resource_sample","sampler_id":"run-1","t":9766.977634535,"time_unix":1790001915.796} +{"allow_penalties":false,"event":"rl_sampler_start","loop_guard":null,"max_backlog_groups":32,"max_prompt_tokens":2048,"num_rollouts":1000000,"provenance":{"sampler_id":"run-1"},"sampler_id":"run-1","sampling":{"allow_penalties":false,"frequency_penalty":0.0,"max_tokens":8192,"n":8,"presence_penalty":0.0,"repetition_penalty":1.0,"seed":null,"stop":[],"temperature":1.0,"top_k":-1,"top_p":1.0},"time_unix":1790001915.797} +{"data_cursor":null,"event":"rl_sampler_join","groups":0,"incarnations":[],"latest_release":null,"next_rollout_id":0,"partial_rounds":[],"sampler_id":"run-1","time_unix":1790002235.186,"written_rounds":0} +{"attempt":0,"event":"rl_dispense_claim","row":0,"sampler_id":"run-1","slot":0,"stream":0,"time_unix":1790002237.888} +{"attempt":0,"event":"rl_dispense_claim","row":1,"sampler_id":"run-1","slot":1,"stream":0,"time_unix":1790002238.588} +{"attempt":0,"event":"rl_dispense_claim","row":2,"sampler_id":"run-1","slot":2,"stream":0,"time_unix":1790002239.401} +{"attempt":0,"event":"rl_dispense_claim","row":3,"sampler_id":"run-1","slot":3,"stream":0,"time_unix":1790002240.028} +{"attempt":0,"event":"rl_dispense_claim","row":4,"sampler_id":"run-1","slot":4,"stream":0,"time_unix":1790002241.208} +{"attempt":0,"event":"rl_dispense_claim","row":5,"sampler_id":"run-1","slot":5,"stream":0,"time_unix":1790002241.892} +{"attempt":0,"event":"rl_dispense_claim","row":6,"sampler_id":"run-1","slot":6,"stream":0,"time_unix":1790002242.643} +{"attempt":0,"event":"rl_dispense_claim","row":7,"sampler_id":"run-1","slot":7,"stream":0,"time_unix":1790002243.636} +{"attempt":0,"elapsed_s":380.822,"event":"rl_dispense_settle","position":0,"row":0,"sampler_id":"run-1","slot":0,"time_unix":1790002619.69} +{"attempt":0,"elapsed_s":381.102,"event":"rl_dispense_settle","position":1,"row":1,"sampler_id":"run-1","slot":1,"time_unix":1790002620.351} +{"attempt":0,"elapsed_s":380.95,"event":"rl_dispense_settle","position":2,"row":2,"sampler_id":"run-1","slot":2,"time_unix":1790002620.989} +{"attempt":0,"elapsed_s":380.962,"event":"rl_dispense_settle","position":3,"row":3,"sampler_id":"run-1","slot":3,"time_unix":1790002621.622} +{"attempt":0,"elapsed_s":380.414,"event":"rl_dispense_settle","position":4,"row":4,"sampler_id":"run-1","slot":4,"time_unix":1790002622.326} +{"attempt":0,"elapsed_s":380.434,"event":"rl_dispense_settle","position":5,"row":5,"sampler_id":"run-1","slot":5,"time_unix":1790002623.042} +{"attempt":0,"elapsed_s":380.399,"event":"rl_dispense_settle","position":6,"row":6,"sampler_id":"run-1","slot":6,"time_unix":1790002623.703} +{"attempt":0,"elapsed_s":380.067,"event":"rl_dispense_settle","position":7,"row":7,"sampler_id":"run-1","slot":7,"time_unix":1790002624.45} +{"acked":true,"event":"rl_dispense_written","released":0,"requeued":0,"rollout_id":0,"sampler_id":"run-1","tasks":[{"attempt":0,"elapsed_s":380.822,"position":0,"row":0,"slot":0},{"attempt":0,"elapsed_s":381.102,"position":1,"row":1,"slot":1},{"attempt":0,"elapsed_s":380.95,"position":2,"row":2,"slot":2},{"attempt":0,"elapsed_s":380.962,"position":3,"row":3,"slot":3},{"attempt":0,"elapsed_s":380.414,"position":4,"row":4,"slot":4},{"attempt":0,"elapsed_s":380.434,"position":5,"row":5,"slot":5},{"attempt":0,"elapsed_s":380.399,"position":6,"row":6,"slot":6},{"attempt":0,"elapsed_s":380.067,"position":7,"row":7,"slot":7}],"time_unix":1790002624.45,"trained":8} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"stop":64},"groups":8,"policy_version":0,"response_tokens":172231,"rollout_id":0,"sampler_id":"run-1","samples":64,"time_unix":1790002624.45} +{"buffer_end":8,"event":"rl_sampler_round","generate_s":387.779,"groups":8,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":0,"sampler_id":"run-1","time_unix":1790002624.657,"version_lag":0} +{"attempt":0,"event":"rl_dispense_claim","row":8,"sampler_id":"run-1","slot":8,"stream":0,"time_unix":1790002626.772} +{"attempt":0,"event":"rl_dispense_claim","row":9,"sampler_id":"run-1","slot":9,"stream":0,"time_unix":1790002628.172} +{"attempt":0,"event":"rl_dispense_claim","row":10,"sampler_id":"run-1","slot":10,"stream":0,"time_unix":1790002628.904} +{"attempt":0,"event":"rl_dispense_claim","row":11,"sampler_id":"run-1","slot":11,"stream":0,"time_unix":1790002629.556} +{"attempt":0,"event":"rl_dispense_claim","row":12,"sampler_id":"run-1","slot":12,"stream":0,"time_unix":1790002630.248} +{"attempt":0,"event":"rl_dispense_claim","row":13,"sampler_id":"run-1","slot":13,"stream":0,"time_unix":1790002630.864} +{"attempt":0,"event":"rl_dispense_claim","row":14,"sampler_id":"run-1","slot":14,"stream":0,"time_unix":1790002631.529} +{"attempt":0,"event":"rl_dispense_claim","row":15,"sampler_id":"run-1","slot":15,"stream":0,"time_unix":1790002632.217} +{"attempt":0,"elapsed_s":652.169,"event":"rl_dispense_settle","position":8,"row":8,"sampler_id":"run-1","slot":8,"time_unix":1790003279.734} +{"attempt":0,"elapsed_s":651.563,"event":"rl_dispense_settle","position":9,"row":9,"sampler_id":"run-1","slot":9,"time_unix":1790003280.472} +{"attempt":0,"elapsed_s":651.569,"event":"rl_dispense_settle","position":10,"row":10,"sampler_id":"run-1","slot":10,"time_unix":1790003281.162} +{"attempt":0,"elapsed_s":651.607,"event":"rl_dispense_settle","position":11,"row":11,"sampler_id":"run-1","slot":11,"time_unix":1790003282.0} +{"attempt":0,"elapsed_s":651.752,"event":"rl_dispense_settle","position":12,"row":12,"sampler_id":"run-1","slot":12,"time_unix":1790003283.06} +{"attempt":0,"elapsed_s":652.196,"event":"rl_dispense_settle","position":13,"row":13,"sampler_id":"run-1","slot":13,"time_unix":1790003283.724} +{"attempt":0,"elapsed_s":652.195,"event":"rl_dispense_settle","position":14,"row":14,"sampler_id":"run-1","slot":14,"time_unix":1790003284.413} +{"attempt":0,"elapsed_s":652.196,"event":"rl_dispense_settle","position":15,"row":15,"sampler_id":"run-1","slot":15,"time_unix":1790003285.48} +{"acked":true,"event":"rl_dispense_written","released":0,"requeued":0,"rollout_id":1,"sampler_id":"run-1","tasks":[{"attempt":0,"elapsed_s":652.169,"position":8,"row":8,"slot":8},{"attempt":0,"elapsed_s":651.563,"position":9,"row":9,"slot":9},{"attempt":0,"elapsed_s":651.569,"position":10,"row":10,"slot":10},{"attempt":0,"elapsed_s":651.607,"position":11,"row":11,"slot":11},{"attempt":0,"elapsed_s":651.752,"position":12,"row":12,"slot":12},{"attempt":0,"elapsed_s":652.196,"position":13,"row":13,"slot":13},{"attempt":0,"elapsed_s":652.195,"position":14,"row":14,"slot":14},{"attempt":0,"elapsed_s":652.196,"position":15,"row":15,"slot":15}],"time_unix":1790003285.48,"trained":8} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"length":18,"stop":46},"groups":8,"policy_version":0,"response_tokens":333359,"rollout_id":1,"sampler_id":"run-1","samples":64,"time_unix":1790003285.48} +{"buffer_end":16,"event":"rl_sampler_round","generate_s":660.087,"groups":8,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":1,"sampler_id":"run-1","time_unix":1790003285.716,"version_lag":0} +{"attempt":0,"event":"rl_dispense_claim","row":16,"sampler_id":"run-1","slot":16,"stream":0,"time_unix":1790003287.853} +{"attempt":0,"event":"rl_dispense_claim","row":17,"sampler_id":"run-1","slot":17,"stream":0,"time_unix":1790003289.094} +{"attempt":0,"event":"rl_dispense_claim","row":18,"sampler_id":"run-1","slot":18,"stream":0,"time_unix":1790003289.848} +{"attempt":0,"event":"rl_dispense_claim","row":19,"sampler_id":"run-1","slot":19,"stream":0,"time_unix":1790003290.618} +{"attempt":0,"event":"rl_dispense_claim","row":20,"sampler_id":"run-1","slot":20,"stream":0,"time_unix":1790003291.244} +{"attempt":0,"event":"rl_dispense_claim","row":21,"sampler_id":"run-1","slot":21,"stream":0,"time_unix":1790003291.875} +{"attempt":0,"event":"rl_dispense_claim","row":22,"sampler_id":"run-1","slot":22,"stream":0,"time_unix":1790003292.556} +{"attempt":0,"event":"rl_dispense_claim","row":23,"sampler_id":"run-1","slot":23,"stream":0,"time_unix":1790003293.309} +{"attempt":0,"elapsed_s":529.251,"event":"rl_dispense_settle","position":16,"row":16,"sampler_id":"run-1","slot":16,"time_unix":1790003818.002} +{"attempt":0,"elapsed_s":528.908,"event":"rl_dispense_settle","position":17,"row":17,"sampler_id":"run-1","slot":17,"time_unix":1790003818.734} +{"attempt":0,"elapsed_s":528.886,"event":"rl_dispense_settle","position":18,"row":18,"sampler_id":"run-1","slot":18,"time_unix":1790003819.486} +{"attempt":0,"elapsed_s":528.868,"event":"rl_dispense_settle","position":19,"row":19,"sampler_id":"run-1","slot":19,"time_unix":1790003820.19} +{"attempt":0,"elapsed_s":528.946,"event":"rl_dispense_settle","position":20,"row":20,"sampler_id":"run-1","slot":20,"time_unix":1790003821.041} +{"attempt":0,"elapsed_s":529.166,"event":"rl_dispense_settle","position":21,"row":21,"sampler_id":"run-1","slot":21,"time_unix":1790003822.475} +{"attempt":0,"elapsed_s":529.919,"event":"rl_dispense_settle","position":22,"row":22,"sampler_id":"run-1","slot":22,"time_unix":1790003823.616} +{"attempt":0,"elapsed_s":530.308,"event":"rl_dispense_settle","position":23,"row":23,"sampler_id":"run-1","slot":23,"time_unix":1790003824.27} +{"acked":true,"event":"rl_dispense_written","released":0,"requeued":0,"rollout_id":2,"sampler_id":"run-1","tasks":[{"attempt":0,"elapsed_s":529.251,"position":16,"row":16,"slot":16},{"attempt":0,"elapsed_s":528.908,"position":17,"row":17,"slot":17},{"attempt":0,"elapsed_s":528.886,"position":18,"row":18,"slot":18},{"attempt":0,"elapsed_s":528.868,"position":19,"row":19,"slot":19},{"attempt":0,"elapsed_s":528.946,"position":20,"row":20,"slot":20},{"attempt":0,"elapsed_s":529.166,"position":21,"row":21,"slot":21},{"attempt":0,"elapsed_s":529.919,"position":22,"row":22,"slot":22},{"attempt":0,"elapsed_s":530.308,"position":23,"row":23,"slot":23}],"time_unix":1790003824.27,"trained":8} +{"dropped_groups":0,"event":"rl_sampler_engine_round","finish":{"length":20,"stop":44},"groups":8,"policy_version":0,"response_tokens":268410,"rollout_id":2,"sampler_id":"run-1","samples":64,"time_unix":1790003824.271} +{"buffer_end":24,"event":"rl_sampler_round","generate_s":537.611,"groups":8,"policy_version":0,"releases":{"deferred_in_flight":0,"lineage_resets":0,"loaded":0,"rebuild_failures":0,"stale_reads":0,"superseded":0},"rollout_id":2,"sampler_id":"run-1","time_unix":1790003824.524,"version_lag":0} +{"attempt":0,"event":"rl_dispense_claim","row":24,"sampler_id":"run-1","slot":24,"stream":0,"time_unix":1790003826.6} +{"attempt":0,"event":"rl_dispense_claim","row":25,"sampler_id":"run-1","slot":25,"stream":0,"time_unix":1790003827.266} +{"attempt":0,"event":"rl_dispense_claim","row":26,"sampler_id":"run-1","slot":26,"stream":0,"time_unix":1790003827.987} +{"attempt":0,"event":"rl_dispense_claim","row":27,"sampler_id":"run-1","slot":27,"stream":0,"time_unix":1790003828.769} +{"attempt":0,"event":"rl_dispense_claim","row":28,"sampler_id":"run-1","slot":28,"stream":0,"time_unix":1790003829.446} +{"attempt":0,"event":"rl_dispense_claim","row":29,"sampler_id":"run-1","slot":29,"stream":0,"time_unix":1790003830.125} +{"attempt":0,"event":"rl_dispense_claim","row":30,"sampler_id":"run-1","slot":30,"stream":0,"time_unix":1790003831.555} +{"attempt":0,"event":"rl_dispense_claim","row":31,"sampler_id":"run-1","slot":31,"stream":0,"time_unix":1790003832.219} diff --git a/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl.source.jsonl b/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl.source.jsonl new file mode 100644 index 000000000..da33e1238 --- /dev/null +++ b/tests/fixtures/dashboard/lora_ended/mirror/nc-sampler-1.jsonl.source.jsonl @@ -0,0 +1 @@ +{"error":"Connection to 192.0.2.1 closed by remote host.","event":"dashboard_source_lost","island_id":"nc-sampler-1","offset":21482,"time_unix":1790004025.976} diff --git a/tests/fixtures/dashboard/lora_ended/pool/events-run.jsonl b/tests/fixtures/dashboard/lora_ended/pool/events-run.jsonl new file mode 100644 index 000000000..15df0c7d2 --- /dev/null +++ b/tests/fixtures/dashboard/lora_ended/pool/events-run.jsonl @@ -0,0 +1,6 @@ +{"account":5,"continues_shard":null,"data_slot":null,"event":"nc_elastic_ordered","replaces":null,"shard":"run-0","t":1790000352.001,"ticket":"tk1"} +{"attempt":1,"detail":"sampler exited exit=2 before join","event":"nc_elastic_sampler_restart","shard":"run-0","t":1790001495.553,"ticket":"tk1"} +{"detail":"sampler exited exit=2 before join","event":"nc_elastic_sampler_crash_loop","restarts":1,"shard":"run-0","t":1790001583.95,"ticket":"tk1"} +{"event":"nc_elastic_shard_fenced","shard":"run-0","t":1790001585.42,"ticket":"tk1","why":"sampler crash loop"} +{"attempt":1,"detail":"sampler exited exit=2 before join","event":"nc_elastic_sampler_restart","shard":"run-1","t":1790001740.626,"ticket":"tk1"} +{"account":5,"continues_shard":null,"event":"nc_elastic_member_ready","grant_to_ready_s":1865,"order_to_ready_s":769,"restarts":0,"shard":"run-1","t":1790002267.316,"ticket":"tk1"} diff --git a/tests/fixtures/dashboard/lora_ended/run/fleet.jsonl b/tests/fixtures/dashboard/lora_ended/run/fleet.jsonl new file mode 100644 index 000000000..834b8fd50 --- /dev/null +++ b/tests/fixtures/dashboard/lora_ended/run/fleet.jsonl @@ -0,0 +1,5 @@ +{"cpus":32,"event":"island_ready","gpus":8,"initial":true,"island":"run-l0-modal","island_id":0,"memory_gib":256.0,"price_key":null,"region":null,"time_unix":1790002561.28} +{"event":"cost_tick","time_unix":1790002561.283} +{"event":"cost_tick","time_unix":1790002867.908} +{"cpus":32,"event":"island_ready","gpus":8,"initial":true,"island":"run-l0-modal","island_id":0,"memory_gib":256.0,"price_key":null,"region":null,"time_unix":1790003134.189} +{"event":"cost_tick","time_unix":1790003134.189} diff --git a/tests/fixtures/dashboard/lora_ended/run/teardown.json b/tests/fixtures/dashboard/lora_ended/run/teardown.json new file mode 100644 index 000000000..3ba9e9d64 --- /dev/null +++ b/tests/fixtures/dashboard/lora_ended/run/teardown.json @@ -0,0 +1 @@ +{"provider": "modal", "confirmed_stopped": false} diff --git a/tests/golden/decoupling/decoupled.json b/tests/golden/decoupling/decoupled.json index 003ef5a81..3544ba667 100644 --- a/tests/golden/decoupling/decoupled.json +++ b/tests/golden/decoupling/decoupled.json @@ -155,7 +155,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -199,7 +199,7 @@ "10", "--group-query-attention" ], - "miles_argv_sha256": "6ff6da47c19cf9f9eac0ec0b9040e9e199220578244928625c5bec362379ccf6", + "miles_argv_sha256": "ff738cd485fb4d5191a55a91d9d46c488443854cbfbf0a32685d7b2c31a33f5a", "name": "decoupled", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/drgrpo.json b/tests/golden/decoupling/drgrpo.json index 063db0178..4d05be949 100644 --- a/tests/golden/decoupling/drgrpo.json +++ b/tests/golden/decoupling/drgrpo.json @@ -202,7 +202,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -249,7 +249,7 @@ "10", "--group-query-attention" ], - "miles_argv_sha256": "8545800520cd18d7ee951fbb8a06d012cbb6bbd39d70968e85f35c1b05d2c6e7", + "miles_argv_sha256": "7da9d36df6bd1baaacc20e92030467993f6250a47405d873476a64cf10b742eb", "name": "drgrpo", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/elastic.json b/tests/golden/decoupling/elastic.json index c1f30a3da..a91ec6100 100644 --- a/tests/golden/decoupling/elastic.json +++ b/tests/golden/decoupling/elastic.json @@ -158,7 +158,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -200,12 +200,13 @@ "--apply-chat-template", "--distributed-timeout-minutes", "10", - "--use-miles-router", - "--update-weight-group-timeout-s", + "--router", + "miles", + "--update-weight-engine-request-timeout", "120", "--group-query-attention" ], - "miles_argv_sha256": "5aeca40e53d9f1b1a614a6a6ec814e5d037b354da3f448f04c1d10417751d135", + "miles_argv_sha256": "86b77438b77681b0d135744e22e4877e6a6dab3b00f3923f7bfc64adb054bca7", "name": "elastic", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/fake_engine_tapes.json b/tests/golden/decoupling/fake_engine_tapes.json index 4c5791928..af9cf6cf5 100644 --- a/tests/golden/decoupling/fake_engine_tapes.json +++ b/tests/golden/decoupling/fake_engine_tapes.json @@ -258,7 +258,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -293,7 +293,7 @@ "rl/group_p50_seconds": "", "rl/group_p95_seconds": "", "rl/group_p99_seconds": "", - "rl/local_delta_norm": 0.0, + "rl/local_delta_norm": null, "rl/mixed_version_group_count": 0, "rl/policy_hash": "bb33e8048442a640a8964d4daa420b126b01271eb30abdd70eeb8b3159c5de1b", "rl/reward_mean": 0.5, @@ -722,7 +722,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -757,7 +757,7 @@ "rl/group_p50_seconds": "", "rl/group_p95_seconds": "", "rl/group_p99_seconds": "", - "rl/local_delta_norm": 0.0, + "rl/local_delta_norm": null, "rl/mixed_version_group_count": 0, "rl/policy_hash": "ae0f94bd3145f02b37ccf9248280d9cb87718dc21475c29a8408799c367617ad", "rl/reward_mean": 0.5, @@ -949,7 +949,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -984,7 +984,7 @@ "rl/group_p50_seconds": "", "rl/group_p95_seconds": "", "rl/group_p99_seconds": "", - "rl/local_delta_norm": 0.0, + "rl/local_delta_norm": null, "rl/mixed_version_group_count": 0, "rl/policy_hash": "ae0f94bd3145f02b37ccf9248280d9cb87718dc21475c29a8408799c367617ad", "rl/reward_mean": 0.5, @@ -1520,7 +1520,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -1655,7 +1655,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, @@ -1805,7 +1805,7 @@ "clip_fraction": null, "completed_groups": 2, "completed_trajectories": 4, - "delta_l2_norm": 0.0, + "delta_l2_norm": null, "dynamic_filter_dropped_groups": 0, "dynamic_filter_generated_groups": 0, "dynamic_filter_replacement_attempts": 0, diff --git a/tests/golden/decoupling/fn_2x8.json b/tests/golden/decoupling/fn_2x8.json index 7b0734f37..51569154e 100644 --- a/tests/golden/decoupling/fn_2x8.json +++ b/tests/golden/decoupling/fn_2x8.json @@ -184,7 +184,7 @@ "--global-batch-size", "64", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "8192", "--rollout-max-response-len", "4096", @@ -220,8 +220,9 @@ "--min-lr", "0", "--apply-chat-template", - "--use-miles-router", - "--update-weight-group-timeout-s", + "--router", + "miles", + "--update-weight-engine-request-timeout", "120", "--spec", "miles_plugins.models.qwen3_8_next.qwen3_8_next", @@ -352,7 +353,7 @@ "--sglang-ep-size", "8" ], - "miles_argv_sha256": "8d056171747e96ed982042e21b6adbb4fed746aa560f34a47e57d980756086e9", + "miles_argv_sha256": "3ec26e53a3d33d66ae9d1bbb628ba0ecc95f6d47cbc68a244ea036b23ea8eae8", "name": "fn_2x8", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/grpo_default.json b/tests/golden/decoupling/grpo_default.json index 0d2436052..0d36826c6 100644 --- a/tests/golden/decoupling/grpo_default.json +++ b/tests/golden/decoupling/grpo_default.json @@ -151,7 +151,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -195,7 +195,7 @@ "10", "--group-query-attention" ], - "miles_argv_sha256": "bb315aa0c82a03c2821899a3a376db84f09d30f1d32e2f6640833aec455d8634", + "miles_argv_sha256": "9c9a04e7ec1baf2dde4bfe792f37e5dff44d5d3879e8a64875f7f54353259301", "name": "grpo_default", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/grpo_tis.json b/tests/golden/decoupling/grpo_tis.json index 2978b6113..24a70ca09 100644 --- a/tests/golden/decoupling/grpo_tis.json +++ b/tests/golden/decoupling/grpo_tis.json @@ -198,7 +198,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -247,7 +247,7 @@ "10", "--group-query-attention" ], - "miles_argv_sha256": "70d04ad1b86c39f254ac2631b2be4a32558c00f4e0e7cc8c1767e03ea84b03ee", + "miles_argv_sha256": "94a8f220ada63fc4863e269384723fe8742216fb0a935fb04c80e4f0f91db2d3", "name": "grpo_tis", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/golden/decoupling/seq_adv_maxrl.json b/tests/golden/decoupling/seq_adv_maxrl.json index a59eab36a..f407cf830 100644 --- a/tests/golden/decoupling/seq_adv_maxrl.json +++ b/tests/golden/decoupling/seq_adv_maxrl.json @@ -208,7 +208,7 @@ "--global-batch-size", "8", "--balance-data", - "--rollout-max-context-len", + "--max-seq-len", "2048", "--rollout-max-response-len", "128", @@ -254,7 +254,7 @@ "10", "--group-query-attention" ], - "miles_argv_sha256": "fd095a4b6d682aa8ba80b1953964b57575721859844217c78253847b7349a820", + "miles_argv_sha256": "47744c4fcfa51c09b5baed4fd29f5cc680e7c6707bdf222779984c6f8bff481a", "name": "seq_adv_maxrl", "placement": { "allow_cross_node_engine_tp": false, diff --git a/tests/hf_assets.py b/tests/hf_assets.py index 4421eb16c..b28b1750f 100644 --- a/tests/hf_assets.py +++ b/tests/hf_assets.py @@ -12,6 +12,7 @@ from __future__ import annotations import os +from pathlib import Path import pytest @@ -47,6 +48,9 @@ REQUIRE_ENV = "YETO_REQUIRE_HF_ASSETS" +TOKENIZER_VOCAB_FILES = ("tokenizer.json", "vocab.json", "tokenizer.model", "tiktoken.model", "vocab.txt") + + def hf_snapshot_dir(repo: str) -> str | None: """Local snapshot directory of ``repo`` at its pinned revision, or None.""" try: @@ -54,10 +58,15 @@ def hf_snapshot_dir(repo: str) -> str | None: except ImportError: return None try: - return snapshot_download(repo, revision=HF_TEST_REPOS[repo], allow_patterns=HF_ALLOW_PATTERNS, + path = snapshot_download(repo, revision=HF_TEST_REPOS[repo], allow_patterns=HF_ALLOW_PATTERNS, local_files_only=True) except Exception: return None + # an interrupted prefetch can leave tokenizer_config.json without the vocabulary; transformers then + # builds an empty tokenizer that encodes text to nothing (a silent wrong test, not a skip) + if not any((Path(path) / name).exists() for name in TOKENIZER_VOCAB_FILES): + return None + return path def require_hf_repo(repo: str) -> str: diff --git a/tests/import_boundary_allowlist.txt b/tests/import_boundary_allowlist.txt index 50bab4cd4..82e17dbd4 100644 --- a/tests/import_boundary_allowlist.txt +++ b/tests/import_boundary_allowlist.txt @@ -9,7 +9,7 @@ # # --- algorithm extensions (A1-A5) yeto/rl/algos/reducers.py megatron # A3 :35,41 -- kept for now (reducers are not moved) -yeto/rl/algos/vendor/miles_mis.py miles # A3 :350 cp_utils -- kept for now +yeto/rl/algos/vendor/miles_mis.py miles # A3 :350 data.context_parallel -- kept for now yeto/rl/grad_audit.py megatron # A3 :209,320 -- kept for now # # --- rewards and filters (R1-R3) diff --git a/tests/js/dashboard_smoke.js b/tests/js/dashboard_smoke.js index 34a7f2f4a..abddbb7bb 100644 --- a/tests/js/dashboard_smoke.js +++ b/tests/js/dashboard_smoke.js @@ -37,7 +37,8 @@ global.setTimeout = () => 0; eval(appM[1]); const out = { title: els.title.innerHTML, kpis: els.kpis.innerHTML, islands: els.islands.innerHTML, cost: els.costBox.innerHTML, events: els.evs.innerHTML, folds: els.foldBox.innerHTML, - metric: els.cMetric.innerHTML, svg_nodes: svgNodes.length, wall_hidden: els.islandWall.hidden }; + metric: els.cMetric.innerHTML, svg_nodes: svgNodes.length, wall_hidden: els.islandWall.hidden, + nodes_note: (els.rNodesNote || {}).textContent }; // generation-stage utilization panel out.util = { empty_hidden: mk("utilEmpty").hidden, body_hidden: mk("utilBody").hidden, ctl: mk("utilCtl").innerHTML, cut: mk("uCut").innerHTML, carry_hidden: mk("uCarryBox").hidden, carry: mk("uCarry").innerHTML, @@ -45,10 +46,10 @@ out.util = { empty_hidden: mk("utilEmpty").hidden, body_hidden: mk("utilBody").h cut_lines: svgNodes.filter((n) => n.attrs["class"] === "ucut").length }; // hover a timeline/duration segment with a mousemove handler -> tooltip const hov = svgNodes.find((n) => n.listeners.mousemove && n.attrs["class"] === "tseg"); -if (hov) { hov.listeners.mousemove[0]({ clientX: 10, clientY: 10 }); out.tip = els.tip.innerHTML; } +if (hov) { hov.listeners.mousemove[0]({ clientX: 10, clientY: 10 }); out.tip = mk("tip").innerHTML; } // keyboard: right arrow on the rounds panel const kd = (els.rounds.listeners.keydown || [])[0]; -if (kd) { kd({ key: "ArrowRight", preventDefault() {} }); out.tip_key = els.tip.innerHTML; } +if (kd) { kd({ key: "ArrowRight", preventDefault() {} }); out.tip_key = mk("tip").innerHTML; } els.layoutBtn.onclick(); out.title_after_switch = els.title.innerHTML; out.wall_hidden_after_switch = els.islandWall.hidden; out.wall = els.wall.innerHTML; process.stdout.write(JSON.stringify(out)); diff --git a/tests/multinode_gpu/multinode_judge.py b/tests/multinode_gpu/multinode_judge.py index b02b33ce7..800ab4db9 100755 --- a/tests/multinode_gpu/multinode_judge.py +++ b/tests/multinode_gpu/multinode_judge.py @@ -93,8 +93,8 @@ def trainer_cross_node_evidence(head, worker): is a finite number > 0 for >= 1 round; param_update = rl_publication payload hash changes between consecutive policy versions AND policy_version strictly increases over >= 2 publications (driver.publish: sync/publication_payload_hash is the hash of the LoRA - weights pushed to the engines). rl_local_round.delta_l2_norm is hard-coded 0.0 by the driver - (driver.py rl_local_round emit) and is therefore NOT used.""" + weights pushed to the engines). rl_local_round.delta_l2_norm is null (not measured) on the + engine-driver path (driver.py _stats) and is therefore NOT used.""" both = bool(re.search(r"MegatronTrainRayActor", head)) and bool(re.search(r"MegatronTrainRayActor", worker)) tokens = [e.get("action_tokens") or 0 for e in local_rounds] grads = [e.get("grad_norm") for e in local_rounds if isinstance(e.get("grad_norm"), (int, float))] @@ -108,7 +108,7 @@ def trainer_cross_node_evidence(head, worker): and len({h for _, h in hashes if h}) >= 2), } notes.append(f"grad_norms={grads[:4]} pub_hashes={[(v, (h or '')[:8]) for v, h in hashes][:4]} " - f"loss_logged={bool(re.search(r'lm loss|forward', read('pulled/run.log')))} (delta_l2_norm unused: driver hard-codes 0.0)") + f"loss_logged={bool(re.search(r'lm loss|forward', read('pulled/run.log')))} (delta_l2_norm unused: not measured by the driver)") return ev if rc == 124 and not (CASE == "g0" and len(rounds_trained) >= 1 and len(generates) >= 1): diff --git a/tests/reward_equivalence_cases.py b/tests/reward_equivalence_cases.py index a0ba3ac6e..4c873e8d5 100644 --- a/tests/reward_equivalence_cases.py +++ b/tests/reward_equivalence_cases.py @@ -139,6 +139,7 @@ def _metadata_seed(i: int): def render() -> dict[str, Any]: from yeto.rl import filters, gsm8k_reward, length_reward, math_reward + from yeto.rl.adapters.miles.proc_state import proc_state from yeto.rl.algos import gdpo_reward out: dict[str, Any] = {} @@ -176,7 +177,7 @@ def render() -> dict[str, Any]: args.yeto_rl_dynamic_sampling_max_replacements = limit group = [FakeSample("r", None, index=ix, reward=r) for r, ix in zip(rewards, indices)] result = filters.bounded_nonzero_reward_std(args, group) - state = args._yeto_bounded_filter_state + state = proc_state(args, filters.BOUNDED_FILTER_STATE_ATTR) rows.append({"keep": result.keep, "reason": result.reason, "state": {k: state[k] for k in ("rollout_id", "rejections", "forced")}, "decisions": len(state["decisions"])}) diff --git a/tests/test_agentic_alignment_loss.py b/tests/test_agentic_alignment_loss.py new file mode 100644 index 000000000..1b22c9306 --- /dev/null +++ b/tests/test_agentic_alignment_loss.py @@ -0,0 +1,153 @@ +"""token_alignment losses of the Harbor + pi path (K3.1 run, 2026-11-10). + +Root cause seen in the pi sessions: pi omits an empty reasoning and +re-serializes tool-call arguments (``JSON.stringify``), so its echo of a turn +the proxy generated hashed differently -> ``retry_fork`` -> a new recorder +session that re-tokenizes the history. When the sampled ids are not the +tokenizer's canonical encoding of their text (byte-level pieces of binary / +non-UTF-8 output), the new session's prompt does not extend the old stream +and the whole trial was dropped. A byte-level mock tokenizer stands in for +the K3.1 tokenizer (``tiktoken`` is not installed here).""" + +from __future__ import annotations + +import asyncio +import json + +import pytest + +from yeto.rl.agentic import TokenAlignmentError, TokenTurn +from yeto.rl.agentic.finish import finish_sample +from yeto.rl.agentic.tokens import join_turns +from yeto.rl.harness.gateway.chains import ChainRegistry, LocateKind + + +class ByteTokenizer: + """ids 0-255: single bytes; 256: the merged pair b"\\xc3\\xa9" ("é"). Encoding + merges, so sampled ids [0xc3, 0xa9] re-encode as [256].""" + + def encode(self, text: str) -> list[int]: + raw, out, i = text.encode("utf-8", errors="surrogateescape"), [], 0 + while i < len(raw): + if raw[i:i + 2] == b"\xc3\xa9": + out.append(256) + i += 2 + else: + out.append(raw[i]) + i += 1 + return out + + def decode(self, ids: list[int]) -> str: + return b"".join(b"\xc3\xa9" if i == 256 else bytes([i]) for i in ids).decode("utf-8", errors="replace") + + +TOK = ByteTokenizer() + + +def _turn(prompt: list[int], completion: list[int], *, lp: float = -0.5) -> TokenTurn: + return TokenTurn(prompt_ids=tuple(prompt), completion_ids=tuple(completion), + completion_logprobs=tuple([lp] * len(completion))) + + +def _forked_turns() -> list[TokenTurn]: + """Turn 0 samples "é" as two byte tokens; the fork's session re-encodes it merged.""" + p0 = TOK.encode("do it") + c0 = [*TOK.encode("caf"), 0xC3, 0xA9, *TOK.encode("")] # non-canonical: merge would give 256 + tool = TOK.encode("ok") + p1 = [*p0, *c0, *tool] + c1 = TOK.encode("done") + reencoded = TOK.encode(TOK.decode(p1)) # what a new session renders + encodes + assert reencoded != p1 and TOK.decode(reencoded) == TOK.decode(p1) + return [_turn(p0, c0, lp=-0.1), _turn(p1, c1, lp=-0.2), _turn([*reencoded, *c1, *tool], TOK.encode("bye"))] + + +def test_reencoded_history_does_not_extend_the_stream_and_the_error_says_where(): + turns = _forked_turns() + with pytest.raises(TokenAlignmentError) as info: + join_turns(turns) + exc = info.value + assert exc.turn == 2 and exc.position == len(turns[0].prompt_ids) + 3 + assert exc.stream[exc.position:exc.position + 2] == (0xC3, 0xA9) and exc.prompt[exc.position] == 256 + + +def test_rebase_trains_only_tokens_sampled_on_the_engines_stream(): + turns = _forked_turns() + seg = join_turns(turns, rebase=True) + assert seg.rebased_turns == 1 + last = turns[-1] + # the training stream is exactly the last call's prompt + completion: what the engine saw + assert (*seg.prompt_ids, *seg.response_ids) == (*last.prompt_ids, *last.completion_ids) + trained = [(t, lp) for t, m, lp in zip(seg.response_ids, seg.response_mask, seg.response_logprobs) if m] + assert trained == [(t, -0.5) for t in last.completion_ids] # earlier completions: context, mask 0 + assert not any(m for m in seg.response_mask[:len(seg.response_ids) - len(last.completion_ids)]) + + +def test_rebase_still_refuses_a_stream_that_does_not_repeat_the_first_prompt(): + p0 = TOK.encode("a") + with pytest.raises(TokenAlignmentError): + join_turns([_turn(p0, [1, 2]), _turn(TOK.encode("b"), [3])], rebase=True) + + +def _js_stringify(arguments: str) -> str: + return json.dumps(json.loads(arguments), separators=(",", ":"), ensure_ascii=False) + + +def test_pi_echo_without_empty_reasoning_and_restringified_arguments_continues_the_chain(): + registry = ChainRegistry() + request = [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}] + chain = registry.open_chain("rec0", request, registry.locate(request)) + args = '{"path":"/w/p.py","edits":[{"oldText": "a", "newText": "b"}],"timeout":30.0}' # model's literal text + generated = {"role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": [{"id": "c0", "type": "function", "function": {"name": "edit", "arguments": args}}]} + registry.record_generation(chain, generated) + echoed = {"role": "assistant", "content": "", + "tool_calls": [{"id": "c0", "type": "function", + "function": {"name": "edit", "arguments": _js_stringify(args).replace("30.0", "30")}}]} + located = registry.locate([*request, echoed, {"role": "tool", "tool_call_id": "c0", "content": "ok"}]) + assert located.kind is LocateKind.continue_ and located.chain is chain + assert chain.messages[2] is generated # the next turn renders what the engine sampled + + +def test_a_changed_call_still_forks(): + registry = ChainRegistry() + request = [{"role": "user", "content": "u"}] + chain = registry.open_chain("rec0", request, registry.locate(request)) + registry.record_generation(chain, {"role": "assistant", "content": "", "reasoning_content": "r", "tool_calls": [ + {"id": "c0", "type": "function", "function": {"name": "bash", "arguments": '{"command":"ls"}'}}]}) + other = {"role": "assistant", "content": "", "tool_calls": [ + {"id": "c0", "type": "function", "function": {"name": "bash", "arguments": '{"command":"pwd"}'}}]} + assert registry.locate([*request, other]).kind is LocateKind.fork + + +class _Recorder: + def __init__(self, turns): + self.turns = turns + + async def close(self, session): + return self.turns + + def decode(self, ids): + return TOK.decode(ids) + + +class _Metrics: + def reclassify(self, old, new): + self.moved = (old, new) + + +def test_finish_sample_reports_a_token_alignment_loss(): + from yeto.rl.agentic import AgentTask, AgentTrajectory, Outcome + from yeto.rl.agentic.types import TrialVerdict + + p0 = TOK.encode("a") + turns = [_turn(p0, [0xC3, 0xA9]), _turn(TOK.encode("b"), [3])] + task = AgentTask(task_id="t", agent="pi", provider="docker", trajectory_id="s1") + verdict = TrialVerdict(rewards={"reward": 1.0}, trial_dir="/trials/x") + trajectory = AgentTrajectory(task=task, outcome=Outcome.OK, reward=1.0, reason="", verdict=verdict) + done = asyncio.run(finish_sample(trajectory, _Recorder(turns), "s1", _Metrics(), max_response_len=64)) + assert done.segments is None and done.trajectory.outcome is Outcome.ABORTED + loss = done.loss + assert loss["stage"] == "join_turns" and loss["turn"] == 1 and loss["trial_dir"] == "/trials/x" + assert "does not extend" in loss["error"] and loss["position"] == 5 + assert loss["before_text"] == "" and loss["stream_text"].startswith("a") + json.dumps(loss) diff --git a/tests/test_agentic_engine_sampler.py b/tests/test_agentic_engine_sampler.py index 6b9f13ce1..fc9c60231 100644 --- a/tests/test_agentic_engine_sampler.py +++ b/tests/test_agentic_engine_sampler.py @@ -183,8 +183,10 @@ def test_infra_failed_trial_is_run_again_and_its_group_written_whole(tmp_path, q agentic = round_event["agentic"] assert agentic["lost"] == {} and agentic["groups_below_min"] == 0 assert agentic["infra_retries"] == {"RuntimeError": 1} and agentic["groups_refilled"] == 1 + assert (agentic["retried_trials"], agentic["lost_trials"]) == (1, 0) groups = [f for n, f in events if n == "rl_agentic_group"] assert sorted(g["infra_retries"] for g in groups) == [[0, 0], [1, 0]] + assert sorted(g["retried_trials"] for g in groups) == [0, 1] and all(g["lost_trials"] == 0 for g in groups) def test_infra_retries_stop_at_the_limit(tmp_path, qwen): @@ -324,3 +326,18 @@ def test_nemotron3_codex_loop_records_one_aligned_stream(nemotron3): assert records["counters"].get("render_drift_turns", 0) == 0 (sample,) = records["sessions"][0]["samples"] assert [t["kind"] for t in sample["turns"]] == ["action"] * 3 + + +def test_chat_renderer_passes_the_template_kwargs(): + from yeto.rl.engine.engine_sampler_entry import chat_renderer + + seen = {} + + class Tok: + def apply_chat_template(self, messages, **kw): + seen.update(kw) + return messages[0]["content"] + + render = chat_renderer("unused", template_kwargs={"thinking_effort": "low"}, tokenizer=Tok()) + assert render("q") == "q" + assert seen["thinking_effort"] == "low" and seen["add_generation_prompt"] is True diff --git a/tests/test_agentic_harbor.py b/tests/test_agentic_harbor.py index 4e1281df0..99a2a7b41 100644 --- a/tests/test_agentic_harbor.py +++ b/tests/test_agentic_harbor.py @@ -460,7 +460,7 @@ def test_verl_loop_aborts_on_misaligned_tokens_and_infra_errors(verl_loop): loop_mod, _ = verl_loop from yeto.rl.harness.harbor import HarborHarness - bad = [_turn([5, 6], [40]), _turn([5, 6, 77, 1], [41])] # history rewritten (compaction) + bad = [_turn([5, 6], [40]), _turn([5, 9, 77, 1], [41])] # even the first prompt rewritten (no rebase) _FakeTrial.result = _result({"reward": 1.0}) loop = _make_loop(loop_mod, _Recorder(bad), HarborHarness(lambda t, u: {}, trial_cls=_FakeTrial)) diff --git a/tests/test_agentic_round_streaming.py b/tests/test_agentic_round_streaming.py index 014e048f9..6e48dc429 100644 --- a/tests/test_agentic_round_streaming.py +++ b/tests/test_agentic_round_streaming.py @@ -569,3 +569,156 @@ async def close(self, trajectory_id): rnd.close() (group,) = [f for n, f in events if n == GROUP_EVENT] assert group["lost"] == {LOST_VERSION: N} and not group["written"] + + +class _AlwaysInFlight: + """A streaming round that always holds a group whose trials have not ended + (agentic trials of 20+ min: the sampler never sees a gap between groups).""" + + def __init__(self) -> None: + self.calls = 0 + + def holds_weights(self) -> bool: + self.calls += 1 + return True + + +def test_a_release_deferred_past_the_bound_is_loaded_with_forced_and_deferred_s(tmp_path): + from yeto.rl.engine.sampler_loop import SamplerLoop + + releases = PolicyReleaseStore(CheckpointStore(tmp_path)) + loaded = [] + now = {"t": 0.0} + follower = PolicyFollower(releases, lambda release, directory: loaded.append(release.version), + timer=lambda: now["t"], max_defer_s=600.0) + follower.hold = _AlwaysInFlight().holds_weights + events = [] + loop = SamplerLoop(follower=follower, buffer=None, generate=lambda rid: None, num_rollouts=1, + max_backlog_groups=1, emit=lambda n, **f: events.append((n, f)), sleep=lambda s: None) + releases.release(1, "h1", lambda d: (d / "model.safetensors").write_text("w")) + for t in (0.0, 300.0, 599.9): # still within the bound: deferred + now["t"] = t + loop._load_newest() + assert not loaded and follower.counters.deferred_in_flight == 3 and follower.counters.forced_loads == 0 + assert not [f for n, f in events if n == "rl_sampler_policy_loaded"] + now["t"] = 600.0 + loop._load_newest() + assert loaded == [1] and follower.version == 1 and follower.counters.forced_loads == 1 + (event,) = [f for n, f in events if n == "rl_sampler_policy_loaded"] + assert event["forced"] is True and event["deferred_s"] == 600.0 and event["policy_version"] == 1 + # the next release starts a new deferral clock + releases.release(2, "h2", lambda d: (d / "model.safetensors").write_text("w")) + now["t"] = 700.0 + loop._load_newest() + assert loaded == [1] and follower.counters.deferred_in_flight == 4 + now["t"] = 1300.0 + loop._load_newest() + assert loaded == [1, 2] and follower.counters.forced_loads == 2 + assert [f["deferred_s"] for n, f in events if n == "rl_sampler_policy_loaded"] == [600.0, 600.0] + + +def test_the_deferral_bound_never_forces_a_load_into_a_marked_round_or_when_unset(tmp_path): + releases = PolicyReleaseStore(CheckpointStore(tmp_path)) + loaded = [] + now = {"t": 0.0} + follower = PolicyFollower(releases, lambda release, directory: loaded.append(release.version), + timer=lambda: now["t"], max_defer_s=10.0) + releases.release(1, "h1", lambda d: (d / "model.safetensors").write_text("w")) + with follower.generating(): # a round that cannot take weights mid-generation + assert follower.refresh() is None + now["t"] = 1e6 + assert follower.refresh() is None and not loaded + unbounded = PolicyFollower(releases, lambda release, directory: loaded.append(release.version), + timer=lambda: now["t"]) + unbounded.hold = lambda: True + for t in (0.0, 1e9): + now["t"] = t + assert unbounded.refresh() is None + assert not loaded and unbounded.counters.forced_loads == 0 + with pytest.raises(ValueError): + PolicyFollower(releases, lambda r, d: None, max_defer_s=0) + + +def test_an_unforced_load_carries_no_forced_field(tmp_path): + from yeto.rl.engine.sampler_loop import SamplerLoop + + releases = PolicyReleaseStore(CheckpointStore(tmp_path)) + follower = PolicyFollower(releases, lambda release, directory: None, max_defer_s=600.0) + events = [] + loop = SamplerLoop(follower=follower, buffer=None, generate=lambda rid: None, num_rollouts=1, + max_backlog_groups=1, emit=lambda n, **f: events.append((n, f)), sleep=lambda s: None) + releases.release(1, "h1", lambda d: (d / "model.safetensors").write_text("w")) + loop._load_newest() + (event,) = [f for n, f in events if n == "rl_sampler_policy_loaded"] + assert "forced" not in event and "deferred_s" not in event and not follower.last_forced + + +def test_spec_reload_max_defer_s_defaults_to_600_and_null_disables(): + from yeto.rl.engine.engine_sampler_entry import island_config + + base = {"sampler_id": "s", "num_rollouts": 1, "max_backlog_groups": 1, "sampling": {"n": 2, "max_tokens": 8}, + "max_prompt_tokens": 16} + assert island_config(base).reload_max_defer_s == 600.0 + assert island_config({**base, "reload_max_defer_s": 120}).reload_max_defer_s == 120.0 + assert island_config({**base, "reload_max_defer_s": None}).reload_max_defer_s is None + + +def test_a_row_dispensed_twice_in_one_round_gets_its_own_sessions_and_group(tmp_path): + # q06-nc-e2e-20261110T2255Z: dispense positions 74 and 75 both mapped to row7 + # (epoch wrap of a small dataset); both groups opened '-r410-row7-0' and the + # recorder's "already open" stopped the island with INFRA_FAILED. + harness = Harness({"row7": 0.0}) + + class Twice(Prompts): + def __call__(self, rollout_id): + meta = [{"agent_task": dict(TASK), DISPENSE_POSITION_KEY: spot} for spot in (74, 75)] + return [PromptItem("row7", "task row7", metadata=m) for m in meta] + + prompts = Twice({}, dispensed=True) + + class Strict(Recorder): + async def open(self, trajectory_id, sampling_params): + assert trajectory_id not in self.opened, f"{trajectory_id} already open" + return await super().open(trajectory_id, sampling_params) + + rnd, engine, events = _round(tmp_path, harness, prompts, incarnation="132f9179") + rnd.recorder = Strict(engine) + try: + rnd(410) + rnd.flush() + finally: + rnd.close() + assert len(set(harness.started)) == len(harness.started) == 2 * N + entries, _ = _entries(tmp_path) + assert sorted(e.sample_id for e in entries) == ["s-132f9179-r410-row7", "s-132f9179-r410-row7-p75"] + assert sorted(c[1] for c in prompts.calls) == [[74], [75]] + + +def test_a_recorder_session_already_open_loses_that_trial_not_the_island(tmp_path): + from yeto.rl.agentic import HarnessConfigError, SessionAlreadyOpen + from yeto.rl.engine.agentic_round import LOST_SESSION_COLLISION + + harness = Harness({"a": 0.0}) + prompts = Prompts({0: ["a"], 1: ["a"]}) + + class Colliding(Recorder): + async def open(self, trajectory_id, sampling_params): + if trajectory_id.endswith("-r0-a-1"): + raise SessionAlreadyOpen(f"recorder session {trajectory_id!r} is already open") + return await super().open(trajectory_id, sampling_params) + + assert issubclass(SessionAlreadyOpen, HarnessConfigError) + rnd, engine, events = _round(tmp_path, harness, prompts, incarnation="i") + rnd.recorder = Colliding(engine) + try: + rnd(0) + rnd(1) # the island keeps going + rnd.flush() + finally: + rnd.close() + assert rnd.session_collisions == 1 + groups = {f["rollout_id"]: f for n, f in events if n == GROUP_EVENT} + assert groups[0]["lost"] == {LOST_SESSION_COLLISION: 1} and groups[0]["kept"] == N - 1 + assert groups[1]["kept"] == N + rounds = [f for n, f in events if n == ROUND_EVENT] + assert rounds[-1]["agentic"]["session_collisions_total"] == 1 diff --git a/tests/test_agentic_setup.py b/tests/test_agentic_setup.py new file mode 100644 index 000000000..4f3dd0e59 --- /dev/null +++ b/tests/test_agentic_setup.py @@ -0,0 +1,261 @@ +"""agentic.setup (yeto.launch.agentic_setup): parsing, local checks, the node steps +against a fake node, the tunnel keep-alive loop, the ssh_sampler hook. No SSH.""" + +from __future__ import annotations + +import json +import subprocess +from types import SimpleNamespace + +import pytest + +from yeto.launch import agentic_node_agent as agent +from yeto.launch import agentic_setup as su +from yeto.rl.harness.harbor.pins import HARBOR_COMMIT, HARBOR_VERSION + +# built by join: the hygiene check reads a literal "/home/" as a user home +SECRET_PATH = "/".join(["/work/harbor", "home", "records-proxy.secret"]) +NVME_SECRET_PATH = "/mnt/nvme" + SECRET_PATH + + +def _spec(tmp_path, **setup): + key = tmp_path / "docker.key" + key.write_text("-----BEGIN OPENSSH " + "PRIVATE KEY-----\nfake\n-----END OPENSSH " + "PRIVATE KEY-----\n") + key.chmod(0o600) + tasks = tmp_path / "tasks" + for t in ("a", "b"): + (tasks / t).mkdir(parents=True, exist_ok=True) + (tasks / t / "task.toml").write_text("[task]\n") + doc = {"sandbox": {"docker_host": "ssh://sbx@sandbox.invalid", "key": {"path": str(key)}}, + "tunnel": {"listen": "192.0.2.1:19001"}, + "tasks": {"source": str(tasks), "ids": ["a", "b"]}, + "records_secret": "generate", **setup} + return {"sampler_id": "s-0", "agentic": {"proxy": {"host": "127.0.0.1", "port": 18080}, + "task_defaults": {"provider": "remote-docker"}, "setup": doc}} + + +def test_parse_defaults_and_container_env(tmp_path): + s = su.parse(_spec(tmp_path)) + assert (s.harbor_version, s.harbor_commit) == (HARBOR_VERSION, HARBOR_COMMIT) + assert s.sandbox.alias == "yeto-sandbox" and s.tunnel.target_port == 18080 and s.tunnel.key == s.sandbox.key + env = s.container_env() + assert env["DOCKER_HOST"] == "ssh://yeto-sandbox" and env["HOME"] == "/work/harbor/home" + assert env["YETO_RECORDS_PROXY_PUBLIC_BASE"] == "http://192.0.2.1:19001" + assert env["YETO_RECORDS_PROXY_SECRET_FILE"] == SECRET_PATH + assert su.local_problems(s) == [] + assert su.parse({"agentic": {}}) is None + assert "setup" not in su.strip(_spec(tmp_path))["agentic"] + + +@pytest.mark.parametrize("change, match", [ + ({"bogus": 1}, "unknown keys"), + ({"images": ["x"]}, "agentic.envs"), + ({"harbor": {"version": "0.24.0"}}, "verified against Harbor"), + ({"docker_cli": {"version": "latest"}}, "pinned"), + ({"sandbox": {"docker_host": "tcp://h", "key": {"path": "k"}}}, "ssh://"), + ({"tunnel": {"listen": "nope"}}, "listen"), + ({"root": "/etc"}, "under /work"), + ({"records_secret": None}, "records_secret must be"), +]) +def test_parse_refusals(tmp_path, change, match): + with pytest.raises(ValueError, match=match): + su.parse(_spec(tmp_path, **change)) + + +def test_tunnel_needs_secret_and_local_problems(tmp_path): + spec = _spec(tmp_path) + del spec["agentic"]["setup"]["records_secret"] + with pytest.raises(ValueError, match="public"): + su.parse(spec) + s = su.parse(_spec(tmp_path, tasks={"source": str(tmp_path / "tasks"), "ids": ["a", "zz"]})) + (tmp_path / "docker.key").chmod(0o644) + problems = " ".join(su.local_problems(s)) + assert "0600" in problems and "zz" in problems + lines, probs = su.dry_run_report(_spec(tmp_path, sandbox={"docker_host": "ssh://u@h", "key": {"env": "NOPE_K"}}), + environ={}) + assert any("tunnel" in ln for ln in lines) and any("NOPE_K" in p for p in probs) + + +def test_classify_ssh_failure_names_the_fingerprint(): + msg = su.classify_ssh_failure("sbx@h: Permission denied (publickey).", who="sbx@h:22", key="docker key", + fingerprint="SHA256:abc", purpose="docker") + assert "SHA256:abc" in msg and "authorized_keys" in msg + assert "permitlisten" in su.classify_ssh_failure("Error: remote port forwarding failed for listen port 1", + who="u@h", key="k", fingerprint="f", purpose="p") + + +class FakeNode: + """Answers the commands the provisioner sends, with node state in a dict.""" + + def __init__(self, *, harbor_ok=False, tools_ok=False, docker_ok=True, tunnel_err=None): + self.files: dict[str, tuple[str, str]] = {} + self.harbor_ok, self.tools_ok, self.docker_ok, self.tunnel_err = harbor_ok, tools_ok, docker_ok, tunnel_err + self.tunnel_running, self.label, self.events = False, "", [] + self.calls: list[str] = [] + + def r(self, out="", code=0, err=""): + return SimpleNamespace(returncode=code, stdout=out, stderr=err) + + def run(self, argv, **kw): + cmd = argv[-1] if argv[0] == "ssh" else " ".join(argv) + self.calls.append(cmd) + inp = kw.get("input") + if argv[0] == "ssh-keygen": + return self.r("256 SHA256:fake comment (ED25519)\n" if "-lf" in argv else "ssh-ed25519 AAAA\n") + if "sha256sum" in cmd: + path = cmd.split("sha256sum ")[1].split(" ")[0].strip("'") + if path in self.files: + import hashlib + text, mode = self.files[path] + return self.r(f"{hashlib.sha256(text.encode()).hexdigest()}\n{mode}\n") + return self.r("") + if inp is not None and "cat >" in cmd: + path = cmd.split("cat > ")[1].split(".tmp")[0] + mode = cmd.split("chmod ")[1].split(" ")[0] + self.files[path] = (inp, mode) + return self.r() + if "docker --version" in cmd: + return self.r("Docker version 29.0.2, build x\nDocker Compose version v2.40.3\n" if self.tools_ok else "") + if "curl -fsSL" in cmd and "docker-compose" in cmd: + self.tools_ok = True + return self.r() + if "pip install" in cmd: + self.harbor_ok = True + return self.r("YETO_SETUP_PRUNED pydantic,requests\n") + if "YETO_SETUP_HARBOR" in cmd or "metadata.distribution" in cmd: + st = {"version": HARBOR_VERSION, "commit": HARBOR_COMMIT, "import": "ok"} if self.harbor_ok else \ + {"error": "PackageNotFoundError: harbor"} + return self.r("YETO_SETUP_HARBOR " + json.dumps(st) + "\n") + if "docker -H ssh://" in cmd: + return self.r("29.0.2\n[name=rootless]\n") if self.docker_ok else \ + self.r("", 1, "sbx@sandbox.invalid: Permission denied (publickey).") + if "records-proxy.secret" in cmd and "urandom" in cmd: + made = NVME_SECRET_PATH not in self.files + self.files.setdefault(NVME_SECRET_PATH, ("x" * 40, "600")) + return self.r("made\n" if made else "kept\n") + if "stat -c %a" in cmd and "wc -c" in cmd: + return self.r("600\n40\n") + if "sudo test -f" in cmd and ".yeto-setup-" in cmd: + return self.r(code=0 if any(".yeto-setup-" in f for f in self.files) else 1) + if "sudo tar -C" in cmd: + return self.r() + if "sudo touch" in cmd: + self.files[cmd.split("sudo touch ")[1].strip("'")] = ("", "644") + return self.r() + if "task.toml" in cmd: + return self.r() + if "docker inspect" in cmd and su.TUNNEL_CONTAINER in cmd: + return self.r(f"true {self.label}\n" if self.tunnel_running else "") + if "docker run -d --name " + su.TUNNEL_CONTAINER in cmd: + self.tunnel_running = True + self.label = cmd.split("yeto.setup=")[1].split(" ")[0] + self.events.append({"event": su.TUNNEL_EVENT, "state": "down", "stderr": self.tunnel_err} + if self.tunnel_err else {"event": su.TUNNEL_EVENT, "state": "up"}) + return self.r() + if "tail -n 50" in cmd or "sudo cat" in cmd: + return self.r("".join(json.dumps(e) + "\n" for e in self.events)) + return self.r() + + +def _node(fake): + return su.SshNode("nc-t1", "img", "/mnt/nvme", "/mnt/nvme/yeto/abc", "/opt/runtime-venv/bin/python3", "ubuntu", + run=fake.run) + + +def _prov(tmp_path, fake, **kw): + s = su.parse(_spec(tmp_path, **kw)) + events = [] + p = su.Provisioner(s, _node(fake), emit=events.append, sleep=lambda _s: None, local_run=fake.run) + p.node.stream_tar = lambda *a, **k: None + return p, events + + +def test_provision_from_scratch_then_idempotent(tmp_path): + fake = FakeNode() + p, events = _prov(tmp_path, fake) + report = p.run() + assert report == {"layout": "ok", "docker_cli": "changed", "harbor": "changed", "sandbox": "changed", + "records_secret": "changed", "tasks": "changed", "tunnel": "changed"} + assert all(e["event"] == su.SETUP_EVENT for e in events) + assert fake.files["/mnt/nvme/agentic-tunnel/key"][1] == "600" + assert "ControlMaster auto" in fake.files["/mnt/nvme/work/harbor/home/.ssh/config"][0] + # the tunnel key is never under /work (the sampler container sees only /work) + assert not any(f.startswith("/mnt/nvme/work") and "agentic-tunnel" in f for f in fake.files) + n_runs = sum("docker run -d --name " + su.TUNNEL_CONTAINER in c for c in fake.calls) + p2, _ = _prov(tmp_path, fake) + assert set(p2.run().values()) == {"ok"} + assert sum("docker run -d --name " + su.TUNNEL_CONTAINER in c for c in fake.calls) == n_runs + assert not any("pip install" in c for c in fake.calls[len(fake.calls) // 2:]) or fake.harbor_ok + + +def test_refused_docker_key_names_the_fingerprint(tmp_path): + fake = FakeNode(docker_ok=False) + p, events = _prov(tmp_path, fake) + with pytest.raises(su.SetupError, match="SHA256:fake.*authorized_keys") as info: + p.run() + assert info.value.step == "sandbox" and events[-1]["status"] == "failed" + + +def test_refused_forward_is_reported(tmp_path): + fake = FakeNode(tunnel_err="Error: remote port forwarding failed for listen port 19001") + p, _ = _prov(tmp_path, fake, tunnel={"listen": "192.0.2.1:19001", "ready_timeout_s": 0}) + with pytest.raises(su.SetupError, match="permitlisten"): + p.run() + + +def test_tunnel_loop_reconnects_and_emits(tmp_path): + class Proc: + def __init__(self, lives): + self.lives, self.returncode = lives, None + self.stderr = SimpleNamespace(read=lambda: "Connection reset") + + def wait(self, timeout=None): + if self.lives and timeout is not None: + raise subprocess.TimeoutExpired("ssh", timeout) + self.returncode = 255 + return 255 + + procs = iter([Proc(True), Proc(False), Proc(True)]) + sleeps = [] + events = tmp_path / "t.jsonl" + agent.tunnel_loop(["ssh", "-N"], events, popen=lambda *a, **k: next(procs), sleep=sleeps.append, + clock=iter(range(0, 1000, 7)).__next__, max_attempts=3, stable_s=1000) + recs = [json.loads(ln) for ln in events.read_text().splitlines()] + assert [r["state"] for r in recs] == ["up", "down", "down", "up", "down"] + assert recs[1]["stderr"] == "Connection reset" and len(sleeps) == 3 and sleeps[1] > sleeps[0] + + +def test_start_sampler_runs_setup_and_layers_env_files(tmp_path, monkeypatch): + from yeto.launch.nc import client as ncc + from yeto.launch.nc import ssh_sampler as ss + + uploads, ran = [], [] + monkeypatch.setattr(ss, "upload_code", lambda alias, run: "/mnt/nvme/yeto/x") + monkeypatch.setattr(ss, "upload_file", lambda a, local, remote, **k: uploads.append((str(local), remote))) + monkeypatch.setattr(su.Provisioner, "run", lambda self: ran.append(self.node.alias) or {}) + calls = [] + ok = lambda argv, **k: calls.append(argv) or SimpleNamespace(returncode=0, stdout="", stderr="") # noqa: E731 + lease = ncc.NodeLease(1, "t1", "192.0.2.7", "ubuntu", "ssh-ed25519 K", None, "2026-11-09T16:00:00Z", 8, 1) + spec = _spec(tmp_path) + spec["sampler_id"] = "nc-0" + state = tmp_path / "state" + state.mkdir() + ss.start_sampler(lease, spec=spec, shard="nc-0", run_prefix="p", r2_env_file="r2", state_dir=state, + agentic_env_file="op.env", run=ok) + assert ran == ["nc-t1-ws"] or len(ran) == 1 + assert "setup" not in json.loads((state / "spec-nc-0.json").read_text())["agentic"] + run_cmd = calls[-1][-1] + assert run_cmd.index("nc-0.agentic-setup.env") < run_cmd.index("nc-0.agentic.env") + assert "PYTHONPATH=/yeto:/work/harbor/site" in run_cmd + assert "DOCKER_HOST=ssh://yeto-sandbox" in (state / "nc-0.agentic-setup.env").read_text() + + +def test_a_task_source_sandbox_prepare_stages_is_no_dry_run_problem(tmp_path, monkeypatch): + """A new run's first `run up --dry-run` comes before `yeto sandbox prepare` stages + ~/.yeto/sandbox//stage/; a missing directory elsewhere is still a problem.""" + monkeypatch.setenv("HOME", str(tmp_path / "home")) + staged = tmp_path / "home" / ".yeto" / "sandbox" / "run-new" / "stage" / "sbx-3" + s = su.parse(_spec(tmp_path, tasks={"source": str(staged), "ids": ["a"]})) + assert not [p for p in su.local_problems(s) if "tasks" in p] + s = su.parse(_spec(tmp_path, tasks={"source": str(tmp_path / "nowhere"), "ids": ["a"]})) + assert any("is not a directory" in p for p in su.local_problems(s)) diff --git a/tests/test_agentic_setup_port_range.py b/tests/test_agentic_setup_port_range.py new file mode 100644 index 000000000..048643369 --- /dev/null +++ b/tests/test_agentic_setup_port_range.py @@ -0,0 +1,25 @@ +import pytest + +from yeto.launch.agentic_setup import _shard_port + + +def test_port_range_gives_each_shard_its_own_listen_port(): + assert [_shard_port("18081-18084", f"run-x-{k}") for k in range(4)] == [18081, 18082, 18083, 18084] + + +@pytest.mark.parametrize("sid", ["run-x-4", "__SHARD__", None]) +def test_port_range_refuses_shard_without_port(sid): + with pytest.raises(ValueError): + _shard_port("18081-18084", sid) + + +def test_pool_warns_when_the_sampler_spec_turns_the_incarnation_lease_off(tmp_path): + import json + + from yeto.launch.nc import cli + + spec = tmp_path / "spec.json" + spec.write_text(json.dumps({"incarnation_lease": False})) + assert any("incarnation_lease is false" in n for n in cli.pool_config_warnings({"spec": str(spec)})) + spec.write_text(json.dumps({})) + assert not any("incarnation_lease" in n for n in cli.pool_config_warnings({"spec": str(spec)})) diff --git a/tests/test_agentic_topk_r3.py b/tests/test_agentic_topk_r3.py new file mode 100644 index 000000000..d4a29ba5a --- /dev/null +++ b/tests/test_agentic_topk_r3.py @@ -0,0 +1,206 @@ +"""Agentic samples carry the rollout top-K log-probs and R3 routed experts. + +Multi-turn records-proxy turns -> ``turns_from_records`` -> ``join_turns`` -> +``agentic_sample`` -> ``miles_sample_dict`` -> Miles ``Sample`` -> +``convert_samples_to_train_data`` with score_centering and routing replay on. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import numpy +import pytest + +from yeto.rl.agentic import AgentTask, AgentTrajectory, Outcome, TokenAlignmentError, TokenTurn, join_turns +from yeto.rl.agentic.finish import FinishedSample +from yeto.rl.engine.agentic_round import agentic_sample +from yeto.rl.engine.buffer_payload import ( + MILES_TOPK_LOG_PROBS, + MILES_TOPK_TOKEN_IDS, + ROLLOUT_ROUTED_EXPERTS, + ROLLOUT_TOP_LOGPROBS, + ROLLOUT_TOP_LOGPROBS_IDS, + decode_group, + encode_group, + miles_sample_dict, +) +from yeto.rl.engine.engine_sampler import PromptItem +from yeto.rl.harness.gateway.proxy import _records_dumps +from yeto.rl.harness.gateway.recorder import turns_from_records + +K, LAYERS, TOPK, EXPERTS = 4, 3, 2, 16 + + +def _top(completion, seed): + ids = numpy.array([[t] + [(t + c) % 1000 for c in range(1, K)] for t in completion], dtype=numpy.int32) + vals = -numpy.random.default_rng(seed).random((len(completion), K)).astype(numpy.float32) - 0.01 + return ids.reshape(len(completion), K), vals + + +def _routed(rows, seed): + return numpy.random.default_rng(seed).integers(0, EXPERTS, size=(rows, LAYERS, TOPK), dtype=numpy.uint16) + + +def _turn(prompt, completion, seed, *, r3=True): + ids, vals = _top(completion, seed) + return TokenTurn(prompt_ids=tuple(prompt), completion_ids=tuple(completion), + completion_logprobs=tuple(float(v) for v in vals[:, 0]), + completion_top_ids=ids, completion_top_logprobs=vals, + routed_experts=_routed(len(prompt) + len(completion) - 1, seed) if r3 else None, + finish_reason="stop") + + +def _three_turns(): + p0 = [1, 2, 3] + c0 = [10, 11] + p1 = p0 + c0 + [20, 21, 22] # tool output + c1 = [12, 13, 14] + p2 = p1 + c1 + [23] + c2 = [15] + return [_turn(p0, c0, 0), _turn(p1, c1, 1), _turn(p2, c2, 2)] + + +def test_join_aligns_topk_with_the_mask_and_keeps_the_last_turn_r3(): + turns = _three_turns() + seg = join_turns(turns) + n = len(seg.response_ids) + assert seg.response_top_ids.shape == seg.response_top_logprobs.shape == (n, K) + mask = numpy.array(seg.response_mask, dtype=bool) + # mask-0 rows: Miles' fill between turns (id -1, log-prob -inf) + assert (seg.response_top_ids[~mask] == -1).all() and numpy.isneginf(seg.response_top_logprobs[~mask]).all() + # mask-1 rows: each turn's own candidates, the sampled token first with its log-prob + assert (seg.response_top_ids[mask, 0] == numpy.array(seg.response_ids)[mask]).all() + assert numpy.allclose(seg.response_top_logprobs[mask, 0], numpy.array(seg.response_logprobs)[mask]) + assert numpy.array_equal(seg.response_top_ids[mask][:2], turns[0].completion_top_ids) + # R3: whole stream minus one row, the last turn's (as a single-turn sample covers its prompt) + assert seg.routed_experts.shape == (len(seg.prompt_ids) + n - 1, LAYERS, TOPK) + assert seg.routed_experts is turns[-1].routed_experts or numpy.array_equal(seg.routed_experts, + turns[-1].routed_experts) + + +def test_join_truncation_cuts_every_array_to_the_kept_tokens(): + seg = join_turns(_three_turns(), max_response_len=6) + assert seg.truncated and len(seg.response_ids) == 6 + assert seg.response_top_ids.shape == (6, K) + assert seg.routed_experts.shape[0] == len(seg.prompt_ids) + 6 - 1 + + +def test_join_without_arrays_is_unchanged_and_mixed_topk_is_refused(): + plain = TokenTurn(prompt_ids=(1, 2), completion_ids=(3,), completion_logprobs=(-0.1,)) + seg = join_turns([plain]) + assert seg.response_top_ids is None and seg.routed_experts is None + with pytest.raises(TokenAlignmentError, match="top-K"): + join_turns([_turn([1, 2], [3], 0), TokenTurn(prompt_ids=(1, 2, 3, 4), completion_ids=(5,), + completion_logprobs=(-0.1,))]) + + +def test_turn_refuses_r3_rows_that_miss_its_positions(): + with pytest.raises(TokenAlignmentError, match="routed experts"): + TokenTurn(prompt_ids=(1, 2), completion_ids=(3,), completion_logprobs=(-0.1,), + routed_experts=_routed(5, 0)) + + +def _records(turns): + """Proxy records of one segment, the way RecordingBackend writes them + (only the newest turn keeps its R3).""" + last = turns[-1] + ids = list(last.prompt_ids) + list(last.completion_ids) + mask = [0] * len(ids) + out, start = [], 0 + for number, turn in enumerate(turns): + added = turn.prompt_ids[start:] + for i in range(len(turn.prompt_ids), len(turn.prompt_ids) + len(turn.completion_ids)): + mask[i] = 1 + out.append({"turn": number, "prompt_added": list(added), "completion": list(turn.completion_ids), + "logprobs": list(turn.completion_logprobs), "finish_reason": "stop", "stop_added": [], + "top_ids": turn.completion_top_ids, "top_logprobs": turn.completion_top_logprobs, + "routed_experts": turn.routed_experts if number == len(turns) - 1 else None}) + start = len(turn.prompt_ids) + len(turn.completion_ids) + return {"sessions": [{"samples": [{"segment": 0, "token_ids": ids, "loss_mask": mask, "turns": out}]}]} + + +def test_recorded_turns_carry_the_arrays_through_the_join(): + turns = _three_turns() + seg = join_turns(turns_from_records(_records(turns))) + ref = join_turns(turns) + assert numpy.array_equal(seg.response_top_ids, ref.response_top_ids) + assert numpy.array_equal(seg.routed_experts, ref.routed_experts) + assert json.loads(_records_dumps(_records(turns))) # the HTTP route serializes the arrays + + +def test_records_backend_records_engine_arrays_and_keeps_only_the_newest_r3(): + import asyncio + + from yeto.rl.adapters.sampler_engines.fake import FakeEngine + from yeto.rl.engine.sampler_engine import SamplingParams + from yeto.rl.harness.gateway.records import RecordingBackend, Segment, _Prompt, _Session + + engine = FakeEngine(token_reply=lambda prompt, i: [7, 8], routed_experts=(EXPERTS, LAYERS, TOPK), + top_logprobs_num=K) + engine.start(None, 3) + backend = object.__new__(RecordingBackend) + backend.engine, backend.sampler_id = engine, "records-proxy" + backend.template = SimpleNamespace(stop_token_ids=(), stop_text=lambda t: None) + backend.counters = {"t": __import__("collections").Counter()} + session, segment = _Session("t", []), Segment(0, head=[], base=0) + params = SamplingParams(max_tokens=4) + asyncio.run(backend._sample(session, segment, _Prompt(stop=(), tail=(1, 2, 3)), params, kind="action")) + asyncio.run(backend._sample(session, segment, _Prompt(stop=(), tail=(4, 5)), params, kind="action")) + first, second = segment.turns + assert first.routed_experts is None and first.top_ids.shape == (2, K) + assert second.routed_experts.shape == (len(segment.token_ids) - 1, LAYERS, TOPK) + seg = join_turns(turns_from_records({"sessions": [{"samples": [segment.sample()]}]})) + assert seg.response_top_ids.shape == (len(seg.response_ids), K) + + +def _sample(index=0): + seg = join_turns(_three_turns()) + task = AgentTask.from_mapping({"task_id": "t", "trajectory_id": f"t-{index}", "agent": "codex", "provider": "fake"}) + trajectory = AgentTrajectory(task=task, outcome=Outcome.OK, reward=float(index % 2), reason="ok", + verdict=None, tokens=seg) + finished = FinishedSample(trajectory=trajectory, segments=seg, turns=_three_turns()) + item = PromptItem(prompt_id="p", prompt="task", metadata={}) + return agentic_sample(item, finished, version=3, provenance={"engine": "fake"}, group_index=0, index=index, + engine_meta={"routed_experts": True, "top_logprobs_num": K}) + + +def test_agentic_sample_has_the_single_turn_fields_and_survives_the_buffer(): + sample = decode_group(encode_group([_sample()]))[0] + n = sample["response_length"] + assert sample[ROLLOUT_TOP_LOGPROBS_IDS].dtype == numpy.int32 and sample[ROLLOUT_TOP_LOGPROBS_IDS].shape == (n, K) + assert sample[ROLLOUT_TOP_LOGPROBS].dtype == numpy.float32 + assert sample[ROLLOUT_ROUTED_EXPERTS].dtype == numpy.uint16 + assert sample[ROLLOUT_ROUTED_EXPERTS].shape[0] == len(sample["tokens"]) - 1 + assert sample["metadata"]["engine"]["top_logprobs_num"] == K + miles = miles_sample_dict(sample, r3_layers=(LAYERS + 1, 1), r3_num_experts=EXPERTS) + assert miles[MILES_TOPK_TOKEN_IDS].shape == (n, K) + assert miles[ROLLOUT_ROUTED_EXPERTS].shape == (len(sample["tokens"]) - 1, LAYERS + 1, TOPK) + + +def test_miles_train_data_conversion_accepts_agentic_samples(): + tdc = pytest.importorskip("miles.ray.rollout.train_data_conversion") + from miles.rollout.generate_utils.rollout_topk_logprobs import validate_rollout_topk_logprobs_sample + from miles.utils.types import Sample + + samples = [] + for i in range(2): + d = miles_sample_dict(decode_group(encode_group([_sample(i)]))[0], r3_layers=(LAYERS, 0), + r3_num_experts=EXPERTS) + d["status"] = Sample.Status.COMPLETED.value + samples.append(Sample.from_dict(d)) + for s in samples: + validate_rollout_topk_logprobs_sample(s, K) + s.validate() + args = SimpleNamespace(advantage_estimator="grpo", rewards_normalization=True, grpo_std_normalization=True, + n_samples_per_prompt=2, rollout_batch_size=1, reward_key=None, + use_dynamic_global_batch_size=False, rollout_top_logprobs_num=K, ci_test=True, + use_output_store=False, use_rollout_routing_replay=True, + use_rollout_indexer_replay=False, length_penalty_coeff=0.0, balance_data=False, + custom_convert_samples_to_train_data_path=None, custom_reward_post_process_path=None, + custom_view=None, enable_sample_ownership_checker=False, global_batch_size=2, + loss_type="policy_loss", multi_lora_n_adapters=0) + data = tdc.convert_samples_to_train_data(args, samples, {}, None, None) + assert len(data["rollout_topk_token_ids"]) == 2 and len(data["rollout_routed_experts"]) == 2 + assert data["loss_masks"][0] == list(samples[0].loss_mask) diff --git a/tests/test_card_telemetry_rocm_fallback.py b/tests/test_card_telemetry_rocm_fallback.py new file mode 100644 index 000000000..7dd8b7184 --- /dev/null +++ b/tests/test_card_telemetry_rocm_fallback.py @@ -0,0 +1,99 @@ +"""ROCm card telemetry without amd-smi / pynvml: rocm-smi (short flag set), then the +amdgpu sysfs files; a cuda-family island on an AMD node falls back to that chain.""" + +from __future__ import annotations + +import json + +import pytest + +from yeto.hw import card_telemetry as ct +from yeto.rl.engine.telemetry import ResourceSampler + +ROCM_MIN = json.dumps({"card0": {"GPU use (%)": "37", "VRAM Total Memory (B)": str(288 * 2**30), + "VRAM Total Used Memory (B)": str(100 * 2**30)}, + "card1": {"GPU use (%)": "0", "VRAM Total Memory (B)": str(288 * 2**30), + "VRAM Total Used Memory (B)": str(2**30)}}) + + +def _runner(outputs): + calls = [] + + def run(argv): + calls.append(tuple(argv)) + out = outputs.get(tuple(argv)) + if out is None: + raise ct.CardTelemetryError(f"{argv[0]}: FileNotFoundError: not installed") + return out + + run.calls = calls + return run + + +def _sysfs(root, cards): + for i, (used, total, busy) in enumerate(cards): + dev = root / f"card{i}" / "device" + dev.mkdir(parents=True) + (dev / "mem_info_vram_used").write_text(f"{used}\n") + (dev / "mem_info_vram_total").write_text(f"{total}\n") + (dev / "gpu_busy_percent").write_text(f"{busy}\n") + (root / "card0-DP-1").mkdir() # a connector entry is skipped + return root + + +def test_rocm_smi_short_flag_set(tmp_path): + run = _runner({ct.RocmSmiCollector.ROCM_SMI_MIN_ARGV: ROCM_MIN}) + c = ct.rocm_collector(run, sysfs_root=str(tmp_path / "none")) + c.open() + assert c.source == "rocm-smi" + cards = c.sample() + assert [x["index"] for x in cards] == [0, 1] + assert cards[0]["mem_used_mb"] == 100 * 1024 and cards[0]["util_pct"] == 37.0 + assert ("rocm-smi", "--showmeminfo", "vram", "--showuse", "--json") in run.calls + + +def test_sysfs_when_no_smi_tool(tmp_path): + root = _sysfs(tmp_path / "drm", [(50 * 2**30, 288 * 2**30, 90), (2**30, 288 * 2**30, 0)]) + c = ct.rocm_collector(_runner({}), sysfs_root=str(root)) + c.open() + assert c.source == "sysfs-amdgpu" + cards = c.sample() + assert [x["mem_used_mb"] for x in cards] == [50 * 1024, 1024] + assert cards[0]["mem_total_mb"] == 288 * 1024 and cards[0]["util_pct"] == 90.0 + + +def test_nothing_available_names_every_attempt(tmp_path): + c = ct.rocm_collector(_runner({}), sysfs_root=str(tmp_path / "none")) + with pytest.raises(ct.CardTelemetryError) as e: + c.open() + assert "amd-smi" in str(e.value) and "rocm-smi" in str(e.value) and "sysfs-amdgpu" in str(e.value) + + +def test_cuda_family_without_pynvml_falls_back_to_rocm(tmp_path, monkeypatch): + def no_nvml(): + raise ModuleNotFoundError("No module named 'pynvml'") + + root = _sysfs(tmp_path / "drm", [(10 * 2**30, 288 * 2**30, 5)]) + monkeypatch.setattr(ct, "SYSFS_DRM", str(root)) + monkeypatch.setattr(ct, "rocm_collector", lambda run=ct.run_command, sysfs_root=str(root): + ct.FallbackCollector(ct.RocmSmiCollector(_runner({})), ct.SysfsAmdCollector(sysfs_root))) + monkeypatch.delenv("YETO_DEVICE_FAMILY", raising=False) + events = [] + s = ResourceSampler(lambda ev, **f: events.append(f), nvml_loader=no_nvml, interval_s=3600) + assert s.tick() + (rec,) = events + assert rec["available"] is True and rec["source"] == "sysfs-amdgpu" + assert rec["gpus"][0]["mem_used_mb"] == 10 * 1024 + + +def test_cuda_with_nvml_is_unchanged(): + class Nvml: + def nvmlInit(self): + pass + + def nvmlDeviceGetCount(self): + return 0 + + c = ct.collector_for("cuda", nvml_loader=lambda: Nvml(), run=_runner({})) + c.open() + assert c.source == "nvml" diff --git a/tests/test_codepack.py b/tests/test_codepack.py index f817216d6..7b7fa935a 100644 --- a/tests/test_codepack.py +++ b/tests/test_codepack.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess from pathlib import Path @@ -151,11 +152,63 @@ def test_resume_takes_the_manifest_pack_after_local_change(repo: Path, tmp_path: manifest = {"code_pack": first.pack.record(), "semantic_sha256": "s"} again = codepack.resolve_run_pack(previous=manifest, repo=repo, store=store, cache_root=tmp_path / "c2", semantic_sha256="s", - log=lambda _m: None) + keep_pinned=True, log=lambda _m: None) assert again.reused and again.pack == first.pack assert (again.tree / "yeto" / "a.py").read_text() == "A = 1\n" +def _git_commit(repo: Path, message: str) -> None: + subprocess.run(["git", "-C", str(repo), "commit", "-qam", message], check=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}) + + +def test_relaunch_after_a_code_change_refuses_the_stale_pack(repo: Path, tmp_path: Path) -> None: + store = DirPackStore(tmp_path / "store", uri_base="file-store://run") + first = codepack.resolve_run_pack(previous=None, repo=repo, store=store, + cache_root=tmp_path / "c1", semantic_sha256="s", + log=lambda _m: None) + manifest = {"code_pack": first.pack.record(), "semantic_sha256": "s"} + # an unchanged tree resumes on its pack without any flag + same = codepack.resolve_run_pack(previous=manifest, repo=repo, store=store, + cache_root=tmp_path / "c2", semantic_sha256="s", + log=lambda _m: None) + assert same.reused and same.pack == first.pack + # a new commit (and, separately, a dirty edit) invalidates the pinned pack + (repo / "yeto" / "a.py").write_text("A = 2\n") + with pytest.raises(codepack.CodePackError, match="dirty"): + codepack.resolve_run_pack(previous=manifest, repo=repo, store=store, + cache_root=tmp_path / "c3", semantic_sha256="s", log=lambda _m: None) + _git_commit(repo, "fix") + with pytest.raises(codepack.CodePackError, match="--rl-code-pack-keep"): + codepack.resolve_run_pack(previous=manifest, repo=repo, store=store, + cache_root=tmp_path / "c4", semantic_sha256="s", log=lambda _m: None) + moved = codepack.resolve_run_pack(previous=manifest, repo=repo, store=store, + cache_root=tmp_path / "c5", semantic_sha256="s", + upgrade_reason="fix", log=lambda _m: None) + assert not moved.reused and moved.pack.commit != first.pack.commit + assert (moved.tree / "yeto" / "a.py").read_text() == "A = 2\n" + + +def test_same_prefix_on_another_store_is_a_new_run(repo: Path, tmp_path: Path) -> None: + # The run manifest is keyed by cluster prefix; a fresh launch of that prefix on a + # new checkpoint store must ship the current tree, not the earlier run's pack. + old_store = DirPackStore(tmp_path / "old", uri_base="file-store://run-a") + first = codepack.resolve_run_pack(previous=None, repo=repo, store=old_store, + cache_root=tmp_path / "c1", semantic_sha256="s", + log=lambda _m: None) + (repo / "yeto" / "a.py").write_text("A = 2\n") + subprocess.run(["git", "-C", str(repo), "commit", "-qam", "change"], check=True) + new_store = DirPackStore(tmp_path / "new", uri_base="file-store://run-b") + manifest = {"code_pack": first.pack.record(), "semantic_sha256": "s"} + fresh = codepack.resolve_run_pack(previous=manifest, repo=repo, store=new_store, + cache_root=tmp_path / "c2", semantic_sha256="s", + log=lambda _m: None) + assert not fresh.reused and fresh.pack.digest != first.pack.digest + assert fresh.pack.pack_uri.startswith("file-store://run-b/") + assert (fresh.tree / "yeto" / "a.py").read_text() == "A = 2\n" + + def test_upgrade_needs_equal_semantic_hash(repo: Path, tmp_path: Path) -> None: store = DirPackStore(tmp_path / "store") first = codepack.resolve_run_pack(previous=None, repo=repo, store=store, diff --git a/tests/test_controller.py b/tests/test_controller.py index 96630e803..fa2c8267b 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -472,6 +472,21 @@ def test_stop_run_cli_only_writes_the_flag(tmp_path, monkeypatch, capsys): assert not (tmp_path / "myrun" / "meta.json").exists() # nothing else touched +def test_a_reused_run_name_starts_without_the_previous_runs_stop_flag(tmp_path, monkeypatch): + """A STOP flag left by an earlier run of the same name (stop-run, the cost guard's + budget/idle stop) must not end the new run at its first fleet poll.""" + from yeto import runs + + monkeypatch.setattr(runs, "RUNS_DIR", tmp_path) + runs.create_run("myrun", {}) + runs.request_stop("myrun") + assert runs.stop_flag_path("myrun").exists() + runs.create_run("myrun", {}) + assert not runs.stop_flag_path("myrun").exists() + runs.request_stop("myrun") # a stop of the new run still works + assert runs.stop_flag_path("myrun").exists() + + def test_launcher_wires_the_stop_flag_of_its_run(monkeypatch, tmp_path, _fake_sky_module): from yeto import runs diff --git a/tests/test_cost_guard.py b/tests/test_cost_guard.py new file mode 100644 index 000000000..be0b5f3ea --- /dev/null +++ b/tests/test_cost_guard.py @@ -0,0 +1,716 @@ +"""Cost protection: resource ledger, billing, budget breaker, idle shutdown, cleanup, report. + +All CPU: fake Modal billing, fake NC / Modal cleanup ops, fake clocks.""" + +from __future__ import annotations + +import json + +import pytest + +from yeto import cli, runs +from yeto.cost import cleanup as cleanup_mod +from yeto.cost import reaper +from yeto.cost.billing import BillingRow, ModalBilling, Rates, resource_costs +from yeto.cost.guard import CostGuard, GuardConfig, TapeTail, guard_config_from_args +from yeto.cost.idle import IdleConfig, IdleDetector +from yeto.cost.ledger import ResourceLedger +from yeto.cost.report import build_report, productive_seconds + +H = 3600.0 +T0 = 1_800_000_000.0 - (1_800_000_000.0 % H) # an hour boundary +MIN = 60.0 + + +@pytest.fixture +def runs_dir(tmp_path, monkeypatch): + monkeypatch.setattr(runs, "RUNS_DIR", tmp_path / "runs") + return tmp_path / "runs" + + +class Clock: + def __init__(self, t: float) -> None: + self.t = t + + def __call__(self) -> float: + return self.t + + +class FakeOps: + def __init__(self, apps_named=None, fail=()): + self.calls: list[tuple] = [] + self.apps_named = apps_named or {} + self.fail = set(fail) + + def _do(self, *a): + self.calls.append(a) + if a[1] in self.fail: + raise RuntimeError(f"boom {a[1]}") + + def modal_cancel_call(self, call_id): + self._do("cancel", call_id) + + def modal_stop_app(self, app_id): + self._do("stop_app", app_id) + + def modal_app_ids_named(self, name, created_after): + self.calls.append(("list", name)) + return list(self.apps_named.get(name, [])) + + def nc_release(self, account, ticket): + self._do("nc_down", ticket, account) + return {"ticket": ticket, "state": "ended"} + + def sky_down(self, cluster): + self._do("sky_down", cluster) + return True + + +def ledger_at(path, t): + return ResourceLedger(path, clock=Clock(t)) + + +# -- ledger ----------------------------------------------------------------------------- + +def test_ledger_folds_create_release_and_survives_a_torn_line(tmp_path): + p = tmp_path / "resources.jsonl" + ledger_at(p, T0).intent("modal_app", "yeto-r1") + ledger_at(p, T0 + 5).created("modal_app", "ap-1", "yeto-r1", gpu="B300", gpus=8) + ledger_at(p, T0 + 6).created("modal_call", "fc-1", "island", app_id="ap-1") + ledger_at(p, T0 + 7).created("nc_lease", "t-9", "nc", account=4) + ledger_at(p, T0 + 8).released("nc_lease", "t-9") + with open(p, "a") as fh: + fh.write('{"op": "create", "kind": "modal_ca') # SIGKILL mid-write + res = {r.key: r for r in ResourceLedger(p).resources()} + assert set(res) == {"modal_app:ap-1", "modal_call:fc-1", "nc_lease:t-9"} + assert res["modal_app:ap-1"].created_at == T0 # from the intent + assert res["modal_app:ap-1"].attrs["gpus"] == 8 + assert not res["nc_lease:t-9"].open + assert {r.key for r in ResourceLedger(p).open_resources()} == {"modal_app:ap-1", "modal_call:fc-1"} + + +def test_stopping_an_app_ends_its_calls_in_the_ledger(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("modal_app", "ap-1", "yeto-r1") + ledger_at(p, T0 + 1).created("modal_call", "fc-1", "i0", app_id="ap-1") + ledger_at(p, T0 + 1).created("modal_call", "fc-2", "i1", app_id="ap-OTHER") + ledger_at(p, T0 + 9).released("modal_app", "ap-1", "yeto-r1") + assert [r.key for r in ResourceLedger(p).open_resources()] == ["modal_call:fc-2"] + + +def test_unknown_kind_is_refused(tmp_path): + with pytest.raises(ValueError): + ResourceLedger(tmp_path / "x").created("vm", "i-1") + + +# -- billing ---------------------------------------------------------------------------- + +def test_modal_cost_is_matched_by_exact_app_id_and_the_unbilled_tail_estimated(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0 + 0.5 * H).created("modal_app", "ap-mine", "yeto-k31-sampler-b300", usd_per_hour=60.0) + rows = [ + BillingRow("ap-mine", "yeto-k31-sampler-b300", T0, 30.0), + BillingRow("ap-mine", "yeto-k31-sampler-b300", T0 + H, 60.0), + # same NAME, another run's app: never counted + BillingRow("ap-theirs", "yeto-k31-sampler-b300", T0 + H, 999.0), + # this app, but an hour before it was this run's + BillingRow("ap-mine", "yeto-k31-sampler-b300", T0 - H, 500.0), + ] + billing = ModalBilling(lambda s, e: rows) + s = resource_costs(ResourceLedger(p).resources(), T0 + 2.5 * H, billing=billing) + (item,) = s.items + assert item.billed_usd == 90.0 + assert item.estimated_usd == pytest.approx(30.0) # 0.5 h after the newest billed hour x $60 + assert s.total_usd == pytest.approx(120.0) + assert set(item.hours) and sum(item.hours.values()) == pytest.approx(120.0) + + +def test_rate_falls_back_to_the_newest_complete_billed_hour_and_billing_errors_keep_estimating(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("modal_app", "ap-1", "a") + good = [BillingRow("ap-1", "a", T0, 50.0)] + state = {"fail": False} + + def fetch(s, e): + if state["fail"]: + raise ConnectionError("billing down") + return good + + billing = ModalBilling(fetch, refresh_s=0) + s = resource_costs(ResourceLedger(p).resources(), T0 + 1.5 * H, billing=billing) + assert s.total_usd == pytest.approx(75.0) + state["fail"] = True + s = resource_costs(ResourceLedger(p).resources(), T0 + 2 * H, billing=billing) + assert s.billing_error and s.total_usd == pytest.approx(100.0) + + +def test_nc_lease_keeps_billing_past_lease_until(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("nc_lease", "t-1", "nc", account=4, granted_at=T0, lease_until=T0 + 4 * H, + gpus=8, rate_ucr_per_gpu_hour=3_000_000) + rates = Rates(nc_usd_per_ucr=1e-6) # $3 / GPU-hour + s = resource_costs(ResourceLedger(p).resources(), T0 + 6 * H, rates=rates) + (item,) = s.items + assert item.usd == pytest.approx(6 * 8 * 3.0) + assert "past lease_until" in item.note + + +def test_a_resource_without_a_rate_is_unpriced_not_zero(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("nc_lease", "t-1", "nc", account=4, rate_ucr_per_gpu_hour=3_000_000) + s = resource_costs(ResourceLedger(p).resources(), T0 + H, + rates=Rates(nc_usd_per_ucr=None, nc_ucr_per_gpu_hour=None)) + assert [i.key for i in s.unpriced] == ["nc_lease:t-1"] and s.items[0].usd is None + + +def test_an_nc_lease_is_priced_by_default_at_about_24_usd_an_hour(tmp_path): + """F: an NC lease with no rate in the ledger still counts against the budget.""" + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("nc_lease", "t-1", "nc", account=4) + s = resource_costs(ResourceLedger(p).resources(), T0 + 2 * H, rates=Rates()) + assert not s.unpriced and s.items[0].usd == pytest.approx(48.0) + env = {"YETO_NC_USD_PER_UCR": "2e-6", "YETO_NC_UCR_PER_GPU_HOUR": "1000000"} + s = resource_costs(ResourceLedger(p).resources(), T0 + H, rates=Rates.from_env(environ=env)) + assert s.items[0].usd == pytest.approx(16.0) # 8 GPUs x 1e6 ucr x 2e-6 USD + assert Rates.from_env(environ={}).nc_usd_per_ucr == 1e-6 + + +def test_the_guard_budget_counts_the_nc_lease(tmp_path): + clock = Clock(T0 + 3 * H) + g, led, calls = make_guard(tmp_path, clock, budget=50.0) + ledger_at(led.path, T0).created("nc_lease", "t-1", "nc", account=4) + g.tick() + assert "cost_budget_exceeded" in events(tmp_path) and calls["graceful"] + + +# -- idle detector ------------------------------------------------------------------------ + +def idle(start=T0, **kw): + cfg = IdleConfig(**{"idle_after_s": 15 * MIN, "stop_after_s": 15 * MIN, "load_grace_s": 45 * MIN, + "max_load_s": 120 * MIN, **kw}) + return IdleDetector(cfg, start) + + +def test_weight_loading_with_zero_gpu_and_no_progress_is_not_idle(): + d = idle() + d.observe({"event": "rl_driver_start"}, T0 + MIN) + for m in range(0, 25): + d.observe({"event": "modal_host_sample", "gpu_util_pct": [0, 0]}, T0 + m * MIN) + assert d.verdict(T0 + 22 * MIN).state == "loading" + assert d.verdict(T0 + 40 * MIN).state == "loading" + # loaded, training + d.observe({"event": "rl_round_trained"}, T0 + 41 * MIN) + d.observe({"event": "modal_host_sample", "gpu_util_pct": [97, 95]}, T0 + 42 * MIN) + assert d.verdict(T0 + 50 * MIN).state == "ok" + + +def test_no_progress_after_loading_warns_then_stops_and_progress_resets(): + d = idle() + d.observe({"event": "rl_round_trained"}, T0 + 50 * MIN) + d.observe({"event": "rl_heartbeat"}, T0 + 60 * MIN) # heartbeats are no progress + assert d.verdict(T0 + 60 * MIN).state == "ok" + v = d.verdict(T0 + 66 * MIN) + assert v.state == "warn" and "no progress" in v.reason + assert d.verdict(T0 + 81 * MIN).state == "stop" + d.observe({"event": "rl_sampler_round"}, T0 + 81 * MIN) + assert d.verdict(T0 + 82 * MIN).state == "ok" + + +def test_all_gpus_at_zero_is_idle_even_with_events_but_one_busy_gpu_is_not(): + d = idle() + d.observe({"event": "rl_round_trained"}, T0 + 46 * MIN) + for m in range(46, 80): + d.observe({"event": "rl_sampler_round"}, T0 + m * MIN) + d.observe({"event": "modal_host_sample", "island": "s0", "gpu_util_pct": [0]}, T0 + m * MIN) + d.observe({"event": "modal_host_sample", "island": "t0", "gpu_util_pct": [80]}, T0 + m * MIN) + assert d.verdict(T0 + 80 * MIN).state == "ok" # the trainer GPU is busy + for m in range(80, 100): + d.observe({"event": "rl_sampler_round"}, T0 + m * MIN) + d.observe({"event": "modal_host_sample", "island": "t0", "gpu_util_pct": [0]}, T0 + m * MIN) + v = d.verdict(T0 + 99 * MIN) + assert v.state == "warn" and "GPU utilization 0" in v.reason + + +def test_a_new_loading_phase_mid_run_is_exempt_but_bounded(): + d = idle() + d.observe({"event": "rl_round_trained"}, T0 + 50 * MIN) + # trainer incarnation restarts and reloads weights for 30 min + d.observe({"event": "rl_trainer_lifecycle", "phase": "submitted"}, T0 + 55 * MIN) + assert d.verdict(T0 + 85 * MIN).state == "loading" + # a loader stuck in a restart loop keeps emitting loading events: bounded by max_load_s + for m in range(55, 300, 10): + d.observe({"event": "rl_startup_step"}, T0 + m * MIN) + assert d.verdict(T0 + 160 * MIN).state == "loading" + v = d.verdict(T0 + 50 * MIN + 120 * MIN + 16 * MIN) + assert v.state == "warn" + + +def test_idle_detection_disabled_with_zero_minutes(): + d = idle(idle_after_s=0) + assert d.verdict(T0 + 100 * H).state == "ok" + + +# -- guard: budget ------------------------------------------------------------------------ + +def make_guard(tmp_path, clock, *, budget=100.0, warn=None, ended=lambda: False, idle_cfg=None, + rows=None, tapes=()): + led = ResourceLedger(tmp_path / "resources.jsonl", clock=clock) + calls = {"graceful": [], "force": [], "release": []} + g = CostGuard("r1", GuardConfig(budget_usd=budget, budget_warn_usd=warn, stop_grace_s=20 * MIN, + idle=idle_cfg or IdleConfig(idle_after_s=0)), + ledger=led, tapes=TapeTail(list(tapes)), + billing=ModalBilling(lambda s, e: list(rows or []), refresh_s=0), + graceful_stop=lambda why: calls["graceful"].append(why), + force_stop=lambda why: calls["force"].append(why), + release_resource=lambda r: calls["release"].append(r.key), + run_ended=ended, clock=clock, log=lambda m: None, start=T0) + return g, led, calls + + +def events(tmp_path): + p = tmp_path / "cost-events.jsonl" + return [json.loads(x)["event"] for x in p.read_text().splitlines()] if p.exists() else [] + + +def test_budget_warns_once_then_stops_gracefully_then_forces(tmp_path): + clock = Clock(T0) + g, led, calls = make_guard(tmp_path, clock, budget=100.0) + led.created("modal_app", "ap-1", "yeto-r1", usd_per_hour=60.0) + clock.t = T0 + 1.0 * H + g.tick() + assert "cost_budget_warn" not in events(tmp_path) + clock.t = T0 + 1.4 * H # $84 >= 80 % of $100 + g.tick() + g.tick() + assert events(tmp_path).count("cost_budget_warn") == 1 and not calls["graceful"] + clock.t = T0 + 1.7 * H # $102 + g.tick() + assert len(calls["graceful"]) == 1 and "budget" in calls["graceful"][0] + assert "cost_budget_exceeded" in events(tmp_path) + clock.t += 10 * MIN + g.tick() + assert not calls["force"] and len(calls["graceful"]) == 1 + clock.t += 11 * MIN # 21 min after the graceful stop, still up + g.tick() + g.tick() + assert len(calls["force"]) == 1 + + +def test_a_reused_run_name_does_not_count_the_earlier_launchs_released_spend(tmp_path): + """Relaunching a run name after a budget stop must not stop the new launch at its + first tick: resources released before this guard started are not its spend; one + still open (left by the earlier launch) bills now and counts.""" + clock = Clock(T0 + H) + g, led, calls = make_guard(tmp_path, clock, budget=50.0) + early = ledger_at(led.path, T0 - 3 * H) + early.created("modal_app", "ap-old", "yeto-r1", usd_per_hour=60.0) + ledger_at(led.path, T0 - H).released("modal_app", "ap-old", "yeto-r1") # $120, before T0 + s = g.tick() + assert s.total_usd == pytest.approx(0.0) and not calls["graceful"] + ledger_at(led.path, T0 - H).created("modal_app", "ap-left", "yeto-r1", usd_per_hour=30.0) + s = g.tick() # still open: 2 h at $30 + assert s.total_usd == pytest.approx(60.0) and calls["graceful"] + + +def test_no_forced_stop_once_the_run_has_ended(tmp_path): + clock = Clock(T0) + ended = {"v": False} + g, led, calls = make_guard(tmp_path, clock, budget=10.0, ended=lambda: ended["v"]) + led.created("modal_app", "ap-1", "yeto-r1", usd_per_hour=60.0) + clock.t = T0 + H + g.tick() + ended["v"] = True + clock.t += H + g.tick() + assert calls["graceful"] and not calls["force"] + + +def test_budget_counts_billing_rows_and_nc_leases_together(tmp_path): + clock = Clock(T0 + 2 * H) + rows = [BillingRow("ap-1", "x", T0, 40.0), BillingRow("ap-1", "x", T0 + H, 40.0)] + g, led, calls = make_guard(tmp_path, clock, budget=100.0, rows=rows) + ledger_at(led.path, T0).created("modal_app", "ap-1", "x", usd_per_hour=40.0) + ledger_at(led.path, T0).created("nc_lease", "t-1", "nc", account=4, granted_at=T0, usd_per_hour=12.0) + s = g.tick() + assert s.total_usd == pytest.approx(80 + 24) and calls["graceful"] + + +def test_graceful_stop_failure_forces_at_once(tmp_path): + clock = Clock(T0 + H) + g, led, calls = make_guard(tmp_path, clock, budget=1.0) + + def bad(why): + raise OSError("no run dir") + + g.graceful_stop = bad + ledger_at(led.path, T0).created("modal_app", "ap-1", "x", usd_per_hour=10.0) + g.tick() + assert calls["force"] + + +def test_unpriced_resource_is_reported_once(tmp_path): + clock = Clock(T0 + H) + g, led, _ = make_guard(tmp_path, clock, budget=None) + ledger_at(led.path, T0).created("sky_cluster", "c-1", "nc") # no rate anywhere + g.tick() + g.tick() + assert events(tmp_path).count("cost_unpriced") == 1 + + +def test_overdue_nc_lease_is_reported_not_released_by_default(tmp_path): + """The NC pool owns its leases: a trainer's cost guard only alerts on an overdue one.""" + clock = Clock(T0 + 4 * H + 6 * MIN) + g, led, calls = make_guard(tmp_path, clock, budget=None) + ledger_at(led.path, T0).created("nc_lease", "t-1", "nc", account=4, lease_until=T0 + 4 * H) + g.tick() + g.tick() + assert calls["release"] == [] and calls["force"] == [] + assert events(tmp_path).count("nc_lease_overdue") == 1 + + +def test_overdue_nc_lease_is_released_once_when_opted_in(tmp_path): + clock = Clock(T0 + 4 * H + 2 * MIN) + g, led, calls = make_guard(tmp_path, clock, budget=None) + g.config.release_overdue_nc = True + ledger_at(led.path, T0).created("nc_lease", "t-1", "nc", account=4, lease_until=T0 + 4 * H) + ledger_at(led.path, T0).created("nc_lease", "t-2", "nc", account=5, lease_until=T0 + 8 * H) + g.tick() + assert not calls["release"] # within the 5 min grace + clock.t = T0 + 4 * H + 6 * MIN + g.tick() + g.tick() + assert calls["release"] == ["nc_lease:t-1"] + assert "nc_lease_overdue" in events(tmp_path) + + +# -- guard: idle via tapes ----------------------------------------------------------------- + +def test_guard_reads_tapes_and_stops_an_idle_run_but_not_a_loading_one(tmp_path): + tape = tmp_path / "events" / "island-0.jsonl" + tape.parent.mkdir() + clock = Clock(T0) + g, led, calls = make_guard(tmp_path, clock, budget=None, tapes=[str(tmp_path / "events" / "*.jsonl")], + idle_cfg=IdleConfig(idle_after_s=15 * MIN, stop_after_s=15 * MIN, + load_grace_s=45 * MIN)) + + def write(rec): + with open(tape, "a") as fh: + fh.write("YETO_RL_EVENT " + json.dumps(rec) + "\n") + + write({"event": "rl_driver_start", "time_unix": T0 + MIN}) + for m in range(0, 44, 2): # 22-min weight load, silent; then waiting + clock.t = T0 + m * MIN + g.tick() + assert not calls["graceful"] and "idle_warn" not in events(tmp_path) + write({"event": "rl_round_trained", "time_unix": T0 + 44 * MIN}) + clock.t = T0 + 50 * MIN + g.tick() + clock.t = T0 + 60 * MIN # 16 min without progress + g.tick() + assert "idle_warn" in events(tmp_path) and not calls["graceful"] + clock.t = T0 + 75 * MIN # 31 min + g.tick() + assert "idle_shutdown" in events(tmp_path) and len(calls["graceful"]) == 1 + + +def test_tape_tail_skips_a_partial_last_line_until_it_completes(tmp_path): + p = tmp_path / "t.jsonl" + p.write_text('{"event": "a"}\n{"event": "b"') + t = TapeTail([str(p)]) + assert [r["event"] for _s, r in t.read()] == ["a"] + with open(p, "a") as fh: + fh.write("}\n") + assert [r["event"] for _s, r in t.read()] == ["b"] + assert t.read() == [] + + +# -- cleanup ------------------------------------------------------------------------------ + +def test_cleanup_releases_exactly_the_ledger_resources_of_this_run(runs_dir): + mine = ResourceLedger.for_run("run-a") + other = ResourceLedger.for_run("run-b") + mine.created("modal_app", "ap-a", "yeto-k31-sampler-b300") + mine.created("modal_call", "fc-a", "sampler", app_id="ap-a") + mine.created("nc_lease", "t-a", "nc", account=4) + mine.created("sky_cluster", "run-a-syncer") + mine.created("modal_call", "fc-done", "x", app_id="ap-a") + mine.released("modal_call", "fc-done") + other.created("modal_app", "ap-b", "yeto-k31-sampler-b300") # same app name, other run + other.created("nc_lease", "t-b", "nc", account=5) + ops = FakeOps() + assert cleanup_mod.cleanup_run("run-a", ops, include_nc=True, log=lambda m: None) + assert ops.calls == [("cancel", "fc-a"), ("stop_app", "ap-a"), ("nc_down", "t-a", 4), + ("sky_down", "run-a-syncer")] + assert not mine.open_resources() + assert len(other.open_resources()) == 2 + # rerun: nothing left to do + ops2 = FakeOps() + assert cleanup_mod.cleanup_run("run-a", ops2, include_nc=True, log=lambda m: None) and ops2.calls == [] + + +def test_cleanup_failure_is_recorded_retried_and_does_not_block_the_rest(runs_dir): + led = ResourceLedger.for_run("r") + led.created("modal_call", "fc-1", "i") + led.created("nc_lease", "t-1", "nc", account=4) + ops = FakeOps(fail={"fc-1"}) + assert not cleanup_mod.cleanup_run("r", ops, include_nc=True, log=lambda m: None) + assert ("nc_down", "t-1", 4) in ops.calls + (left,) = led.open_resources() + assert left.key == "modal_call:fc-1" and "boom" in left.release_error + assert cleanup_mod.cleanup_run("r", FakeOps(), include_nc=True, log=lambda m: None) + + +def test_cleanup_dry_run_changes_nothing(runs_dir): + led = ResourceLedger.for_run("r") + led.created("modal_app", "ap-1", "a") + ops = FakeOps() + assert cleanup_mod.cleanup_run("r", ops, include_nc=True, dry_run=True, log=lambda m: None) + assert ops.calls == [] and len(led.open_resources()) == 1 + + +def test_intent_is_resolved_only_by_one_exact_name_match(runs_dir): + led = ResourceLedger.for_run("r") + led.intent("modal_app", "yeto-r") # crashed during deploy + ops = FakeOps(apps_named={"yeto-r": ["ap-9"], "yeto-r-2": ["ap-x"]}) + assert cleanup_mod.cleanup_run("r", ops, include_nc=True, log=lambda m: None) + assert ("stop_app", "ap-9") in ops.calls and ("stop_app", "ap-x") not in ops.calls + led2 = ResourceLedger.for_run("r2") + led2.intent("modal_app", "app-x") + ops = FakeOps(apps_named={"app-x": ["ap-1", "ap-2"]}) + assert not cleanup_mod.cleanup_run("r2", ops, include_nc=True, log=lambda m: None) + assert not [c for c in ops.calls if c[0] == "stop_app"] + led3 = ResourceLedger.for_run("r3") + led3.intent("modal_app", "yeto-r3") # deploy never created anything + assert cleanup_mod.cleanup_run("r3", FakeOps(), include_nc=True, log=lambda m: None) + assert not led3.open_resources() + + +def test_shared_app_is_not_stopped_only_its_calls_cancelled(runs_dir): + led = ResourceLedger.for_run("r") + led.created("modal_app", "ap-s", "shared", shared=True) + led.created("modal_call", "fc-1", "i", app_id="ap-s") + ops = FakeOps() + assert cleanup_mod.cleanup_run("r", ops, include_nc=True, log=lambda m: None) + assert ops.calls == [("cancel", "fc-1")] + + +def test_production_ops_refuse_a_non_id_app(): + with pytest.raises(ValueError): + cleanup_mod.ProductionCleanupOps().modal_stop_app("yeto-*") + + +# -- launcher crash: reaper ------------------------------------------------------------------ + +def test_reaper_cleans_up_after_a_killed_launcher(runs_dir): + runs.create_run("r", {"cluster_prefix": "r"}) + runs.update_run("r", pid=4242, state=runs.RUNNING) # SIGKILLed: state never updated + led = ResourceLedger.for_run("r") + led.created("modal_app", "ap-1", "yeto-r") + led.created("modal_call", "fc-1", "i", app_id="ap-1") + ResourceLedger.for_run("bystander").created("modal_app", "ap-2", "yeto-r") + alive = iter([True, True, False]) + ops = FakeOps() + code = reaper.reap("r", 4242, alive=lambda pid: next(alive, False), sleep=lambda s: None, + cleanup=lambda run: cleanup_mod.cleanup_run(run, ops, log=lambda m: None), + log=lambda m: None) + assert code == 0 + assert ops.calls == [("cancel", "fc-1"), ("stop_app", "ap-1")] + assert len(ResourceLedger.for_run("bystander").open_resources()) == 1 + + +def test_reaper_leaves_keep_runs_and_clean_runs_alone(runs_dir): + runs.create_run("k", {"cluster_prefix": "k", "keep": True}) + ResourceLedger.for_run("k").created("modal_app", "ap-1", "yeto-k") + called = [] + reaper.reap("k", 1, alive=lambda p: False, cleanup=lambda r: called.append(r), log=lambda m: None) + runs.create_run("c", {"cluster_prefix": "c"}) + led = ResourceLedger.for_run("c") + led.created("modal_app", "ap-2", "yeto-c") + led.released("modal_app", "ap-2") + reaper.reap("c", 1, alive=lambda p: False, cleanup=lambda r: called.append(r), log=lambda m: None) + assert called == [] + + +def test_worker_backstop_cleans_ledger_left_overs_after_a_launcher_exception(runs_dir, monkeypatch): + runs.create_run("w", {"cluster_prefix": "w"}) + ResourceLedger.for_run("w").created("modal_call", "fc-1", "i") + import yeto.launcher as launcher + + def boom(args, **kw): + raise RuntimeError("launcher crashed") + + monkeypatch.setattr(launcher, "run", boom) + seen = [] + monkeypatch.setattr(cleanup_mod, "cleanup_run", lambda run, *a, **k: seen.append(run) or True) + assert cli.cmd_worker("w") == 1 + assert seen == ["w"] + + +# -- report & CLI ----------------------------------------------------------------------------- + +def test_effective_utilization_counts_only_close_progress_gaps(): + recs = [("t", {"event": "rl_round_trained", "time_unix": T0 + m * MIN}) for m in (0, 5, 10, 40, 45)] + prod, _ = productive_seconds(recs, gap_s=10 * MIN) + assert prod["t"] == pytest.approx(15 * MIN) # 0-5-10 and 40-45; the 30-min gap is idle + + +def test_cost_report_per_resource_per_hour_and_utilization(tmp_path): + p = tmp_path / "r.jsonl" + ledger_at(p, T0).created("modal_app", "ap-1", "a", usd_per_hour=10.0) + ResourceLedger(p, clock=Clock(T0 + 2 * H)).released("modal_app", "ap-1") + tape = tmp_path / "t.jsonl" + tape.write_text("".join(json.dumps({"event": "rl_round_trained", "time_unix": T0 + m * MIN}) + "\n" + for m in range(0, 61, 5))) + s = resource_costs(ResourceLedger(p).resources(), T0 + 3 * H) + rep = build_report("r", summary=s, tape_globs=[str(tape)]) + assert rep["total_usd"] == 20.0 and len(rep["by_hour"]) == 2 + assert rep["effective_utilization"] == pytest.approx(0.5) + + +def test_cli_parses_cost_cleanup_and_launch_guard_flags(runs_dir, capsys, monkeypatch): + p = cli.build_parser() + a = p.parse_args(["cost", "--run", "r", "--json", "--no-billing"]) + assert a.command == "cost" and a.no_billing + a = p.parse_args(["cleanup", "--run", "r", "--dry-run"]) + assert a.command == "cleanup" and a.dry_run + a = p.parse_args(["launch", "--model", "m", "--data", "d", "--gpu", "H100:8", "--budget-usd", "500"]) + cfg = guard_config_from_args(a) + assert cfg.budget_usd == 500 and cfg.warn_usd == 400 + assert cfg.idle.idle_after_s == 15 * MIN and cfg.idle.stop_after_s == 15 * MIN + assert cfg.idle.load_grace_s == 45 * MIN and cfg.stop_grace_s == 20 * MIN + led = ResourceLedger.for_run("r") + ResourceLedger(led.path, clock=Clock(T0)).created("modal_app", "ap-1", "a", usd_per_hour=6.0) + monkeypatch.setattr("time.time", lambda: T0 + H) + assert cli.main(["cost", "--run", "r", "--no-billing", "--json"]) == 0 + out = json.loads(capsys.readouterr().out) + assert out["total_usd"] == 6.0 + ops = FakeOps() + monkeypatch.setattr(cleanup_mod, "ProductionCleanupOps", lambda: ops) + assert cli.main(["cleanup", "--run", "r"]) == 0 and ops.calls == [("stop_app", "ap-1")] + + +def test_modal_ops_record_deploy_spawn_cancel_and_stop(tmp_path, monkeypatch): + from yeto import modal_runner + + led = ResourceLedger(tmp_path / "r.jsonl") + + class Call: + object_id = "fc-7" + + class Fn: + def spawn(self, payload): + return Call() + + def cancel(self, **kw): + pass + + class App: + app_id = "ap-7" + + def deploy(self, name): + pass + + class Ctx: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + class FakeModal: + class Function: + @staticmethod + def from_name(app, fn): + return Fn() + + class FunctionCall: + @staticmethod + def from_id(i): + return Fn() + + @staticmethod + def enable_output(): + return Ctx() + + ops = modal_runner.ModalOps("yeto-r") + ops.ledger = led + ops._app = App() + ops._functions = {"island": object()} + monkeypatch.setattr(ops, "_modal", lambda: FakeModal) + + class Cfg: + function_name = "island" + learner_id = 0 + + def to_json(self): + return {} + + ops.deploy() + assert ops.spawn(Cfg()) == "fc-7" + assert {r.key for r in led.open_resources()} == {"modal_app:ap-7", "modal_call:fc-7"} + ops.cancel("fc-7") + assert {r.key for r in led.open_resources()} == {"modal_app:ap-7"} + + class Proc: + returncode = 0 + stdout = "stopped" + stderr = "" + + monkeypatch.setattr(modal_runner.subprocess, "run", lambda *a, **k: Proc()) + ops.stop_app() + assert led.open_resources() == [] + + +def test_engine_sampler_launch_records_its_ephemeral_app_and_call(tmp_path, monkeypatch): + from yeto.cloud import modal_engine_sampler as mes + from yeto.cost.ledger import LEDGER_ENV + + led_path = tmp_path / "resources.jsonl" + monkeypatch.setenv(LEDGER_ENV, str(led_path)) + + class Call: + object_id = "fc-s" + + class Fn: + def spawn(self, rec): + return Call() + + class App: + app_id = "ap-s" + + def run(self, detach): + import contextlib + + return contextlib.nullcontext() + + monkeypatch.setattr(mes, "build_app", lambda c: (App(), Fn())) + + class C: + app_name, gpu, tp, shard, trainer_buffer_shards = "yeto-k31-sampler-b300", "B300", 8, "s0", "x" + + def record(self): + return {} + + assert mes.launch(C(), wait=False, log=lambda m: None) == "fc-s" + res = {r.key: r for r in ResourceLedger(led_path).open_resources()} + assert set(res) == {"modal_app:ap-s", "modal_call:fc-s"} + assert res["modal_app:ap-s"].attrs["gpus"] == 8 + + +def test_automatic_cleanup_keeps_the_nc_pool_lease(tmp_path, monkeypatch): + """A trainer launch's backstop / guard cleanup must not release the NC pool's node.""" + monkeypatch.setenv("HOME", str(tmp_path)) + from yeto.cost import cleanup as cleanup_mod + from yeto.cost.ledger import ResourceLedger + + led = ResourceLedger.for_run("r-nc") + led.created("nc_lease", "t1", "r-nc", account=5) + calls = [] + + class Ops: + def release_nc(self, r): + calls.append(r.id) + return {"released": True} + + assert cleanup_mod.cleanup_run("r-nc", Ops(), log=lambda m: None) + assert calls == [] and [r.kind for r in led.open_resources()] == ["nc_lease"] diff --git a/tests/test_dashboard_async_grpo.py b/tests/test_dashboard_async_grpo.py new file mode 100644 index 000000000..2f1142de5 --- /dev/null +++ b/tests/test_dashboard_async_grpo.py @@ -0,0 +1,367 @@ +"""Async GRPO on the dashboard: every panel of a sample-buffer run has data. + +One synthetic run (2 buffer trainers, 2 samplers, the NC pool, fleet and an +NVML resource sample) is folded into the reducer; each test reads one panel's +inputs from ``full_view`` (the exported page's data) and requires them non-empty. +Status rules: loading is not stale, a sampler is judged by its own cadence, an +island that left is not stale, samplers finish with their trainers.""" + +from __future__ import annotations + +import json + +import pytest + +from yeto.dashboard.export import export_view, render_page +from yeto.dashboard.reducer import Reducer +from yeto.dashboard.sources import TapeSource, load_all + +T0 = 1_790_000_000.0 +ROUNDS = 6 + + +def trainer_records(iid: int, *, t0: float = T0 + 100, rounds: int = ROUNDS) -> list[dict]: + """A buffer trainer: driver phases, the per-round events a buffer step writes, + one release per round (``buffer_selected`` = its read position).""" + out = [{"event": "rl_driver_start", "island_id": iid, "time_unix": t0}] + t = t0 + for rid in range(rounds): + out.append({"event": "rl_driver_phase", "island_id": iid, "phase": "generate", "rollout_id": rid, + "policy_version": rid, "time_unix": t}) + out.append({"event": "rl_buffer_stale_skipped", "island_id": iid, "rollout_id": rid, + "stale_groups": 1 if rid == 3 else 0, "max_policy_age": 2, "time_unix": t + 20, + "incomplete_groups": 1, "dropped_small_groups": 0, "carried_over_groups": 1 if rid == 2 else 0, + "padded_samples": 2 if rid == 4 else 0}) + out.append({"event": "rl_group_policies", "island_id": iid, "rollout_id": rid, + "ages": {"0": 4, "1": 3, "2": 1}, "merged_version": rid, "time_unix": t + 20}) + out.append({"event": "rl_driver_phase", "island_id": iid, "phase": "train", "rollout_id": rid, + "time_unix": t + 20}) + out.append({"event": "rl_round_trained", "island_id": iid, "rollout_id": rid, "time_unix": t + 40, + "trained_groups": 8, "trained_samples": 32, "reward_mean": 0.3 + 0.05 * rid, + "reward_std": 0.4, "reward_p10": 0.0, "reward_p50": 0.25, "reward_p90": 1.0, + "grad_norm": 0.5 + 0.01 * rid, "mean_kl": 0.001 * (rid + 1), "tok_per_s": 600.0, + "resp_len_mean": 220.0, "resp_len_p95": 400.0, "truncated_frac": 0.0625, + "mismatch": {"tis_clipfrac": 0.002, "train_rollout_kl": 0.001 * (rid + 1)}, + "train_metrics": {"ppo_kl": 0.0, "train_rollout_logprob_abs_diff": 0.013, + "pg_clipfrac": 0.0}}) + out.append({"event": "offpolicy_gate", "island_id": iid, "rollout_id": rid, "time_unix": t + 40, + "counts": {"group_dropped": 0, "kept": 8, "unknown_version_dropped": 0}}) + out.append({"event": "rl_driver_phase", "island_id": iid, "phase": "sync", "rollout_id": rid, + "time_unix": t + 40}) + out.append({"event": "rl_driver_phase", "island_id": iid, "phase": "publish", + "policy_version": rid + 1, "time_unix": t + 42}) + if iid == 0: + out.append({"event": "rl_policy_released", "island_id": iid, "policy_version": rid + 1, + "released_at": t + 44, "buffer_selected": 9 * (rid + 1), "release_bytes": 10, + "release_timings": {"upload_s": 2.0}, "origin": {"kind": "merged"}, + "time_unix": t + 45}) + out.append({"event": "rl_publication", "island_id": iid, "policy_version": rid + 1, "time_unix": t + 45}) + out.append({"event": "rl_round_phases", "island_id": iid, "rollout_id": rid, "time_unix": t + 45, + "phase_seconds": {"generate": 20, "train": 20, "sync": 2, "publish": 3}, "wall_seconds": 45}) + t += 45 + out.append({"event": "rl_learner_finalized", "island_id": iid, "time_unix": t + 1}) + return out + + +def sampler_records(name: str, *, t0: float = T0 + 10, rounds: int = ROUNDS + 2) -> list[dict]: + """An NC engine sampler: no island_id, only ``sampler_id`` (as on the A cards).""" + out = [{"event": "rl_sampler_start", "sampler_id": name, "max_backlog_groups": 24, "time_unix": t0, + "sampling": {"n": 4, "max_tokens": 1024, "temperature": 1.0}}, + {"event": "rl_resource_sample", "sampler_id": name, "available": False, + "reason": "ModuleNotFoundError: No module named 'pynvml'", "time_unix": t0 + 1}] + t = t0 + 30 + for rid in range(rounds): + out.append({"event": "rl_sampler_engine_round", "sampler_id": name, "rollout_id": rid, + "policy_version": rid // 2, "groups": 4, "samples": 16, "response_tokens": 3200, + "finish": {"length": 1, "stop": 15}, "dropped_groups": 0, "zero_variance_groups": 1, + "reward_mean": 0.4, "reward_p10": 0.0, "reward_p50": 0.5, "reward_p90": 1.0, + "zero_variance_seen": 1, "truncated": 1, "time_unix": t + 30, + "agentic": {"retried_trials": 2, "lost_trials": 1}}) + out.append({"event": "rl_sampler_round", "sampler_id": name, "rollout_id": rid, + "policy_version": rid // 2, "version_lag": rid % 2, "groups": 4, "buffer_end": 4 * (rid + 1), + "generate_s": 30.0, "time_unix": t + 30}) + if rid % 2 == 1: + released = T0 + 144 + 45 * (rid // 2) # trainer 0's release of this version + out.append({"event": "rl_sampler_policy_loaded", "sampler_id": name, "policy_version": rid // 2 + 1, + "released_at": released, "seen_at": released + 10, "buffer_selected": 9, + "follower_timings": {"resolve_s": 5.0, "load_s": 3.0}, "time_unix": released + 20}) + t += 35 + out.append({"event": "rl_sampler_backlog_full", "sampler_id": name, "backlog_groups": 25, + "max_backlog_groups": 24, "rollout_id": rounds, "time_unix": t + 1}) + return out + + +def pool_records() -> list[dict]: + return [{"t": T0 + 200, "event": "nc_elastic_node_lost", "ticket": "tk1", "shard": 1, "detail": "unreachable"}, + {"t": T0 + 210, "event": "nc_elastic_ordered", "ticket": "tk2", "shard": 1, "account": "a"}, + {"t": T0 + 260, "event": "nc_elastic_member_ready", "ticket": "tk2", "shard": 1}] + + +def fleet_records() -> list[dict]: + return [{"event": "island_ready", "island_id": i, "island": f"run-l{i}-modal", "cloud": "modal", "gpu": "H100", + "gpus": 1, "time_unix": T0} for i in (0, 1)] + [ + {"event": "cost_tick", "time_unix": T0 + 300, "islands": []}] + + +def run_reducer(extra: list[dict] = ()) -> Reducer: + r = Reducer(run="async-t") + recs = trainer_records(0) + trainer_records(1) + sampler_records("nc-a1") + sampler_records("nc-a2") \ + + pool_records() + fleet_records() + list(extra) + [ + {"event": "rl_resource_sample", "island_id": 0, "time_unix": T0 + 150, "available": True, + "gpus": [{"mem_used_mb": 40000, "mem_total_mb": 81920, "util_pct": 90}]}] + for rec in sorted(recs, key=lambda x: x.get("time_unix", x.get("t", 0))): + r.feed(rec) + return r + + +@pytest.fixture(scope="module") +def view(): + return export_view(run_reducer(), generated_at=T0 + 1000) + + +def _card(view, iid): + return next(c for c in view["overview"]["islands"] if c["id"] == iid) + + +def test_islands_and_roles(view): + ids = {c["id"]: c["role"] for c in view["overview"]["islands"]} + assert ids == {"0": "trainer", "1": "trainer", "nc-a1": "sampler", "nc-a2": "sampler"} + + +@pytest.mark.parametrize("key", ["reward", "kl", "kl_tr", "tis_clip", "grad_norm", "resp_len", "tok_s"]) +def test_per_round_metric_series_non_empty(view, key): + assert len(view["overview"]["series"]["0"][key]) == ROUNDS + + +@pytest.mark.parametrize("key", ["reward", "reward_std", "p10", "p50", "p90", "kl_tr", "tis_clip", + "logprob_diff", "grad_norm", "resp_mean", "resp_p95", "trunc", "tok_s"]) +def test_round_records_fields_non_empty(view, key): + rounds = view["page"]["islands"]["0"]["rounds"] + assert [r[key] for r in rounds if r[key] is not None] and len(rounds) == ROUNDS + + +def test_phase_spans_derived_from_driver_phases(view): + rounds = view["page"]["islands"]["0"]["rounds"] + assert all(set(r["dur"]) >= {"R", "T", "S"} for r in rounds) + assert rounds[0]["dur"]["R"] == 20.0 and rounds[0]["dur"]["T"] == 20.0 + assert all("P" in r["phases"] for r in rounds[:-1]) + + +def test_sampler_table(view): + s = view["page"]["async"]["samplers"]["nc-a1"] + assert s["n_rounds"] == ROUNDS + 2 and s["policy_version"] == (ROUNDS + 1) // 2 + assert s["lag_hist"] == {"0": 4, "1": 4} and s["lag_max"] == 1 + assert s["zero_variance_seen"] == ROUNDS + 2 and s["zero_variance_dropped"] == ROUNDS + 2 + assert s["truncated"] == ROUNDS + 2 and s["samples"] == 16 * (ROUNDS + 2) + assert s["reward_mean_last"] == 0.4 and s["generate_p50_s"] == 30.0 + assert s["resource"]["available"] is False + assert (s["retried_trials"], s["lost_trials"]) == (2 * (ROUNDS + 2), ROUNDS + 2) + + +def test_trainer_async_table(view): + t = view["page"]["async"]["trainers"]["0"] + assert t["stale_groups"] == 1 and t["gate_dropped"] == 0 and t["trained_groups"] == 8 * ROUNDS + assert (t["incomplete_groups"], t["carried_over_groups"], t["padded_samples"]) == (ROUNDS, 1, 2) + assert t["age_hist"] == {"0": 4 * ROUNDS, "1": 3 * ROUNDS, "2": ROUNDS} + assert all(r.get("ages") for r in t["rounds"]) + + +def test_backlog_series(view): + b = view["page"]["async"]["backlog"] + assert b["has_reads"] and b["written"] == 2 * 4 * (ROUNDS + 2) + assert b["read"] == 9 * ROUNDS + assert b["points"][-1][3] == b["written"] - b["read"] + assert set(b["reported"]) == {"nc-a1", "nc-a2"} + + +def test_release_timeline(view): + rel = view["page"]["async"]["releases"] + assert [r["policy_version"] for r in rel][:3] == [1, 2, 3] + first = rel[0] + assert first["trainer"] == "0" and first["upload_s"] == 2.0 + assert set(first["loads"]) == {"nc-a1", "nc-a2"} + assert all(v["latency_s"] is not None and v["latency_s"] > 0 for v in first["loads"].values()) + + +def test_nc_pool_events_in_stream_and_timeline(view): + pool = view["page"]["async"]["pool"] + assert [p["event"] for p in pool] == ["nc_elastic_node_lost", "nc_elastic_ordered", "nc_elastic_member_ready"] + assert pool[0]["ts"] == T0 + 200 and pool[0]["shard"] == 1 + types = [e["type"] for e in view["events"]["events"]] + assert "nc_elastic_node_lost" in types + assert view["overview"]["counts"]["pool"] == 3 + + +def test_gpu_memory(view): + assert view["page"]["islands"]["0"]["node_series"]["nvml"][0][1] == pytest.approx(39.06, abs=0.01) + assert view["page"]["islands"]["0"]["gpu_mem_note"] is None + assert "pynvml" in view["page"]["islands"]["nc-a1"]["gpu_mem_note"] + assert "--rl-resource-sample-interval" in view["page"]["islands"]["1"]["gpu_mem_note"] + + +def test_cost_and_syncerless_page(view): + assert view["overview"]["cost"]["islands"] + assert view["overview"]["global_status"]["level"] in ("ok", "warn") + + +def test_finished_run_statuses(view): + assert _card(view, "0")["status"] == "done" + # the samplers run until their trainers end: finished, not waiting / stale + assert _card(view, "nc-a1")["status"] == "done" + assert not [a for a in view["overview"]["alerts"] if a["rule"] in ("heartbeat", "startup")] + + +def test_exported_page_has_async_section_and_nojs_banner(view): + html = render_page(view) + assert 'id="async"' in html and 'id="nojs"' in html and "