From 277a69e4dfc463c93047e894bf0b1c546e74dc6f Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:53:36 -0600 Subject: [PATCH 1/2] Let an operator cancel an installation or settle its unfinished switch `sudo sparkring install --cancel` stops the installation that is running instead of leaving its candidate's containers and an abandoned switch record. The installation reads the request before each model-switch step, stops the candidate through that deployment's own stop, starts the model the switch replaced, and records the switch as `cancelled` with the time and the operator; the active pointer stays on the model that served. A restore that fails records `needs-attention`, which another installation already refuses to replace without approval. The commit itself is not a cancel boundary. With no installation running, the command settles a switch left unfinished by SIGTERM, Ctrl-C or a Spark restart, holding that slot's admission while it stops the candidate and starts the replaced model again. `sparkring status` and the group-readable operator view report a switch in progress or one that stopped. A `cancelled` switch reserves no Spark and keeps no deployment as a rollback target, and each request keeps the operator and time that asked. Validation: pytest runtime/host (2082 passed, 86 skipped), scripts/test_sparkring_cli.py, ruff E,F,W, layout and link checks. Not run: a cancel against live Sparks; the deployment adapters are exercised with recorded adapters rather than dockerd and real deployments. --- docs/operations/commands.md | 45 ++- docs/operations/install-reference.md | 4 + runtime/host/cancel.py | 265 ++++++++++++++++++ runtime/host/checkpoints.py | 5 +- runtime/host/controller.py | 10 + runtime/host/install_workflow.py | 45 ++- runtime/host/operator_view.py | 15 + runtime/host/placement.py | 6 + runtime/host/rollout.py | 219 +++++++++++++-- runtime/host/slices.py | 3 +- runtime/host/test_cancel.py | 399 +++++++++++++++++++++++++++ 11 files changed, 985 insertions(+), 31 deletions(-) create mode 100644 runtime/host/cancel.py create mode 100644 runtime/host/test_cancel.py diff --git a/docs/operations/commands.md b/docs/operations/commands.md index 3b955c72..5e1756c8 100644 --- a/docs/operations/commands.md +++ b/docs/operations/commands.md @@ -54,7 +54,8 @@ sudo sparkring up --execute # start it again With `--json`, `install`, `hairpin`, `checkpoints` and `storage` print one JSON document. They exit with 0 on success or plan, 3 when input is needed and 2 on -failure. +failure, and also on a run that `install --cancel` stopped (state +`cancelled`). ### Without sudo @@ -93,6 +94,7 @@ serves, it releases what older deployments hold on the Sparks | `--profile PROFILE` | Exact profile from `sparkring models`; asked in a terminal when omitted | | `--on ARC` | Run the profile on these consecutive Sparks: `0,1`, `0-3`, `4-7`, or `6-1` across the cable to Node A ([models on part of the fabric](install-reference.md#models-on-part-of-the-fabric)). Default: every Spark, or for a profile of fewer Sparks the one group that divides the fabric from Node A and serves no model. The `prepared` transport runs two-Spark models only on a four-Spark ring's halves, `0,1` and `2,3` | | `--plan` | Print and save the setup, checkpoint and model plan; change nothing. A later `--yes` run stays within the saved plan. Before the first setup, use `sudo sparkring setup --plan` | +| `--cancel` | [Stop the running installation](#cancel-an-installation) at its next step boundary, or settle a model switch an interrupted installation left; takes no other option | | `--yes` | Approve setup, the checkpoint plan, ConnectX restarts on an idle ring and the model switch. Unknown SSH host keys still need confirmation, and stopping another program's GPU containers still asks unless you add `--stop-workloads` | | `--json` | One JSON result on stdout; progress on stderr | | `--override-hold` | Proceed although another operator [holds the cluster](#hold); the override is recorded | @@ -126,6 +128,43 @@ In a terminal, without `--yes` and without the API options, `install` asks which address the model's API listens on and which port it uses; Enter keeps both ([API endpoint](install-reference.md#api-endpoint)). +### Cancel an installation + +`sudo sparkring install --cancel` stops the installation that is running now; +killing the process leaves the candidate's containers and a switch record +behind instead. It records the request in +`/var/lib/sparkring/controller/install-cancel.json` and waits up to two +minutes. The installation reads the request before each step of its model +switch, stops there, stops the candidate deployment through that deployment's +own stop, starts the model the switch replaced again and records the switch as +`cancelled` with the time and the operator. The active pointer never leaves +the model that served. The interrupted run prints the reason and exits 2 with +state `cancelled`; `--cancel` itself exits 0 when it asks, stops or settles a +switch, and 2 when another command holds the cluster or a switch it tried to +settle still needs attention. + +```bash +sudo sparkring install --cancel +``` + +- A step that cannot be abandoned, such as a checkpoint download, is the + boundary: the installation stops when that step ends. The request outlives + the command, and `sparkring status` reports the switch until it settles. +- With no installation running, the same command settles a model switch an + earlier installation left unfinished, for example after `SIGTERM`, `Ctrl-C` + or a Spark restart. It stops that switch's candidate on every Spark and + starts the model the switch replaced, so the candidate's own `sparkring down + PROFILE --instance i` is no longer needed for this. +- `--cancel` takes no other option and no lock: it ends the operation whose + admission already holds them. An operator's [hold](#hold) does not refuse + it, because a hold reserves the cluster for work rather than for keeping an + unwanted installation. While another command changes the cluster (`up`, + `down` or `restart`), it names that command and changes nothing: repeat it + when the operation finishes. +- A switch in progress, or one that stopped before it finished, is reported by + [status](#status) as `Model switch: ... | next: ...`, and by the + group-readable operator view a member prints without sudo. + ## install.sh [`install.sh`](../../install.sh) builds the SparkRing package from a full Git @@ -467,7 +506,9 @@ attention, the saved model (the active deployment, or the one named) and automatic recovery. While an operator [holds the cluster](#hold), the first line names the holder, the reason and the times, such as `Held by alice since 2026-11-02 09:00 UTC until 2026-11-02 18:00 UTC: acceptance run | release: sudo -sparkring hold --release`. On a fabric that serves +sparkring hold --release`. A model switch that is in progress or stopped before +it finished adds its own line, `Model switch: ... | next: ...`, naming the step +that [settles](#cancel-an-installation) or resumes it. On a fabric that serves [several models](install-reference.md#models-on-part-of-the-fabric) it prints each group's model under its Sparks, such as `Sparks 0-3:` and `Sparks 4-7:`, with a `Group:` line naming its shape, positions and API Spark: diff --git a/docs/operations/install-reference.md b/docs/operations/install-reference.md index f8af46de..4922aa01 100644 --- a/docs/operations/install-reference.md +++ b/docs/operations/install-reference.md @@ -248,6 +248,10 @@ are host settings that SparkRing neither applies nor has measured resumes the switch. Another profile or checkpoint replaces it instead: after the GPU check, the installer stops the unfinished model through its own deployment and switches from the last model that served. + `sudo sparkring install --cancel` settles it without installing anything: it + stops the unfinished candidate on every Spark, starts the model the switch + replaced again and records the switch as `cancelled` + ([cancel an installation](commands.md#cancel-an-installation)). - **Selected model's last start did not complete**, for example when a first installation failed its readiness check or a restart of the installed model failed: the same command first stops that model on every Spark, then diff --git a/runtime/host/cancel.py b/runtime/host/cancel.py new file mode 100644 index 00000000..9dabaef4 --- /dev/null +++ b/runtime/host/cancel.py @@ -0,0 +1,265 @@ +"""Cancelling an installation: one request record on Node A and the steps an installation honours it at. + +``sudo sparkring install --cancel`` names the installations running at that +moment in ``install-cancel.json``, one entry per operation with the time and +the operator. A running installation reads its own entry before each step of +its model switch (``runtime.host.rollout``): it stops there, stops the +candidate deployment through its own adapter, brings back the model it +replaced and records the switch as ``cancelled``. With no installation +running, the same command settles the model switch an earlier installation +left unfinished, under the admission of that switch's Sparks. + +The request stands outside admission: it takes no installation lock and needs +no hold override, because the operation it ends already holds them. It names +operations by the identifier their lock records, so a request that outlives +its installation cancels nothing later. +""" +from __future__ import annotations + +import time +from pathlib import Path + +from runtime.common import installer, process_lock +from runtime.host import hold, node + +SCHEMA = "sparkring-install-cancel/v1" +RESULT_SCHEMA = "sparkring-install-cancel-result/v1" +NAME = "install-cancel.json" +# Serializes a request's read-modify-write with another request or clear; it is +# never held while an installation runs. +LOCK = "install-cancel.lock" +WAIT_SECONDS = 120 +POLL_SECONDS = 2 +# The operation string an admission records for an installation is ``install [PROFILE]``. +INSTALL = "install" + +# What the boundary names to an operator: the switch step the installation never started. +BOUNDARIES = {"preparing": "preparing the deployment", + "stopping-previous": "stopping the model it replaces", + "starting": "starting the selected model", + "verifying": "checking the selected model", + "committing": "committing the switch"} + + +class Cancelled(RuntimeError): + """An installation stopped at a step boundary because ``install --cancel`` asked it to.""" + + def __init__(self, record, boundary): + self.record, self.boundary = record, boundary + super().__init__("Installation cancelled at your request while " + + BOUNDARIES.get(boundary, boundary) + ".") + + +def path(state_root): + return Path(state_root) / NAME + + +def read(state_root): + """The standing request, or None: no record, or one this package does not read. + + A damaged request cancels nothing: a record that cannot be read must not + stop an installation, because the installation is what reports its own + outcome. + """ + try: + value = installer.read(path(state_root)) + except (OSError, ValueError): + return None + return value if isinstance(value, dict) and value.get("schema") == SCHEMA else None + + +def entries(state_root): + """``{operation id: {"at", "by"}}`` of the standing request.""" + record = read(state_root) + operations = (record or {}).get("operations") + return dict(operations) if isinstance(operations, dict) else {} + + +def request(state_root, operations, *, now=None, who=None): + """Record that the installations ``operations`` must stop at their next step boundary.""" + ids = sorted({str(item) for item in operations if item}) + if not ids: + raise ValueError("No installation is running to cancel") + moment = time.time() if now is None else now + operator = who or hold.operator() + with process_lock.hold(Path(state_root) / LOCK, wait=True): + record = read(state_root) or {"schema": SCHEMA, "operations": {}} + record["operations"] = {**(record.get("operations") or {}), + **{item: {"at": moment, "by": operator} for item in ids}} + node.save(Path(state_root), NAME, record, mode=0o600) + return record + + +def pending(state_root, operation): + """The entry naming ``operation``, or None.""" + return entries(state_root).get(operation) + + +def check(state_root, operation, boundary): + """Raise ``Cancelled`` when ``install --cancel`` named this installation.""" + entry = pending(state_root, operation) + if entry is not None: + raise Cancelled(entry, boundary) + + +def clear(state_root, operation=None): + """Forget ``operation``'s entry, and the record when no other entry stands.""" + with process_lock.hold(Path(state_root) / LOCK, wait=True): + record = read(state_root) + if record is None: + return None + operations = dict(record.get("operations") or {}) + if operation is None: + operations.clear() + else: + operations.pop(operation, None) + if operations: + node.save(Path(state_root), NAME, {**record, "operations": operations}, mode=0o600) + return {**record, "operations": operations} + path(state_root).unlink(missing_ok=True) + return {**record, "operations": operations} + + +def installations(rows): + """The rows of ``slices.active`` that are installations this command can stop.""" + return [row for row in rows if str(row.get("operation") or "").split(" ")[0] == INSTALL] + + +def describe(rows): + """``install PROFILE on Sparks 0-1``: what the request names, as the operator sees it.""" + return " and ".join( + f"{row.get('operation') or INSTALL} on " + + ("every Spark" if row.get("positions") is None else "Sparks " + ",".join(map(str, row["positions"]))) + for row in rows) + + +def cancelled_since(state_root, moment): + """``[(slot, record)]`` of switches recorded cancelled at or after ``moment``.""" + from runtime.host import placement as placements + found = [] + for slot in placements.slots(state_root): + try: + record = placements.journal(state_root, slot) + except (OSError, ValueError): + continue + if not record or record.get("state") != placements.CANCELLED: + continue + if ((record.get("cancelled") or {}).get("at") or 0) >= moment: + found.append((slot, record)) + return found + + +def run(state_root, *, now=None, who=None, say=print, sleep=time.sleep, active=None, wait=WAIT_SECONDS, + clock=time.monotonic): + """``sparkring install --cancel``: ask the running installation to stop, or settle an unfinished switch. + + Returns the result document. ``operations`` names what the request held, + ``switches`` the model switches this command settled, ``busy`` the slots + another operation kept, and ``message`` the one line an operator reads. + """ + from runtime.host import slices + state_root = Path(state_root) + active = slices.active if active is None else active + document = {"schema": RESULT_SCHEMA, "operations": [], "switches": [], "busy": []} + running = installations(active(state_root)) + if running: + record = request(state_root, [row.get("id") for row in running], now=now, who=who) + document["operations"] = sorted(record["operations"]) + moment = min(entry["at"] for entry in record["operations"].values()) + say(f"Asked {describe(running)} to stop at its next step boundary.") + deadline = clock() + wait + while clock() < deadline: + left = [row for row in installations(active(state_root)) if row.get("id") in document["operations"]] + if not left: + break + sleep(POLL_SECONDS) + left = [row for row in installations(active(state_root)) if row.get("id") in document["operations"]] + if left: + # The installation is inside a step it cannot abandon, such as a + # checkpoint download; the request outlives this command and the + # installation honours it at the next boundary. + document["state"] = "requested" + document["message"] = (f"The request stands; {describe(left)} stops at its next step boundary. " + "Its result names the switch it leaves.") + say(document["message"]) + return document + # The installation clears its own entry as it stops; one that died + # first leaves it here, and it names an operation that no longer runs. + for operation in document["operations"]: + clear(state_root, operation) + stopped = cancelled_since(state_root, moment) + document["state"] = "cancelled" + document["message"] = ("The installation stopped at the request and recorded the switch cancelled." + if stopped else + "The installation finished before it reached the request; its own result describes " + "what it did.") + say(document["message"]) + else: + # up, down and restart take the same locks as an installation and are + # not one: a request cannot stop them, and settling a switch under a + # running one would fight it. + others = [row for row in active(state_root) + if str(row.get("operation") or "") in ("up", "down", "restart")] + if others: + document["state"] = "busy" + document["message"] = (f"{describe(others)} is running; install --cancel stops installations. Repeat it " + "when that operation finishes.") + say(document["message"]) + return document + say("No installation is running.") + settled, busy = settle(state_root, now=now, who=who, say=say) + document["switches"], document["busy"] = settled, busy + if busy: + document["state"] = "busy" + document["message"] = ("Another operation holds " + + ", ".join(item["placement"] or "the whole ring" for item in busy) + + "; its model switch is left for it. Repeat the command when it finishes.") + elif settled and document.get("state") != "cancelled": + document["state"] = "cancelled" + document["message"] = ("Cancelled the unfinished model switch to " + + ", ".join(Path(item["candidate"] or "").name for item in settled) + ".") + elif not settled and document.get("state") != "cancelled": + document["state"] = "idle" + document["message"] = "No installation is running and no model switch is unfinished." + say(document["message"]) + return document + + +def settle(state_root, *, apply=None, verify=None, now=None, who=None, say=print): + """Settle every unfinished model switch under its own admission; returns ``(settled, busy)``. + + A switch's Sparks may belong to another operation that started after the + request, so each one is settled while holding that slot's admission. A slot + another operation keeps is reported and left alone, and a switch another + cancel settled between the scan and the admission is skipped. + """ + from runtime.host import placement as placements, retained_source, rollout, slices + state_root = Path(state_root) + cache = state_root / "retained-sources" + + def adapter(path, action): + return retained_source.apply(Path(path), action, cache=cache) + + apply = adapter if apply is None else apply + verify = (lambda path: apply(path, "verify")) if verify is None else verify + settled, busy = [], [] + for slot, record in rollout.unfinished(state_root): + try: + guard = slices.hold(state_root, slot, "install --cancel") + guard.__enter__() + except ValueError as error: + say(f"{placements.text(slot)}: {error}") + busy.append({"placement": list(slot) if slot else None, "reason": str(error)}) + continue + try: + current = placements.journal(state_root, slot) + if not current or current.get("state") not in placements.PENDING: + continue + candidate = Path(current.get("candidate") or "") + say(f"Stopping the unfinished model switch to {candidate.name} on {placements.text(slot)}.") + result = rollout.settle(state_root, slot, apply=apply, verify=verify, now=now, who=who, say=say) + settled.append({"placement": list(slot) if slot else None, "candidate": result.get("candidate"), + "state": result.get("state"), "recovered": bool(result.get("recovered"))}) + finally: + guard.__exit__(None, None, None) + return settled, busy diff --git a/runtime/host/checkpoints.py b/runtime/host/checkpoints.py index 2b397741..4dfec026 100644 --- a/runtime/host/checkpoints.py +++ b/runtime/host/checkpoints.py @@ -72,6 +72,7 @@ from runtime.host import slices from runtime.host import checkpoint_place +from runtime.host import placement as placements from runtime.host.install_errors import NeedsInput ROOT = Path(__file__).resolve().parents[2] @@ -86,8 +87,8 @@ STATE_SUFFIX = ".sparkring" STATE_FILES = frozenset({"owner.json", "journal.json", "lock"}) _TEMPORARY = re.compile(r"(?:owner|journal)\.json\.[0-9]+\.[0-9]+\.writing") -# Model-switch states in which a candidate deployment may hold containers or still be started. -SETTLED = frozenset({"complete", "failed", "preparation-failed", "failed-recovered"}) +# Model-switch states that leave nothing of the switch running (placement.SETTLED). +SETTLED = placements.SETTLED JSON_LIMIT = 16 << 20 GIB = 1024 ** 3 diff --git a/runtime/host/controller.py b/runtime/host/controller.py index c9dd06d6..d53d8d70 100644 --- a/runtime/host/controller.py +++ b/runtime/host/controller.py @@ -768,6 +768,13 @@ def _lifecycle(argv, *, output, locks): result = node.snapshot() if args.refresh else node.status() # An operator's hold (runtime.host.hold) comes first: other operators' changes refuse while it stands. result["hold"] = hold.current(STATE) + from runtime.host import rollout + # A model switch that stopped before it finished, and the step that settles it. + switches = rollout.unfinished(STATE) + if switches: + result["switches"] = [{"placement": list(slot) if slot else None, + "candidate": record.get("candidate"), "previous": record.get("previous"), + "state": record.get("state")} for slot, record in switches] plan_id = size = layout = None if (STATE / "cluster.json").exists(): cluster = installer.read(STATE / "cluster.json") @@ -820,6 +827,9 @@ def _lifecycle(argv, *, output, locks): if result["hold"]: clear = "sudo sparkring hold --release" + (" --override-hold" if result["hold"].get("invalid") else "") print(colour.render(hold.text(result["hold"]) + " | release: " + colour.action(clear), sys.stdout)) + for slot, record in switches: + print(colour.render(rollout.switch_line(slot, record, size) + " | next: " + + colour.action(rollout.SWITCH_NEXT), sys.stdout)) for view in views: model = view.get("model") if model and model["state"] != "serving": diff --git a/runtime/host/install_workflow.py b/runtime/host/install_workflow.py index 5af3125a..9ddd6e30 100644 --- a/runtime/host/install_workflow.py +++ b/runtime/host/install_workflow.py @@ -75,6 +75,7 @@ from runtime.common import transport as transports from runtime.host import ( api_endpoint, + cancel, checkpoint_plan, checkpoint_search, colour, @@ -1724,9 +1725,16 @@ def prepare(path): # A candidate whose own start did not complete, for example a first # installation that failed its readiness check, stops before it is # prepared again. - result = rollout.execute(directory, previous, state_root=state_root, prepare=prepare, apply=apply, - verify=lambda path: apply(path, "verify"), supersede=True, serving=serving, - unfinished=installer.unfinished, placement=placement, displaced=displaced) + operation_id = operation.get("id") + try: + result = rollout.execute(directory, previous, state_root=state_root, prepare=prepare, apply=apply, + verify=lambda path: apply(path, "verify"), supersede=True, serving=serving, + unfinished=installer.unfinished, placement=placement, displaced=displaced, + cancelled=lambda boundary: cancel.check(state_root, operation_id, boundary)) + finally: + # Every step a cancel names is behind this run either way, so its + # request must not stand for a later installation. + cancel.clear(state_root, operation_id) # Records the deployment's choice (on unless --no-auto-recover, also # after sparkring recover off), resets failures and restarts, records # the generation's boots and enables the timer. @@ -1799,6 +1807,10 @@ def main(argv=None): parser.add_argument("--plan", action="store_true", help="print and save the setup, checkpoint and model plan without changing any Spark; a later " "--yes stays within a saved plan") + parser.add_argument("--cancel", action="store_true", + help="stop the running installation at its next step boundary and bring back the model it " + "replaces, or settle a model switch an interrupted installation left behind; takes no " + "other option") parser.add_argument("--yes", action="store_true", help="approve the displayed setup, checkpoint plan, the listed ConnectX driver restarts on an " "idle ring, and model replacement; SSH trust is still required") @@ -1851,6 +1863,22 @@ def main(argv=None): sizes, _ = size_parser.parse_known_args(options) args.serving_text = {name: value for name, value in vars(sizes).items() if value is not None} colour.set_mode(args.color) + if args.cancel: + # Ending an installation changes nothing an operator holds: the request + # takes no installation lock and no hold override, and the command + # carries no other option. + used = sorted(name for name, value in vars(args).items() + if name not in ("cancel", "json", "color") and value not in (None, False, {}, [])) + if used: + parser.error("--cancel takes no other option; it stops the running installation") + try: + document = cancel.run(controller.STATE, say=(lambda line: None) if args.json else print) + except (OSError, ValueError, RuntimeError) as error: + document = {"schema": cancel.RESULT_SCHEMA, "state": "failed", "message": str(error), + "operations": [], "switches": [], "busy": []} + if args.json: + print(json.dumps(document, indent=2), file=sys.stdout) + return 0 if document["state"] in ("idle", "cancelled", "requested") else 2 if args.profile: # Every later step, and every record, uses the profile's ID (models.ALIASES). args.profile = models.canonical(args.profile) @@ -1890,6 +1918,15 @@ def journals(): result = execute(args) except NeedsInput as error: result, code = {"schema": "sparkring-install-result/v1", **error.document()}, 3 + except cancel.Cancelled as stop: + # The run stopped where the operator asked it to, so its record is + # the outcome: the model the switch replaced serves again. + result, code = {"schema": "sparkring-install-result/v1", "state": "cancelled", + "message": str(stop)}, 2 + for journal in journals(): + if journal.exists() and journal.read_bytes() != before.get(journal): + result["transaction"] = installer.read(journal) + break except (ValueError, RuntimeError, OSError, KeyError, TypeError, subprocess.SubprocessError) as error: result, code = {"schema": "sparkring-install-result/v1", "state": "failed", "message": str(error)}, 2 if getattr(error, "activity", None): @@ -1914,6 +1951,8 @@ def journals(): print("Model ready: " + result["api_url"]) for line in summary_lines(result): print(line) + elif result["state"] == "cancelled": + print(result["message"]) elif result["state"] == "needs_input": print(result["message"]) # A checkpoint message already names every file, path and size its diff --git a/runtime/host/operator_view.py b/runtime/host/operator_view.py index d54089a5..26ecaeac 100644 --- a/runtime/host/operator_view.py +++ b/runtime/host/operator_view.py @@ -99,6 +99,15 @@ def view(state_root: Path) -> dict: except (OSError, ValueError, KeyError, TypeError): continue document["deployments"] = deployments + try: + from runtime.host import rollout + # A switch that stopped before it finished, as a group member's status reads it. + document["switches"] = [{"placement": list(slot) if slot else None, + "candidate": Path(record.get("candidate") or "").name, + "previous": Path(record["previous"]).name if record.get("previous") else None, + "state": record.get("state")} for slot, record in rollout.unfinished(state_root)] + except (OSError, ValueError, KeyError, TypeError): + document["switches"] = [] try: from runtime.host import hold standing = hold.current(state_root) @@ -198,4 +207,10 @@ def lines(document: dict, *, now=time.time) -> list[str]: + (f", {recovery_record['failures']} failed attempt(s)" if recovery_record.get("failures") else "")) if not document.get("deployments"): result.append("No model deployment is recorded.") + for switch in document.get("switches") or []: + where = ("Sparks " + ",".join(map(str, switch["placement"])) + if switch.get("placement") else "every Spark") + name = f"{switch['previous']} to {switch['candidate']}" if switch.get("previous") else switch.get("candidate") + result.append(f"Model switch: {name} on {where}, recorded {switch['state']}" + " | next: an operator's sudo sparkring install --cancel") return result diff --git a/runtime/host/placement.py b/runtime/host/placement.py index 78547cb7..7e34817f 100644 --- a/runtime/host/placement.py +++ b/runtime/host/placement.py @@ -544,6 +544,12 @@ def displaced(state_root, placement): PENDING = ("stopping-previous", "starting", "verifying", "recovering-previous", "needs-attention") +# The state a switch records when an operator cancelled it: its candidate is +# stopped and the model it replaced serves, so nothing it left is unfinished. +CANCELLED = "cancelled" +# States whose switch left nothing of itself running: no candidate to stop and +# no model to bring back, so a candidate is not a rollback target. +SETTLED = frozenset({"complete", "failed", "preparation-failed", "failed-recovered", CANCELLED}) def unfinished_switches(state_root, placement): diff --git a/runtime/host/rollout.py b/runtime/host/rollout.py index 85d035d6..ad64b3fd 100644 --- a/runtime/host/rollout.py +++ b/runtime/host/rollout.py @@ -1,8 +1,10 @@ """Prepare a replacement before downtime, then recover the retained deployment on failure.""" +import time from pathlib import Path from runtime.common import installer -from runtime.host import node, placement as placements +from runtime.host import cancel, hold, node +from runtime.host import placement as placements # What the candidate's stop before preparation tells the operator, by the unfinished operation it clears. UNFINISHED = {"up": "This model's last start did not complete; it stops on every Spark first.", @@ -10,7 +12,7 @@ def execute(directory, previous, *, state_root, prepare, apply, verify, supersede=False, serving=None, - unfinished=None, placement=None, displaced=()): + unfinished=None, placement=None, displaced=(), cancelled=None): """All mutations pass through deployment adapters; the active pointer commits last. ``placement`` selects the slot (``runtime.host.placement``) whose switch @@ -33,6 +35,12 @@ def execute(directory, previous, *, state_root, prepare, apply, verify, supersed checks ownership labels, stops only that deployment's containers and refuses to repeat a stop action whose outcome is uncertain. The transaction records the operation as ``unfinished``. + + ``cancelled(boundary)`` is asked before each step the switch records, and + raises ``cancel.Cancelled`` to stop it there. A switch that has not yet + stopped anything records ``cancelled``; one that has brings back every + deployment it stopped first, and one whose restore fails records + ``needs-attention`` with the restore's error. """ directory = Path(directory).resolve() previous = Path(previous).resolve() if previous else None @@ -47,7 +55,7 @@ def execute(directory, previous, *, state_root, prepare, apply, verify, supersed journal = slot / "transaction.json" if journal.exists(): retained = installer.read(journal) - pending = retained["state"] in ("stopping-previous", "starting", "verifying", "recovering-previous", "needs-attention") + pending = retained["state"] in placements.PENDING # The caller holds every affected Spark's lock, so a journal left mid-switch belongs # to an install that stopped, for example because a Spark restarted. # Installing the same candidate resumes it; another candidate may @@ -74,16 +82,49 @@ def execute(directory, previous, *, state_root, prepare, apply, verify, supersed previous = Path(record["previous"]) if record["previous"] else None displaced = [Path(path) for path in record.get("displaced") or []] resume = True - def save(state): + def save(state, *, honour=True): + if honour and cancelled is not None: + cancelled(state) record["state"] = state node.save(slot, "transaction.json", record, mode=0o600) + + def note_cancel(stop): + record["cancelled"] = {"at": time.time(), "by": (stop.record or {}).get("by"), + "boundary": stop.boundary} + + def bring_back(paths): + """Stop the candidate and bring back every deployment this switch stopped.""" + targets = [] + for path in paths: + resolved = Path(path).resolve() + if resolved not in targets: + targets.append(resolved) + apply(directory, "down") + # A stop of the previous cluster may have left only some ranks stopped. + # Reconcile each retained operation before asking its own source + # version to bring it back. The candidate is already stopped above. + for other in targets: + if other != directory: + apply(other, "down") + for other in targets: + apply(other, "up") + observed = [verify(other) for other in targets] + return observed[0] if len(observed) == 1 else observed + # No other deployment is stopped if prerequisites, transfer, space or asset # verification fail. The existing active pointer remains authoritative. # Only a candidate whose own start or stop did not complete stops before # preparation, also when it is the active deployment. - stopped = False + stopped = attempted = stopped_active = False if not resume: - save("preparing") + try: + save("preparing") + except cancel.Cancelled as stop: + # Nothing has changed yet: the previous model serves and the + # candidate is not prepared, so the record is the whole outcome. + note_cancel(stop) + save("cancelled", honour=False) + raise try: left = unfinished(directory) if unfinished is not None else None if left in UNFINISHED: @@ -94,7 +135,7 @@ def save(state): prepare(directory) except Exception as error: record["error"] = str(error) - save("preparation-failed") + save("preparation-failed", honour=False) raise switched = False restore = ([previous] if previous is not None and previous != directory else []) + displaced @@ -107,49 +148,181 @@ def save(state): print("The installed model does not serve on every Spark; it stops on every Spark and starts again.") save("stopping-previous") apply(directory, "down") + stopped_active = True for other in displaced: save("stopping-previous") switched = True apply(other, "down") save("starting") + attempted = True apply(directory, "up") save("verifying") observed = verify(directory) record["verification"] = observed + # The commit itself cannot be cancelled, so the request is read once + # more while the switch is still undoable. + if cancelled is not None: + cancelled("committing") node.save(slot, "active.json", {"path": str(directory)}, mode=0o600) placements.clear_conflicting(state_root, placement) record["complete"] = True - save("complete") + # The commit is not a step a cancel can undo: the active pointer and + # the conflicting slots' records changed together. + save("complete", honour=False) return record + except cancel.Cancelled as stop: + # The operator asked the installation to stop: record where, and leave + # the slot as it was, with the model this switch replaced serving. + # What to bring back belongs to the journal, not to this call: a + # resumed switch may already have stopped it, and a restart of the + # active deployment starts it again under ``attempted``. + note_cancel(stop) + back = restore or ([previous] if previous is not None + and (stopped_active or attempted or resume) else []) + try: + if resume or switched or stopped_active or attempted: + record["recovery_verification"] = bring_back(back) + if previous is not None and previous != directory: + node.save(slot, "active.json", {"path": str(previous)}, mode=0o600) + if back: + record["recovered"] = True + save("cancelled", honour=False) + except Exception as recovery: + record["recovered"] = False + record["cancellation_error"] = str(recovery) + save("needs-attention", honour=False) + raise RuntimeError("Installation cancelled at your request, but restoring " + + (back[0].name if back else "the previous model") + " failed: " + + str(recovery)) from recovery + raise except Exception as error: record["error"] = str(error) if switched: - save("recovering-previous") + save("recovering-previous", honour=False) try: - apply(directory, "down") - # A failure while stopping the previous cluster may have left - # only some ranks stopped. Reconcile each retained operation - # before asking its own source version to bring it back. - for other in restore: - apply(other, "down") - for other in restore: - apply(other, "up") - record["recovery_verification"] = [verify(other) for other in restore] - if len(restore) == 1: - record["recovery_verification"] = record["recovery_verification"][0] + record["recovery_verification"] = bring_back(restore) if previous is not None and previous != directory: node.save(slot, "active.json", {"path": str(previous)}, mode=0o600) record["recovered"] = True - save("failed-recovered") + save("failed-recovered", honour=False) except Exception as recovery: record["recovered"] = False record["recovery_error"] = str(recovery) - save("needs-attention") + save("needs-attention", honour=False) else: - save("failed") + save("failed", honour=False) raise RuntimeError("Installation did not complete; " + record["state"] + ": " + str(error)) from error def active(state_root, placement=None): """The deployment the slot of ``placement`` records active; the whole cluster's for None.""" return placements.recorded(state_root, placement) + + +def unfinished(state_root): + """``[(slot, record)]`` of every slot whose model switch stopped before it finished. + + A record that cannot be read is left out: ``sparkring status`` reports what + it can read rather than failing, and admission reads the record itself. + """ + found = [] + for slot in placements.slots(state_root): + try: + record = placements.journal(state_root, slot) + except (OSError, ValueError): + continue + if record and record.get("state") in placements.PENDING: + found.append((slot, record)) + return found + + +def switch_line(slot, record, size=None): + """The first half of one ``sparkring status`` line for a model switch in progress or stopped.""" + candidate = Path(record.get("candidate") or "").name + previous = Path(record["previous"]).name if record.get("previous") else None + switch = f"{previous} to {candidate}" if previous else candidate + return (f"Model switch: {switch} on {placements.text(slot, size)}, " + f"recorded {record.get('state')}") + + +# What settles a switch that stopped before it finished, appended to switch_line's line. +SWITCH_NEXT = "sudo sparkring install --cancel stops it and records it cancelled, or install that model again to resume it" + + +def settle(state_root, placement, *, apply, verify=None, now=None, who=None, say=print): + """Stop an unfinished switch's candidate, bring back what it stopped, and record it cancelled. + + The installation that started the switch is gone: another run replaced it, + a Spark restarted, or the operator stopped it. Its own record is then the + only description of what it stopped, so the restore follows ``previous`` + and ``displaced`` and asks each deployment's own adapter. A switch that + committed before it stopped recorded the candidate as active; the restore + writes the deployment that serves again in its place. + """ + slot = placements.slot_directory(state_root, placement) + journal = slot / "transaction.json" + if not journal.exists(): + raise ValueError(f"No model switch is recorded under {slot}") + record = installer.read(journal) + if record.get("state") not in placements.PENDING: + raise ValueError(f"The model switch under {slot} is {record.get('state')}, not unfinished") + directory = Path(record["candidate"]) + previous = Path(record["previous"]) if record.get("previous") else None + # The switch stops the previous deployment and the displaced ones only + # after it records the step that does so; a switch that stopped while + # preparing stopped nothing, and a stopped model needs no restart. + stopped = record["state"] != "preparing" + back = ([previous] if previous is not None and previous != directory else []) if stopped else [] + back = back + ([Path(path) for path in record.get("displaced") or []] if stopped else []) + if not back and stopped and previous is not None and previous == directory: + back = [previous] + verify = (lambda path: apply(path, "verify")) if verify is None else verify + errors = [] + + def attempt(action, path): + try: + apply(path, action) + except Exception as error: # noqa: BLE001 - recorded and reported, one step must not hide the rest + errors.append(f"{path.name} {action}: {error}") + + attempt("down", directory) + for path in back: + attempt("down", path) + for path in back: + attempt("up", path) + verification = [] + for path in back: + try: + verification.append(verify(path)) + except Exception as error: # noqa: BLE001 - recorded as this switch's verification + errors.append(f"{path.name} verify: {error}") + record["cancelled"] = {"at": time.time() if now is None else now, "by": who or hold.operator(), + "boundary": record["state"]} + if not errors: + try: + committed = placements.recorded(state_root, placement) == directory + if committed: + # The switch committed and stopped before it recorded completion: + # the slot names the candidate, which is now stopped. + if previous is not None: + placements.record(state_root, placement, previous) + else: + placements.forget(state_root, placement, directory) + except (OSError, ValueError): + # An unreadable active record is reported by status; it does not + # change what this switch did to the models. + pass + if back: + record["recovered"] = not errors + record["recovery_verification"] = verification[0] if len(verification) == 1 else verification + if errors: + record["cancelled"]["error"] = "; ".join(errors) + record["state"] = "needs-attention" + else: + record["state"] = "cancelled" + node.save(slot, "transaction.json", record, mode=0o600) + say("Its candidate stopped; " + ("the model it replaced serves again." if record.get("recovered") + else "no earlier model was stopped." if not back + else "the model it replaced was not confirmed serving: " + + record["cancelled"]["error"])) + return record diff --git a/runtime/host/slices.py b/runtime/host/slices.py index 14da0fa1..34e20496 100644 --- a/runtime/host/slices.py +++ b/runtime/host/slices.py @@ -90,7 +90,8 @@ def affected(root, positions): continue journal = placement.journal(root, slot) directory = placement.recorded(root, slot) - if (placement.exists(directory) and not placement.stopped(directory)) or (journal and not journal.get("complete")): + if (placement.exists(directory) and not placement.stopped(directory)) or ( + journal and not journal.get("complete") and journal.get("state") != placement.CANCELLED): if slot is None: return None result.update(slot) diff --git a/runtime/host/test_cancel.py b/runtime/host/test_cancel.py new file mode 100644 index 00000000..b562f952 --- /dev/null +++ b/runtime/host/test_cancel.py @@ -0,0 +1,399 @@ +"""Cancelling an installation: the request record, the switch it stops, and the switch it settles.""" +import json +import os + +import pytest + +from runtime.common import installer +from runtime.host import cancel, node, rollout + +NOW = 1_800_000_000.0 + + +class Cluster: + """Adapters over two deployments, as ``test_rollout`` uses them.""" + + def __init__(self, previous, candidate): + self.previous, self.candidate = previous, candidate + self.running = previous + self.events = [] + + def prepare(self, directory): + self.events.append("prepare") + + def apply(self, directory, action): + self.events.append((directory.name, action)) + if action == "down": + if self.running == directory: + self.running = None + else: + self.running = directory + + def verify(self, directory): + self.events.append((directory.name, "verify")) + assert self.running == directory + return {"api_healthy": True} + + +def raise_at(boundary, who="alice"): + """A ``cancelled`` hook that asks to stop at one boundary of the switch.""" + def cancelled(state): + if state == boundary: + raise cancel.Cancelled({"by": who, "at": NOW}, state) + return cancelled + + +def interrupted(tmp_path, state): + """A switch from ``old`` to ``stuck`` whose installation stopped in ``state``, as after a SIGTERM.""" + before, stuck = tmp_path / "old", tmp_path / "stuck" + node.save(tmp_path, "active.json", {"path": str(before)}) + node.save(tmp_path, "transaction.json", {"schema": "sparkring-install-transaction/v1", "candidate": str(stuck), + "previous": str(before), "state": state, "complete": False}) + return before, stuck + + +def test_a_request_keeps_the_operator_and_time_of_each_ask(tmp_path): + first = cancel.request(tmp_path, ["b"], now=NOW, who="alice") + assert first["operations"] == {"b": {"at": NOW, "by": "alice"}} + both = cancel.request(tmp_path, ["a"], now=NOW + 1, who="bob") + assert both["operations"] == {"a": {"at": NOW + 1, "by": "bob"}, "b": {"at": NOW, "by": "alice"}} + assert cancel.pending(tmp_path, "a") == {"at": NOW + 1, "by": "bob"} + assert cancel.pending(tmp_path, "c") is None + assert cancel.clear(tmp_path, "a")["operations"] == {"b": {"at": NOW, "by": "alice"}} + assert cancel.clear(tmp_path, "b")["operations"] == {} + assert cancel.read(tmp_path) is None + with pytest.raises(ValueError, match="No installation is running"): + cancel.request(tmp_path, [], now=NOW, who="alice") + + +def test_a_damaged_request_cancels_nothing(tmp_path): + (tmp_path / cancel.NAME).write_text("{not json", encoding="utf-8") + assert cancel.read(tmp_path) is None + # A record that cannot be read must not stop an installation: the + # installation is what reports the outcome. + cancel.check(tmp_path, "a", "starting") + (tmp_path / cancel.NAME).write_text(json.dumps({"schema": "other/v1", "operations": {"a": {}}}), + encoding="utf-8") + cancel.check(tmp_path, "a", "starting") + + +def test_a_cancel_before_any_stop_leaves_the_serving_model_alone(tmp_path): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + cluster = Cluster(before, after) + with pytest.raises(cancel.Cancelled, match="preparing the deployment"): + rollout.execute(after, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify, cancelled=raise_at("preparing")) + assert cluster.events == [] and cluster.running == before + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and record["complete"] is False + assert record["cancelled"]["by"] == "alice" and record["cancelled"]["boundary"] == "preparing" + assert "recovered" not in record and record["cancelled"].get("at") + assert rollout.active(tmp_path) == before + + +@pytest.mark.parametrize("boundary", ["starting", "verifying", "committing"]) +def test_a_cancel_mid_switch_brings_back_the_model_it_replaced(tmp_path, boundary): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + cluster = Cluster(before, after) + with pytest.raises(cancel.Cancelled, match="Installation cancelled at your request"): + rollout.execute(after, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify, cancelled=raise_at(boundary)) + # The candidate never serves: whatever started stops, then the previous + # model is reconciled and started again. + assert cluster.events[-4:] == [("candidate", "down"), ("old", "down"), ("old", "up"), ("old", "verify")] + assert cluster.running == before + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and record["recovered"] is True + assert record["cancelled"]["boundary"] == boundary + assert rollout.active(tmp_path) == before + + +def test_a_cancel_as_the_switch_would_stop_the_running_model_stops_nothing(tmp_path): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + cluster = Cluster(before, after) + # The step is recorded before the stop it describes, so a cancel there + # finds both models as the switch began. + with pytest.raises(cancel.Cancelled): + rollout.execute(after, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify, cancelled=raise_at("stopping-previous")) + assert cluster.events == ["prepare"] and cluster.running == before + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and "recovered" not in record + + +def test_a_cancelled_restart_of_the_active_model_starts_it_again(tmp_path): + current = tmp_path / "current" + node.save(tmp_path, "active.json", {"path": str(current)}) + events = [] + + def apply(directory, action): + events.append((directory.name, action)) + + with pytest.raises(cancel.Cancelled): + rollout.execute(current, rollout.active(tmp_path), state_root=tmp_path, + prepare=lambda directory: events.append("prepare"), apply=apply, + verify=lambda directory: events.append((directory.name, "verify")) or {}, + serving=lambda directory: True, cancelled=raise_at("verifying")) + # The switch restarted the only model; a cancel stops it and starts it again. + assert events == ["prepare", ("current", "up"), ("current", "down"), ("current", "up"), ("current", "verify")] + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and record["recovered"] is True + + +def test_a_cancelled_resumed_switch_brings_back_the_model_the_earlier_run_stopped(tmp_path): + before, stuck = interrupted(tmp_path, "starting") + events = [] + with pytest.raises(cancel.Cancelled): + rollout.execute(stuck, before, state_root=tmp_path, prepare=lambda directory: events.append("prepare"), + apply=lambda directory, action: events.append((directory.name, action)), + verify=lambda directory: events.append((directory.name, "verify")) or {}, + cancelled=raise_at("stopping-previous")) + # A resumed switch skips preparation, and its first recorded step is the + # stop the earlier run already performed. + assert events == [("stuck", "down"), ("old", "down"), ("old", "up"), ("old", "verify")] + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and record["recovered"] is True + + +def test_a_cancelled_first_install_stops_its_candidate_and_records_it(tmp_path): + after = tmp_path / "candidate" + events = [] + with pytest.raises(cancel.Cancelled): + rollout.execute(after, None, state_root=tmp_path, prepare=lambda directory: events.append("prepare"), + apply=lambda directory, action: events.append((directory.name, action)), + verify=lambda directory: events.append((directory.name, "verify")) or {}, + cancelled=raise_at("verifying")) + assert events == ["prepare", ("candidate", "up"), ("candidate", "down")] + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and "recovered" not in record + + +def test_the_commit_is_not_a_boundary_a_cancel_stops(tmp_path): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + cluster = Cluster(before, after) + result = rollout.execute(after, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify, cancelled=raise_at("complete")) + assert result["complete"] and rollout.active(tmp_path) == after + assert cluster.running == after and "cancelled" not in result + + +def test_a_cancelled_switch_does_not_block_the_next_install(tmp_path): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + cluster = Cluster(before, after) + with pytest.raises(cancel.Cancelled): + rollout.execute(after, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify, cancelled=raise_at("starting")) + later = tmp_path / "later" + result = rollout.execute(later, before, state_root=tmp_path, prepare=cluster.prepare, apply=cluster.apply, + verify=cluster.verify) + assert result["complete"] and rollout.active(tmp_path) == later + + +def test_a_failed_restore_while_cancelling_keeps_the_slot_blocked(tmp_path): + before, after = tmp_path / "old", tmp_path / "candidate" + node.save(tmp_path, "active.json", {"path": str(before)}) + events = [] + + def apply(directory, action): + events.append((directory.name, action)) + if action == "up" and directory == before: + raise RuntimeError("Node 1 did not answer") + + with pytest.raises(RuntimeError, match="cancelled at your request, but restoring old failed"): + rollout.execute(after, before, state_root=tmp_path, prepare=lambda directory: None, apply=apply, + verify=lambda directory: {}, cancelled=raise_at("starting")) + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "needs-attention" and record["recovered"] is False + assert "Node 1 did not answer" in record["cancellation_error"] + + +def test_settling_an_unfinished_switch_stops_its_candidate_and_restarts_the_previous(tmp_path): + before, stuck = interrupted(tmp_path, "starting") + assert str(before).endswith("old") + events = [] + settled, busy = cancel.settle(tmp_path, apply=lambda directory, action: events.append((directory.name, action)), + verify=lambda directory: events.append((directory.name, "verify")) or {"ok": True}) + assert events == [("stuck", "down"), ("old", "down"), ("old", "up"), ("old", "verify")] + assert busy == [] + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "cancelled" and record["recovered"] is True + assert record["cancelled"]["boundary"] == "starting" + assert settled == [{"placement": None, "candidate": str(stuck), "state": "cancelled", "recovered": True}] + # Nothing is left for a second run to settle. + assert cancel.settle(tmp_path, apply=lambda directory, action: None) == ([], []) + + +def test_settling_a_committed_switch_rewrites_the_active_record(tmp_path): + before, stuck = tmp_path / "old", tmp_path / "stuck" + # The switch wrote the candidate active and stopped before it recorded completion. + node.save(tmp_path, "active.json", {"path": str(stuck)}) + node.save(tmp_path, "transaction.json", {"schema": "sparkring-install-transaction/v1", "candidate": str(stuck), + "previous": str(before), "state": "verifying", "complete": False}) + cancel.settle(tmp_path, apply=lambda directory, action: None, verify=lambda directory: {}, + say=lambda line: None) + assert rollout.active(tmp_path) == before + + +def test_settling_reports_a_restore_it_could_not_complete(tmp_path): + interrupted(tmp_path, "verifying") + + def apply(directory, action): + if action == "up": + raise RuntimeError("Node 1 did not answer") + + settled, busy = cancel.settle(tmp_path, apply=apply, verify=lambda directory: {}, say=lambda line: None) + record = installer.read(tmp_path / "transaction.json") + assert record["state"] == "needs-attention" + assert settled[0]["state"] == "needs-attention" and settled[0]["recovered"] is False + assert busy == [] and len(rollout.unfinished(tmp_path)) == 1 + + +def test_settling_leaves_a_switch_another_operation_holds(tmp_path): + from runtime.host import slices + interrupted(tmp_path, "starting") + with slices.hold(tmp_path, None, "install other"): + settled, busy = cancel.settle(tmp_path, apply=lambda directory, action: None, + verify=lambda directory: {}, say=lambda line: None) + assert settled == [] and busy and "Another operation is active" in busy[0]["reason"] + # The switch still belongs to the operation that holds it. + assert installer.read(tmp_path / "transaction.json")["state"] == "starting" + + +def test_settling_skips_a_switch_that_became_settled_first(tmp_path, monkeypatch): + interrupted(tmp_path, "cancelled") + stale = (None, {"candidate": str(tmp_path / "stuck"), "previous": str(tmp_path / "old"), "state": "starting"}) + monkeypatch.setattr(rollout, "unfinished", lambda root: [stale]) + applied = [] + settled, busy = cancel.settle(tmp_path, apply=lambda directory, action: applied.append(action), + verify=lambda directory: {}, say=lambda line: None) + assert (settled, busy, applied) == ([], [], []) + + +def test_status_names_the_switch_and_the_command_that_settles_it(tmp_path): + _, stuck = interrupted(tmp_path, "starting") + found = rollout.unfinished(tmp_path) + assert [(slot, record["candidate"]) for slot, record in found] == [(None, str(stuck))] + assert rollout.switch_line(found[0][0], found[0][1], 2) == ( + "Model switch: old to stuck on both Sparks, recorded starting") + assert "install --cancel" in rollout.SWITCH_NEXT and "install that model again" in rollout.SWITCH_NEXT + # A settled switch is not reported again. + cancel.settle(tmp_path, apply=lambda directory, action: None, verify=lambda directory: {}, + say=lambda line: None) + assert rollout.unfinished(tmp_path) == [] + + +def test_the_operator_view_names_an_unfinished_switch_too(tmp_path): + from runtime.host import operator_view + interrupted(tmp_path, "starting") + document = operator_view.view(tmp_path) + assert document["switches"] == [{"placement": None, "candidate": "stuck", "previous": "old", + "state": "starting"}] + text = "\n".join(operator_view.lines(document)) + assert "Model switch: old to stuck on every Spark, recorded starting" in text + assert "sudo sparkring install --cancel" in text + + +def test_a_record_that_cannot_be_read_is_not_reported(tmp_path): + from runtime.host import operator_view + interrupted(tmp_path, "starting") + (tmp_path / "transaction.json").write_text("{not json", encoding="utf-8") + assert rollout.unfinished(tmp_path) == [] + assert operator_view.view(tmp_path)["switches"] == [] + + +def test_a_cancelled_switch_reserves_no_spark_and_no_rollback_target(tmp_path): + from runtime.host import checkpoints, slices + node.save(tmp_path, "transaction.json", {"schema": "sparkring-install-transaction/v1", + "candidate": str(tmp_path / "stuck"), + "previous": str(tmp_path / "old"), "state": "cancelled", + "complete": False}) + # The candidate is stopped and the replaced model serves, so a model on + # part of the ring needs no admission over the rest of it. + assert slices.affected(tmp_path, (0, 1)) == (0, 1) + assert os.path.realpath(tmp_path / "stuck") not in checkpoints.roles(tmp_path) + # A switch that stopped mid-step still reserves the ring and keeps its candidate. + node.save(tmp_path, "transaction.json", {"schema": "sparkring-install-transaction/v1", + "candidate": str(tmp_path / "stuck"), + "previous": str(tmp_path / "old"), "state": "starting", + "complete": False}) + assert slices.affected(tmp_path, (0, 1)) is None + assert checkpoints.roles(tmp_path)[os.path.realpath(tmp_path / "stuck")] == "switching" + + +def test_cancel_with_nothing_running_reports_nothing_to_do(tmp_path): + lines = [] + document = cancel.run(tmp_path, say=lines.append, sleep=lambda seconds: None, active=lambda root: []) + assert document["state"] == "idle" and document["switches"] == [] and document["busy"] == [] + assert cancel.read(tmp_path) is None + assert "No installation is running" in lines[0] + + +def test_cancel_asks_the_running_installation_to_stop(tmp_path): + rows = [{"id": "op-1", "operation": "install glm53", "positions": None}] + lines = [] + + def active(root): + return list(rows) + + def sleep(seconds): + # The installation reads the request at its next step boundary and leaves. + rows.clear() + + document = cancel.run(tmp_path, say=lines.append, sleep=sleep, active=active, now=NOW, who="alice", + clock=lambda: 0.0) + assert document["state"] == "cancelled" and document["operations"] == ["op-1"] + assert "install glm53 on every Spark" in lines[0] + assert cancel.read(tmp_path) is None + + +def test_cancel_reports_an_installation_that_finished_before_reaching_the_request(tmp_path): + rows = [{"id": "op-1", "operation": "install glm53", "positions": None}] + lines = [] + + def sleep(seconds): + rows.clear() + + document = cancel.run(tmp_path, say=lines.append, sleep=sleep, active=lambda root: list(rows), + now=NOW, who="alice", clock=lambda: 0.0) + assert document["state"] == "cancelled" + assert "finished before it reached the request" in document["message"] + + +def test_cancel_that_the_installation_cannot_reach_yet_leaves_its_request_standing(tmp_path): + rows = [{"id": "op-1", "operation": "install glm53", "positions": [0, 1]}] + lines = [] + document = cancel.run(tmp_path, say=lines.append, sleep=lambda seconds: None, active=lambda root: list(rows), + now=NOW, who="alice", clock=lambda: 0.0, wait=0) + assert document["state"] == "requested" + assert "install glm53 on Sparks 0,1" in document["message"] + assert cancel.pending(tmp_path, "op-1") is not None + + +def test_cancel_does_not_settle_a_switch_under_a_running_lifecycle_operation(tmp_path): + interrupted(tmp_path, "starting") + lines = [] + rows = [{"id": "op-2", "operation": "down", "positions": [0, 1]}] + document = cancel.run(tmp_path, say=lines.append, sleep=lambda seconds: None, active=lambda root: list(rows)) + assert document["state"] == "busy" and "down on Sparks 0,1 is running" in document["message"] + # The switch is left for the operation that owns the locks. + assert installer.read(tmp_path / "transaction.json")["state"] == "starting" + assert len(rollout.unfinished(tmp_path)) == 1 + + +def test_the_command_needs_no_cluster_and_refuses_other_options(tmp_path, monkeypatch): + from runtime.host import install_workflow + monkeypatch.setattr(install_workflow.controller, "STATE", tmp_path) + monkeypatch.setattr(install_workflow.cancel, "run", + lambda root, **kwargs: {"schema": cancel.RESULT_SCHEMA, "state": "idle", + "message": "No installation is running and no model switch is " + "unfinished.", "operations": [], "switches": [], + "busy": []}) + assert install_workflow.main(["--cancel"]) == 0 + for refused in (["--profile", "glm53-flash"], ["--yes"], ["--events", str(tmp_path / "events.jsonl")]): + with pytest.raises(SystemExit): + install_workflow.main(["--cancel", *refused]) From 483771a8cb54f93a9a57a3ddb83c081901e6233a Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:45:38 -0600 Subject: [PATCH 2/2] Report a switch that still needs attention as failed, not cancelled `install --cancel` treats every settled switch as cancelled, so a restore that failed leaves the switch recorded `needs-attention` while the command exits 0, contrary to its documented exit contract. A settled switch now carries the restore error, and the command exits 2 with state `failed` and a message naming the switch and the error. Validation: pytest runtime/host/test_cancel.py runtime/host/test_rollout.py runtime/host/test_install_workflow.py (162 passed); ruff check --select E,F,W --ignore E501 runtime scripts; scripts/check_repository_layout.py; scripts/check_markdown_links.py . --- docs/operations/commands.md | 14 ++++++------ runtime/host/cancel.py | 24 +++++++++++++++++--- runtime/host/test_cancel.py | 44 ++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/docs/operations/commands.md b/docs/operations/commands.md index 5e1756c8..631d9683 100644 --- a/docs/operations/commands.md +++ b/docs/operations/commands.md @@ -94,7 +94,7 @@ serves, it releases what older deployments hold on the Sparks | `--profile PROFILE` | Exact profile from `sparkring models`; asked in a terminal when omitted | | `--on ARC` | Run the profile on these consecutive Sparks: `0,1`, `0-3`, `4-7`, or `6-1` across the cable to Node A ([models on part of the fabric](install-reference.md#models-on-part-of-the-fabric)). Default: every Spark, or for a profile of fewer Sparks the one group that divides the fabric from Node A and serves no model. The `prepared` transport runs two-Spark models only on a four-Spark ring's halves, `0,1` and `2,3` | | `--plan` | Print and save the setup, checkpoint and model plan; change nothing. A later `--yes` run stays within the saved plan. Before the first setup, use `sudo sparkring setup --plan` | -| `--cancel` | [Stop the running installation](#cancel-an-installation) at its next step boundary, or settle a model switch an interrupted installation left; takes no other option | +| `--cancel` | [Stop the running installation](#cancel-an-installation) at its next step boundary, or settle a model switch an interrupted installation left; takes no other installation option | | `--yes` | Approve setup, the checkpoint plan, ConnectX restarts on an idle ring and the model switch. Unknown SSH host keys still need confirmation, and stopping another program's GPU containers still asks unless you add `--stop-workloads` | | `--json` | One JSON result on stdout; progress on stderr | | `--override-hold` | Proceed although another operator [holds the cluster](#hold); the override is recorded | @@ -155,12 +155,12 @@ sudo sparkring install --cancel or a Spark restart. It stops that switch's candidate on every Spark and starts the model the switch replaced, so the candidate's own `sparkring down PROFILE --instance i` is no longer needed for this. -- `--cancel` takes no other option and no lock: it ends the operation whose - admission already holds them. An operator's [hold](#hold) does not refuse - it, because a hold reserves the cluster for work rather than for keeping an - unwanted installation. While another command changes the cluster (`up`, - `down` or `restart`), it names that command and changes nothing: repeat it - when the operation finishes. +- `--cancel` takes no other installation option and no lock: it ends the + operation whose admission already holds them. An operator's [hold](#hold) + does not refuse it, because a hold reserves the cluster for work rather than + for keeping an unwanted installation. While another command changes the + cluster (`up`, `down` or `restart`), it names that command and changes + nothing: repeat it when the operation finishes. - A switch in progress, or one that stopped before it finished, is reported by [status](#status) as `Model switch: ... | next: ...`, and by the group-readable operator view a member prints without sudo. diff --git a/runtime/host/cancel.py b/runtime/host/cancel.py index 9dabaef4..39f93c87 100644 --- a/runtime/host/cancel.py +++ b/runtime/host/cancel.py @@ -156,8 +156,11 @@ def run(state_root, *, now=None, who=None, say=print, sleep=time.sleep, active=N Returns the result document. ``operations`` names what the request held, ``switches`` the model switches this command settled, ``busy`` the slots another operation kept, and ``message`` the one line an operator reads. + ``state`` is ``requested`` while a running installation finishes a step it + cannot abandon, ``failed`` when a switch this command settled still needs + attention, and otherwise ``cancelled`` or ``idle``. """ - from runtime.host import slices + from runtime.host import placement as placements, slices state_root = Path(state_root) active = slices.active if active is None else active document = {"schema": RESULT_SCHEMA, "operations": [], "switches": [], "busy": []} @@ -209,7 +212,17 @@ def run(state_root, *, now=None, who=None, say=print, sleep=time.sleep, active=N say("No installation is running.") settled, busy = settle(state_root, now=now, who=who, say=say) document["switches"], document["busy"] = settled, busy - if busy: + attention = [item for item in settled if item["state"] != placements.CANCELLED] + if attention: + # A switch that stopped with a model it could not bring back is not + # settled: its slot stays reserved and status keeps reporting it until + # an operator or a later installation acts on it. + first = next((item for item in attention if item.get("error")), attention[0]) + document["state"] = "failed" + document["message"] = (f"The model switch to {Path(first.get('candidate') or '').name} still needs attention" + + (f": {first['error']}" if first.get("error") else "") + + ". sparkring status names it.") + elif busy: document["state"] = "busy" document["message"] = ("Another operation holds " + ", ".join(item["placement"] or "the whole ring" for item in busy) @@ -232,6 +245,10 @@ def settle(state_root, *, apply=None, verify=None, now=None, who=None, say=print request, so each one is settled while holding that slot's admission. A slot another operation keeps is reported and left alone, and a switch another cancel settled between the scan and the admission is skipped. + + Each settled entry names its placement, candidate, state and ``recovered``, + and ``error`` the restore failure that left the switch needing attention + when its state is not ``placements.CANCELLED``. """ from runtime.host import placement as placements, retained_source, rollout, slices state_root = Path(state_root) @@ -259,7 +276,8 @@ def adapter(path, action): say(f"Stopping the unfinished model switch to {candidate.name} on {placements.text(slot)}.") result = rollout.settle(state_root, slot, apply=apply, verify=verify, now=now, who=who, say=say) settled.append({"placement": list(slot) if slot else None, "candidate": result.get("candidate"), - "state": result.get("state"), "recovered": bool(result.get("recovered"))}) + "state": result.get("state"), "recovered": bool(result.get("recovered")), + "error": (result.get("cancelled") or {}).get("error")}) finally: guard.__exit__(None, None, None) return settled, busy diff --git a/runtime/host/test_cancel.py b/runtime/host/test_cancel.py index b562f952..28f7a2cf 100644 --- a/runtime/host/test_cancel.py +++ b/runtime/host/test_cancel.py @@ -223,7 +223,8 @@ def test_settling_an_unfinished_switch_stops_its_candidate_and_restarts_the_prev record = installer.read(tmp_path / "transaction.json") assert record["state"] == "cancelled" and record["recovered"] is True assert record["cancelled"]["boundary"] == "starting" - assert settled == [{"placement": None, "candidate": str(stuck), "state": "cancelled", "recovered": True}] + assert settled == [{"placement": None, "candidate": str(stuck), "state": "cancelled", "recovered": True, + "error": None}] # Nothing is left for a second run to settle. assert cancel.settle(tmp_path, apply=lambda directory, action: None) == ([], []) @@ -385,6 +386,29 @@ def test_cancel_does_not_settle_a_switch_under_a_running_lifecycle_operation(tmp assert len(rollout.unfinished(tmp_path)) == 1 +def test_cancel_reports_a_settle_that_still_needs_attention(tmp_path, monkeypatch): + from runtime.host import retained_source + interrupted(tmp_path, "verifying") + + def apply(directory, action, cache=None): + if action == "up": + raise RuntimeError("Node 1 did not answer") + + monkeypatch.setattr(retained_source, "apply", apply) + lines = [] + document = cancel.run(tmp_path, say=lines.append, sleep=lambda seconds: None, active=lambda root: []) + assert document["state"] == "failed" + assert document["switches"] == [{"placement": None, "candidate": str(tmp_path / "stuck"), + "state": "needs-attention", "recovered": False, + "error": "old up: Node 1 did not answer"}] + assert document["message"] == ("The model switch to stuck still needs attention: old up: Node 1 did not answer. " + "sparkring status names it.") + assert lines[-1] == document["message"] + # The switch still needs attention, so the command does not claim it was cancelled. + assert installer.read(tmp_path / "transaction.json")["state"] == "needs-attention" + assert len(rollout.unfinished(tmp_path)) == 1 + + def test_the_command_needs_no_cluster_and_refuses_other_options(tmp_path, monkeypatch): from runtime.host import install_workflow monkeypatch.setattr(install_workflow.controller, "STATE", tmp_path) @@ -397,3 +421,21 @@ def test_the_command_needs_no_cluster_and_refuses_other_options(tmp_path, monkey for refused in (["--profile", "glm53-flash"], ["--yes"], ["--events", str(tmp_path / "events.jsonl")]): with pytest.raises(SystemExit): install_workflow.main(["--cancel", *refused]) + + +def test_the_command_exits_2_when_a_settle_still_needs_attention(tmp_path, monkeypatch, capsys): + from runtime.host import install_workflow, retained_source + interrupted(tmp_path, "verifying") + + def apply(directory, action, cache=None): + if action == "up": + raise RuntimeError("Node 1 did not answer") + + monkeypatch.setattr(retained_source, "apply", apply) + monkeypatch.setattr(install_workflow.controller, "STATE", tmp_path) + assert install_workflow.main(["--cancel"]) == 2 + assert "still needs attention" in capsys.readouterr().out + assert install_workflow.main(["--cancel", "--json"]) == 2 + document = json.loads(capsys.readouterr().out) + assert document["schema"] == cancel.RESULT_SCHEMA and document["state"] == "failed" + assert document["switches"][0]["state"] == "needs-attention"