diff --git a/docs/development/dashboard-counters.md b/docs/development/dashboard-counters.md index e7c4752a..7c3f50a3 100644 --- a/docs/development/dashboard-counters.md +++ b/docs/development/dashboard-counters.md @@ -50,11 +50,30 @@ SparkRing's `/opt/sparkring/toolchain/toolchain.py serve` wrapper and `python3 -m vllm.entrypoints.cli.main serve` supply the same vLLM launch facts. Literal defaults are used only for recognized server launches without a configuration-file argument. Shell variables are not expanded. Unknown -GPU containers remain visible with engine `unknown`; absent launch facts +GPU containers remain visible with engine `unknown` in the observation row; +the page omits a launch fact it could not read rather than naming it, and a +GPU compute job shows its container and image alone. Absent launch facts cannot establish a cross-Spark group. Bridged Docker port translation and launches discoverable only through environment/configuration files are not inferred. +A bare `flare run` process is recognized from its GPU PID and bounded argv. +The parser accepts `flare` and `flare-` binaries, rejects `flare agent` +and `flare orbit`, and publishes only rank, rendezvous, HTTP bind/port and +the served alias. Flare's documented defaults apply: `--host 127.0.0.1`, +`--port 8000`, rank zero when `--rank` is absent and no HTTP API on peers. +Package paths, publisher keys, trust stores and API-key paths are not +published. Ranks sharing a rendezvous form one group; a process without one +is a one-Spark group. A missing rank zero is named **head Spark not +reporting** rather than counted as a complete smaller group, including when +only one peer rank reports. `flare run` has no world-size option; the +prepared package learns it at rendezvous, so the expected count is the +highest observed rank plus one and ranks above that highest one are not +detected as missing. A non-IP `--host` publishes no API facts rather than +being rewritten as loopback. The same parser runs for a container's +entrypoint, so a containerized Flare server is found when its launch is in +`Path`/`Args`. + The recovery timer writes an allowlisted projection to `/var/lib/sparkring/dashboard/nodes.json` (`sparkring-dashboard-nodes/v1`). It reads all setup-recorded SSH targets through its existing client and @@ -90,21 +109,37 @@ Unmanaged models use the same polling, bounded history, rate calculation, graph controls and integer token-rate formatting as managed models. SGLang's generation/prompt counters, first-token histogram sum/count, running/waiting gauges and token-usage gauge map to the shared numeric -contract. Other absent counters remain unknown. Metric names follow the +contract. Flare's generated/prefill token counters, active/pending gauges, +KV page tokens, draft counters, prefix-cache counters and preemptions map to +the same contract; uptime supplies the up-time line, but Flare publishes no +start time, so a restart does not begin a new history run. Other absent +counters remain unknown. Metric names follow the [SGLang metrics reference](https://docs.sglang.io/docs/references/production_metrics) and its [collector](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/observability/metrics_collector.py). -Launch defaults follow the [vLLM serve reference](https://docs.vllm.ai/en/stable/cli/serve/). +Launch defaults follow the [vLLM serve reference](https://docs.vllm.ai/en/stable/cli/serve/) +and Flare's `flare run --help`. + +The HTTP client sends `Accept: text/plain` only for `/metrics`; model-list +and runtime-status reads request JSON. The Flare parser also handles JSON +metric objects and Helios snapshots through the same series-to-short-name +mapping. Snapshot KV page counts are multiplied by `page_tokens`; missing +or nonnumeric fields remain unknown. A response with no usable Flare +series publishes `metrics_problem` and displays **Metrics unavailable** +instead of silently leaving the card's figures blank. The allowed hosts, +exact read paths, response bounds and redirect refusal are unchanged. The agent unit preserves its shared `/run/sparkring` directory across stop/start, including package upgrades. Operator quiet files and holder records retain their contents and expiry; reboot still clears runtime state. -Offline coverage is in `test_dashboard_unmanaged.py`, `test_dashboard_ring_observation.py` and -`test_dashboard_charts.py`: both engines, one through eleven Sparks, -non-contiguous membership, mixed parallel dimensions, incomplete groups, -stale/stopped records, ownership, bounds, secret exclusion, restricted reads, -quiet mode and in-place browser updates. These tests do not qualify live -Docker, SSH latency, GPU serving or recovery on Spark hardware. +Offline coverage is in `test_dashboard_unmanaged.py`, `test_flare_dashboard.py`, +`test_dashboard_ring_observation.py` and `test_dashboard_charts.py`: both +engines, Flare's Orbit/manual/default launch shapes, one through eleven +Sparks, non-contiguous membership, mixed parallel dimensions, incomplete +groups, missing heads, stale/stopped records, ownership, bounds, secret +exclusion, restricted reads, quiet mode and in-place browser updates. These +tests do not qualify live Docker, SSH latency, GPU serving or recovery on +Spark hardware. The initial history response contains raw `counters` and optional `plugin_counters` envelopes for replay at their `observed_at` time. diff --git a/docs/operations/dashboard.md b/docs/operations/dashboard.md index 477b393f..de7726df 100644 --- a/docs/operations/dashboard.md +++ b/docs/operations/dashboard.md @@ -209,17 +209,37 @@ frozen view of up to 6 hours. Precision, counter sources and cost are in ### Models installed outside SparkRing -The page also shows vLLM and SGLang containers that the host agents find -running. Their cards say **Not installed by SparkRing: no automatic recovery, -no install record** and show the Sparks, engine, image, served name and -parallel sizes. Containers that share a head Spark and rendezvous port form -one group; a group with missing ranks shows **incomplete: N of M Sparks -reporting**. Only rank 0's reported API port is read, with no port scan; -SGLang needs `--enable-metrics` for rates. **Thinking** reads **unknown** -unless the launch or runtime status gives explicit chat defaults. The +The page also shows vLLM and SGLang containers, and bare `flare run` +processes, that the host agents find running. Their cards say **Not +installed by SparkRing: no automatic recovery, no install record** and show +the Sparks, engine, image or process, served name and parallel sizes. A +fact the page could not read is left out rather than named, and a GPU +compute job shows its container and image alone. +Containers and Flare ranks that share a head Spark and rendezvous port form +one group; a Flare process without a rendezvous is a one-Spark group. A +group with missing ranks shows **incomplete: N of M Sparks reporting**, and +a Flare group whose rank zero is absent says **head Spark not reporting**. +Flare's argv carries no world size, so M is inferred from the highest +observed rank: ranks above the highest one seen are not detected as +missing. Only rank 0's reported API port is read, with no port scan; SGLang needs +`--enable-metrics` for rates. **Thinking** reads **unknown** unless the +launch or runtime status gives explicit chat defaults. The [observation contract](../development/dashboard-counters.md#unmanaged-model-observations) states the bounds and freshness rules. +A Flare rank is recognized from the GPU process's bounded argv, whichever +way it was started: Orbit, a manual `flare run`, a systemd unit or a shell +wrapper. Rank, `--rendezvous`, `--host`, `--port` and `--served-model-name` +are published; package paths, trust stores, API-key paths and the rest of +argv stay private. The card reads the head's `/metrics` and `/v1/models` and +uses Flare's own counters for tokens, requests, KV use, draft acceptance and +prefix hits. Two limits follow from Flare's defaults: a launch bound to +`127.0.0.1` (Flare's default) has no live counters from another Spark, and +a launch configured only through environment variables or an opaque +container entrypoint cannot be identified. The host-agent cache updates +about once a minute, so a starting or stopped process may take that long to +appear or leave the page. + ### Other endpoints Models served outside the installer, such as Local Inference Lab's Flare @@ -257,6 +277,16 @@ context length from `/v1/models` and its state. The state follows the ring's rules: metrics answering is **Serving**, a refused connection **No model running**, no answer **Offline**, an HTTP error **Not ready**. +The collector requests Prometheus text (`Accept: text/plain`) for `/metrics` +and JSON (`Accept: application/json`) for `/v1/models` and the permitted +SparkRing runtime-status read. Flare's counters also accept a JSON metrics +response, mapping recognized series or Helios snapshot fields to the same +figures; fields absent from JSON remain unknown. A successful response +without recognized numeric Flare metrics keeps the endpoint's **Serving** +state but shows **Metrics unavailable** and an explanation on its card. +The warning disappears after a usable metrics response. This applies to +both listed endpoints and Flare processes discovered by the host agents. + ### Terminal view The same status in a terminal on Node A, for example over SSH: diff --git a/runtime/host/dashboard.html b/runtime/host/dashboard.html index f3b7a45e..07c7541d 100644 --- a/runtime/host/dashboard.html +++ b/runtime/host/dashboard.html @@ -434,8 +434,8 @@

Events

const PALETTE = ["var(--g1)", "var(--g2)", "var(--g3)", "var(--g4)"]; const JOB_COLOR = "var(--muted)"; const gcolor = (i) => PALETTE[(i || 0) % PALETTE.length]; -const memberColor = m => m.unmanaged || m.gpu_job ? JOB_COLOR : m.external && m.kind === "flare" ? "var(--flare)" : gcolor(m.arc?.color); -const isFlare = m => !!(m && m.external && m.kind === "flare"); +const isFlare = m => !!(m && ((m.external && m.kind === "flare") || (m.unmanaged && (m.flare || m.launch?.engine === "flare")))); +const memberColor = m => isFlare(m) ? "var(--flare)" : m.unmanaged || m.gpu_job ? JOB_COLOR : gcolor(m.arc?.color); const FLAME_PATH = "M12 2c.5 4.2-5 6.6-5 11.4a5 5 0 0 0 10 0c0-2.5-1.3-4.1-2.3-5.2-.2 1.7-1.1 2.8-2 2.8-1.2 0-1.8-1.2-1.3-2.6C12.8 6.6 12.3 4.2 12 2z"; const FLAME = ``; // Folded cards, by member id; kept in this browser only. @@ -1240,14 +1240,40 @@

Events

} function unmanagedFacts(s) { - const f = s.launch || {}; + const f = s.launch || {}, flare = isFlare(s); + // A fact the page could not read is left out, never named, as the Spark details are: + // a GPU compute job shows the container's identity alone, and a Flare process lists + // only the rank, rendezvous, served name and API its launch declared. + const known = v => has(v) && v !== "" && String(v).toLowerCase() !== "unknown"; + const tell = (...parts) => parts.filter(Boolean).join(" · "); + const rendezvous = c => !known(c.master_addr) ? (known(c.master_port) ? "master port " + esc(c.master_port) : "") + : "master " + esc(c.master_addr) + (known(c.master_port) ? ":" + esc(c.master_port) : ""); + const api = c => c.headless ? "headless" : !known(c.api_port) ? "" + : known(c.api_bind) ? "API " + esc(c.api_bind) + ":" + esc(c.api_port) : "API port " + esc(c.api_port); + const rank = c => known(c.node_rank) ? "node rank " + esc(c.node_rank) : ""; + const lines = c => { + const identity = tell(`position ${c.position}: ${esc(c.name)}`, + flare ? "flare process" : known(c.image) ? esc(c.image) : "", + known(c.image_id) ? esc(c.image_id) : ""); + if (flare) { + return [identity, tell("engine flare", known(c.served_model) ? "served name " + esc(c.served_model) : "", rank(c)), + tell(rendezvous(c), api(c))]; + } + return s.gpu_job ? [identity] : [identity, + tell(known(c.engine) ? esc(c.engine) : "", known(c.served_model) ? "served name " + esc(c.served_model) : "", + rank(c), known(c.nnodes) ? `${esc(c.nnodes)} node${c.nnodes === 1 ? "" : "s"}` : ""), + tell(esc(parallelism(c)), rendezvous(c), api(c))]; + }; + const summary = s.gpu_job ? "" : flare + ? tell(esc(s.layout.label), "flare process", known(f.served_model) ? esc(f.served_model) : "") + : tell(esc(s.layout.label), known(f.engine) ? esc(f.engine) : "", + known(f.image_id) ? "image " + esc(f.image_id) : ""); return `

${esc(s.ownership_label)}

${s.group_note ? `

${esc(s.group_note)}

` : ""} - ${s.gpu_job ? "" : `

${esc(s.layout.label)} · ${esc(f.engine || "unknown")} · image ${esc(f.image_id || "unknown")}

`} -
Container launch facts and install records - ${(s.containers || []).map(c => `

position ${c.position}: ${esc(c.name)} · ${esc(c.image)} · ${esc(c.image_id || "unknown")}${s.gpu_job ? "" : `
- ${esc(c.engine || "unknown")} · served name ${esc(c.served_model || "unknown")} · node rank ${c.node_rank ?? "?"} · ${c.nnodes ?? "?"} nodes
- ${esc(parallelism(c))} · master ${esc(c.master_addr || "unknown")}:${c.master_port ?? "?"} · ${c.headless ? "headless" : "API port " + (c.api_port ?? "unknown")}`}

`).join("")} + ${s.api_local ? `

The API is bound to loopback on the head Spark, so this page cannot read live counters. Start it with --host 0.0.0.0 to show them.

` : ""} + ${summary ? `

${summary}

` : ""} +
${flare ? "Process" : "Container"} launch facts and install records + ${(s.containers || []).map(c => `

${lines(c).filter(Boolean).join("
")}

`).join("")} ${(s.stopped_records || []).map(r => `

Stopped install record: ${esc(r.id)} · ${esc(r.model)} · ${esc(positionText(r.positions))}

`).join("")}
`; } @@ -1296,15 +1322,17 @@

Events

const thinking = s.thinking ? `
Thinking: ${esc(s.thinking)}
` : ""; const modelText = s.name === s.model && !s.gpu_job ? (serving ? "" : `Last served: same model`) : `${model}`; const open = !folded.has(s.id); - return `
+ return `
${foldButton(s.id, s.name)}${esc(s.name)} ${serving ? modelText + (sizes ? `${esc(sizes)}` : "") : ""} ${tags.join("")} ${esc(s.startup ? startupLabel(s) : s.state_label)}${esc(since)} + ${s.metrics_problem ? `Metrics unavailable` : ""} ${!serving ? modelText : ""}${endpoint}${start}${recoveryChip}
${open ? ` ${startupDetails(s)} + ${s.metrics_problem ? `

${esc(s.metrics_problem)}

` : ""} ${serving ? thinking + tiles(s) + `
${one(s.live?.steps_ps)} steps/s · ${acceptancePct(s.live?.acceptance)} draft acceptance${has(s.live?.accepted_per_step) ? ` · ${acceptedStep(s.live.accepted_per_step)} accepted tok/step` : ""} · ${sampleSource(s, now)}
` + charts(s) + sparks(s) : ""}
Details
${sub}
@@ -1347,9 +1375,11 @@

Events

${serving ? `${model}` : ""} ${flare ? FLAME : ""}${esc(kind)}${where} ${esc(e.state_label || "Checking")}${e.state_since ? "for " + dur(now - e.state_since) : ""} + ${e.metrics_problem ? `Metrics unavailable` : ""} ${!serving ? `${model}` : ""}${endpoint}
${open ? `
${spark}${esc(kind)} at ${esc(e.host)}:${esc(e.port)}${has(e.max_model_len) ? ` · ${int(e.max_model_len)}-token context` : ""}
+ ${e.metrics_problem ? `

${esc(e.metrics_problem)}

` : ""} ${tiles}${line}
Details
${esc(kind)} at ${esc(e.host)}:${esc(e.port)}${e.root ? ` · ${esc(e.root)}` : ""}${e.metrics_url ? ` · metrics ${esc(e.metrics_url)}` : ""}
diff --git a/runtime/host/dashboard.py b/runtime/host/dashboard.py index 5ae41af8..d6890fcd 100644 --- a/runtime/host/dashboard.py +++ b/runtime/host/dashboard.py @@ -132,6 +132,8 @@ # The only paths the collector requests: the model list, vLLM's Prometheus metrics and the # runtime-status plugin's JSON report. Each is a GET that runs no model. READ_PATHS = frozenset({"/metrics", "/v1/models", "/v1/sparkring/status"}) +READ_ACCEPT = {"/metrics": "text/plain", "/v1/models": "application/json", + "/v1/sparkring/status": "application/json"} METRICS_VIEWED_SECONDS = 2.0 HISTORY_SECONDS = 3 * 3600 HISTORY_SAMPLES = 2161 @@ -246,7 +248,7 @@ def allow(self, hosts) -> None: def get(self, host, port, path) -> Fetched: url = request_url(host, port, path, self.allowed) request = urllib.request.Request(url, method="GET", headers={ - "User-Agent": "sparkring-dashboard", "Accept": "application/json, text/plain"}) + "User-Agent": "sparkring-dashboard", "Accept": READ_ACCEPT[path]}) if request.get_method() != "GET" or request.data is not None: raise RequestRefused("the dashboard sends GET requests without a body only") with self.lock: @@ -359,17 +361,51 @@ def parse_metrics(text: str) -> dict: "flare_free_kv_tokens": "kv_tokens_free", "flare_cached_kv_tokens": "kv_tokens_cached"} FLARE_COUNTERS = {"flare_generated_tokens_total": "generated_tokens", "flare_prefill_tokens_total": "prefill_tokens", "flare_requests_completed_total": "requests_completed", "flare_requests_failed_total": "requests_failed", + "flare_requests_cancelled_total": "requests_cancelled", + "flare_preemptions_total": "preemptions", "flare_speculative_offered_drafts_total": "drafts_offered", "flare_speculative_accepted_drafts_total": "drafts_accepted", + "flare_verification_draft_tokens_total": "draft_tokens", "flare_engram_row_cache_hits_total": "row_cache_hits", "flare_engram_row_cache_misses_total": "row_cache_misses", "flare_cache_lookups_total": "prefix_lookups", "flare_cache_hits_total": "prefix_hits", "flare_http_requests_total": "http_requests"} +FLARE_SERIES = {**FLARE_GAUGES, **FLARE_COUNTERS} +# Helios JSON snapshots use field names rather than Prometheus series names. +FLARE_JSON_KEYS = {"flare_generated_tokens_total": "generated_tokens", + "flare_prefill_tokens_total": "prefill_tokens", + "flare_active_requests": "active", "flare_pending_requests": "pending", + "flare_requests_completed_total": "completed", "flare_requests_failed_total": "failed", + "flare_speculative_accepted_drafts_total": "accepted_drafts", + "flare_verification_draft_tokens_total": "verification_drafts"} +FLARE_METRICS_PROBLEM = "No recognized numeric Flare metrics in /metrics response." def parse_flare_metrics(text: str) -> dict: - """The Flare series of FLARE_GAUGES and FLARE_COUNTERS by their short names; a labelled series is summed - over its labels. Series that are absent are absent from the result.""" + """Flare's Prometheus series or JSON snapshot mapped to the same short names. + + Labelled text series are summed over labels. Only recognized numeric JSON fields + are used; absent series stay absent, including fields that only the text API emits.""" out: dict = {} + try: + document = json.loads(text) + except (ValueError, RecursionError): + pass + else: + if not isinstance(document, dict): + return out + for name, key in FLARE_SERIES.items(): + value = document.get(name, document.get(FLARE_JSON_KEYS.get(name))) + if number(value): + out[key] = value + page_tokens = document.get("page_tokens") + if number(page_tokens): + for field, series in (("total_pages", "flare_total_kv_tokens"), + ("free_pages", "flare_free_kv_tokens"), + ("cached_pages", "flare_cached_kv_tokens")): + pages = document.get(field) + if number(pages): + out.setdefault(FLARE_SERIES[series], pages * page_tokens) + return out for line in text.splitlines(): if not line or line[0] == "#": continue @@ -381,7 +417,7 @@ def parse_flare_metrics(text: str) -> dict: name, rest = line[:brace], line[end + 1:] else: name, _, rest = line.partition(" ") - key = FLARE_GAUGES.get(name) or FLARE_COUNTERS.get(name) + key = FLARE_SERIES.get(name) if key is None: continue parts = rest.split() @@ -395,6 +431,39 @@ def parse_flare_metrics(text: str) -> dict: return out +def parse_flare_as_metrics(text: str) -> dict: + """Flare's series in the shape the model cards and history already read. + + ``parse_metrics`` serves vLLM and SGLang; this adapter keeps Flare's own + names as separate series where they are useful and maps the shared + counters. ``started`` stays None: Flare publishes uptime but no start + time, and deriving one would split history on every restart. + """ + flare = parse_flare_metrics(text) + out: dict = {"finished": {}, "cache_info": None, "model_name": None, "started": None} + out["metrics_problem"] = None if flare else FLARE_METRICS_PROBLEM + for source, key in (("generated_tokens", "gen"), ("prefill_tokens", "prompt"), + ("drafts_accepted", "accepted"), ("drafts_offered", "drafts"), + ("draft_tokens", "draft_tokens"), ("prefix_hits", "pc_hits"), + ("prefix_lookups", "pc_queries"), ("preemptions", "preemptions")): + if source in flare: + out[key] = flare[source] + for source, key in (("active_requests", "running"), ("pending_requests", "waiting"), + ("request_capacity", "request_capacity"), ("uptime_seconds", "uptime_seconds"), + ("tokens_last_minute", "tokens_last_minute"), ("kv_tokens_total", "kv_tokens_total"), + ("kv_tokens_free", "kv_tokens_free"), ("kv_tokens_cached", "kv_tokens_cached")): + if source in flare: + out[key] = flare[source] + total, free = flare.get("kv_tokens_total"), flare.get("kv_tokens_free") + if total and free is not None: + out["kv"] = (total - free) / total + for source, key in (("requests_completed", "stop"), ("requests_failed", "error"), + ("requests_cancelled", "abort")): + if source in flare: + out["finished"][key] = flare[source] + return out + + def read_external(path) -> list: """The endpoints the file at ``path`` lists, validated; [] when the file is absent. @@ -842,23 +911,27 @@ def unmanaged_groups(fabric, documents, deployments, now): positions = sorted({c["position"] for c in containers}) ranks = {c["node_rank"] for c in containers if "node_rank" in c} expected = max([c.get("nnodes", 1) for c in containers] + [max(ranks, default=0) + 1]) - heads = [c for c in containers if c.get("engine") in ("vllm", "sglang") + heads = [c for c in containers if c.get("engine") in ("vllm", "sglang", "flare") and c.get("node_rank", 0 if expected == 1 else -1) == 0 and not c["headless"]] head = heads[0] if len(heads) == 1 else None info = head or containers[0] head_position = head["position"] if head else None head_spark = next((p for p in fabric["positions"] if p["position"] == head_position), {}) bind = (head or {}).get("api_bind") + loopback = bind in ("127.0.0.1", "::1", "localhost") host = (head_spark.get("address") if not bind or bind in ("0.0.0.0", "::") else bind if bind in head_spark.get("addresses", [head_spark.get("address")]) else None) complete = len(ranks) == expected and len(positions) >= expected if expected == 1: complete = True - summary = None if complete else f"incomplete: {len(positions)} of {expected} Sparks reporting" + summary = (None if complete else + "head Spark not reporting" if info.get("engine") == "flare" and 0 not in ranks + else f"incomplete: {len(positions)} of {expected} Sparks reporting") digest = hashlib.sha256(json.dumps(key).encode()).hexdigest()[:20] result.append({"id": "unmanaged-" + digest, "unmanaged": True, "model": head.get("served_model") if head else None, "positions": positions, "head_position": head_position, "api_host": host, "api_port": head.get("api_port") if head and host else None, + "head_port": head.get("api_port") if head else None, "api_local": loopback, "stopped": False, "containers": containers, "launch": info, "expected_sparks": expected, "group_note": summary, "stopped_records": [r for r in deployments if r["stopped"] and set(r["positions"]) & set(positions)]}) @@ -983,6 +1056,8 @@ def observe_metrics(self, metrics: dict, now: float) -> None: self.output_tps = (generated - self.sample[0]) / (now - self.sample[1]) if generated is not None: self.sample = (generated, now) + else: + self.sample, self.output_tps = None, None self.metrics, self.metrics_at = metrics, now @@ -1276,11 +1351,13 @@ def poll(self, model: Model, now: float, viewed: bool) -> bool: cadence = METRICS_VIEWED_SECONDS if model.state == "serving" and model.metrics_at is not None: model.next_metrics = min(model.next_metrics, model.metrics_at + cadence) + engine = (row.get("launch") or {}).get("engine") if now >= model.next_metrics: sent = True fetched = self.client.get(host, port, "/metrics") if fetched.ok: - metrics = parse_metrics(fetched.body.decode("utf-8", "replace")) + text = fetched.body.decode("utf-8", "replace") + metrics = parse_flare_as_metrics(text) if engine == "flare" else parse_metrics(text) if not row.get("unmanaged") and row.get("model") and metrics.get("model_name") and metrics["model_name"] != row["model"]: state = "idle" with self.lock: @@ -1328,7 +1405,7 @@ def poll(self, model: Model, now: float, viewed: bool) -> bool: model.models_ok_at = now model.model = listed.get("id") or model.model model.max_model_len = listed.get("max_model_len") or model.max_model_len - if model.state == "serving" and model.runtime_due: + if model.state == "serving" and model.runtime_due and engine != "flare": sent = True model.runtime_due = False fetched = self.client.get(host, port, "/v1/sparkring/status") @@ -1422,6 +1499,7 @@ def ratio(numerator, denominator): "endpoint": f"http://{row['host']}:{row['port']}/v1", "metrics_url": f"http://{row['host']}:{row['port']}/metrics", "state": external.state, "state_label": LABELS[external.state], "state_since": external.state_since, "model": external.model, "root": external.root, "max_model_len": external.max_model_len, + "metrics_problem": FLARE_METRICS_PROBLEM if serving and row["kind"] == "flare" and not metrics else None, "metrics_at": external.metrics_at, "stats": stats} def external_ring_member(self, external: External, source: Source, now: float, color: int) -> dict: @@ -1680,7 +1758,7 @@ def member(self, model: Model, now: float, color: int) -> dict: nodes = runtime.get("nodes") or [] head_position = row.get("head_position", positions[0]) head = hostnames.get(head_position, {}) - port = row["api_port"] + port = row["api_port"] or row.get("head_port") endpoint = row.get("display_url") or f"http://{row['api_host']}:{port}/v1" if not row["api_host"] or not port: endpoint = None @@ -1704,8 +1782,10 @@ def member(self, model: Model, now: float, color: int) -> dict: capacity = dict(model.capacity) if model.capacity else None if capacity and runtime.get("settings"): capacity["max_num_seqs"] = runtime["settings"].get("max_num_seqs") - container = next((c["name"] for c in row["containers"] if c["position"] == head_position), None) launch = row.get("launch") or {} + flare = launch.get("engine") == "flare" + container = None if flare else next( + (c["name"] for c in row["containers"] if c["position"] == head_position), None) gpu_job = bool(row.get("unmanaged") and launch.get("engine") == "unknown") sparks = [] for position in positions: @@ -1713,14 +1793,16 @@ def member(self, model: Model, now: float, color: int) -> dict: up = True if serving else (False if model.state == "offline" and position == positions[0] else None) sparks.append({"name": spark.get("hostname", str(position)), "address": spark.get("address"), "lan_address": spark.get("lan_address"), "up": up, "position": position, - "container": next((c["name"] for c in row["containers"] if c["position"] == position), None)}) + "container": None if flare else next( + (c["name"] for c in row["containers"] if c["position"] == position), None)}) thought = None if gpu_job else thinking_state(row, runtime.get("settings")) return { "id": model.id, "name": ("Spark " if len(positions) == 1 else "Sparks ") + ", ".join(map(str, positions)) if row.get("unmanaged") else run_name(positions), "profile": row.get("profile"), "start_command": start_command(row), - "unmanaged": bool(row.get("unmanaged")), "gpu_job": gpu_job, "launch": launch, + "unmanaged": bool(row.get("unmanaged")), "gpu_job": gpu_job, "flare": flare, + "api_local": bool(row.get("api_local")), "launch": launch, "ownership_label": "Not installed by SparkRing: no automatic recovery, no install record" if row.get("unmanaged") else None, "group_note": row.get("group_note"), "containers": row["containers"] if row.get("unmanaged") else [], "stopped_records": row.get("stopped_records", []), @@ -1728,12 +1810,13 @@ def member(self, model: Model, now: float, color: int) -> dict: "spark_startup": spark_starting, "state": model.state, "state_label": "GPU in use" if gpu_job else LABELS.get(model.state, "Checking"), "state_since": since, "model": model.model, "model_short": short_model(model.model), "max_model_len": model.max_model_len, + "metrics_problem": metrics.get("metrics_problem") if serving else None, "endpoint": endpoint, # The runtime-status view asks the workers for a report every 5 s while it is open. "details_url": f"http://{row['api_host']}:{port}/v1/sparkring/status/view" if model.runtime else None, - "layout": {"label": parallelism(launch) + "layout": {"label": "Flare" if flare else parallelism(launch) if row.get("unmanaged") else layout_label(len(positions)) if serving else None, "count": len(positions), - "tp": launch.get("tp") if row.get("unmanaged") else runtime.get("tp"), + "tp": len(positions) if flare else launch.get("tp") if row.get("unmanaged") else runtime.get("tp"), "dcp": launch.get("dcp") if row.get("unmanaged") else (runtime.get("settings") or {}).get("decode_context_parallel_size")}, "live": {key: metrics.get(key) for key in ("running", "waiting", "kv")} if model.metrics else None, @@ -1747,7 +1830,9 @@ def member(self, model: Model, now: float, color: int) -> dict: "thinking": thinking.deployment_text(thought) if thought else None, "thinking_default": thought, "thinking_provenance": thinking.provenance_text(thought), "settings": runtime.get("settings"), "nodes": nodes, "capacity": capacity if serving else None, - "workers": runtime.get("workers"), "image": runtime.get("image"), "transport": runtime.get("transport"), + "workers": ({"received": len(positions), "expected": row.get("expected_sparks") or len(positions)} + if flare else runtime.get("workers")), + "image": runtime.get("image"), "transport": runtime.get("transport"), "sparks": sparks, "history": [], "arc": {"positions": positions, "first": positions[0], "last": positions[-1], "size": len(positions), diff --git a/runtime/host/node_containers.py b/runtime/host/node_containers.py index a3d61774..aaf5df54 100644 --- a/runtime/host/node_containers.py +++ b/runtime/host/node_containers.py @@ -1,10 +1,13 @@ -"""Bounded local model-container observations; never publish argv or environment. +"""Bounded local model observations; never publish argv or environment. Docker inspect uses its local Unix socket. GPU PIDs already collected by the -agent identify docker-exec servers whose launch differs from the entrypoint. +agent identify docker-exec servers whose launch differs from the entrypoint, +and bare ``flare run`` processes that the Orbit agent or an operator started. No command is executed and no environment is read from /proc. """ +import hashlib import http.client +import ipaddress import json from pathlib import Path import re @@ -13,6 +16,7 @@ import time MAX_CONTAINERS = 32 +MAX_FLARE_PROCESSES = 16 MAX_ARGV_BYTES = 16384 MAX_ARGS = 256 MAX_GPU_PIDS = 256 @@ -20,6 +24,7 @@ MAX_INSPECT_SECONDS = 1.0 MANAGED_NAME = re.compile(r"(?:sr-)?[A-Za-z0-9][A-Za-z0-9_.-]*-[0-9a-f]{6}-r[0-9]+") MANAGED_LABELS = ("io.sparkring.deployment", "org.sparkring.profile", "io.sparkring.stock-plan") +FLARE_BINARY = re.compile(r"flare(?:-[0-9a-f]{8,64})?") INTEGER_FLAGS = { "tp": ("--tensor-parallel-size", "--tp-size", "--tp", "-tp"), "pp": ("--pipeline-parallel-size", "--pp-size", "--pp", "-pp"), @@ -56,6 +61,98 @@ def bounded_argv(argv, depth=0): return argv +def flare_run_index(argv): + """The index of ``run`` in a ``flare run`` invocation, or None. + + The binary is ``flare`` or a content-addressed ``flare-``; ``flare + agent``, ``flare orbit ...`` and other subcommands are not serving ranks. + """ + for index, arg in enumerate(argv[:4]): + if FLARE_BINARY.fullmatch(Path(arg).name) and index + 1 < len(argv) and argv[index + 1] == "run": + return index + 1 + return None + + +def argument_values(argv, start): + """Literal ``--flag value`` and ``--flag=value`` pairs from ``start``; variables stay unknown.""" + values: dict = {} + index = start + while index < len(argv): + arg = argv[index] + key, separator, value = arg.partition("=") + if not separator: + following = argv[index + 1] if index + 1 < len(argv) else "" + # A negative memory limit is a value, not the next option. + if following and (not following.startswith("-") or re.fullmatch(r"-[0-9][0-9A-Za-z.]*", following)): + value = following + index += 1 + else: + value = "" + values[key] = value + index += 1 + return values + + +def bounded_text(value): + return isinstance(value, str) and 0 < len(value) <= 512 and not any(c in value for c in "$`\n\r") + + +def flare_launch_facts(argv): + """Public launch facts of one ``flare run`` process, or None for another command. + + Only the recognized serving options are published: rank, rendezvous, + HTTP bind and port, and the served alias. Package paths, trust stores, + publisher keys, API keys and the rest of argv stay private. + """ + run = flare_run_index(argv) + if run is None: + return None + values = argument_values(argv, run + 1) + rank = values.get("--rank", "") + if rank and not re.fullmatch(r"[0-9]{1,5}", rank): + return {"engine": "unknown"} + rank = int(rank) if rank else 0 + if rank > 65535: + return {"engine": "unknown"} + facts: dict = {"engine": "flare", "node_rank": rank, "headless": rank != 0} + rendezvous = values.get("--rendezvous") + if isinstance(rendezvous, str) and bounded_text(rendezvous): + host, separator, port = rendezvous.rpartition(":") + host = host.strip("[]") + try: + ipaddress.ip_address(host) + valid_host = True + except ValueError: + valid_host = bool(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,252}", host)) + if separator and port.isdigit() and 0 < int(port) < 65536 and valid_host: + facts.update(master_addr=host, master_port=int(port)) + if rank == 0: + # Flare's --host is an IpAddr with a 127.0.0.1 default. An unparsable + # value is not published as loopback, which would claim reachability + # the process never had. + provided = values.get("--host") + bind = (provided if provided is not None else "127.0.0.1").removeprefix("[").removesuffix("]") + port = values.get("--port") or "8000" + try: + ipaddress.ip_address(bind) + except ValueError: + bind = None + if bind is not None and re.fullmatch(r"[0-9]{1,5}", port) and 0 < int(port) < 65536: + facts.update(api_bind=bind, api_port=int(port)) + served = values.get("--served-model-name") + if isinstance(served, str) and bounded_text(served): + facts["served_model"] = served + return facts + + +def flare_row_name(facts): + """A stable row name for one bare process, independent of its PID.""" + key = [facts.get("master_addr"), facts.get("master_port"), facts.get("node_rank", 0), + facts.get("api_bind"), facts.get("api_port")] + digest = hashlib.sha256(json.dumps(key, sort_keys=True).encode()).hexdigest()[:10] + return f"flare-r{facts.get('node_rank', 0)}-{digest}" + + def thinking_kwargs(value): """Only bounded thinking arguments are public launch facts; arbitrary template data stays private.""" if isinstance(value, str): @@ -75,6 +172,9 @@ def thinking_kwargs(value): def launch_facts(argv): argv = bounded_argv(argv) + flare = flare_launch_facts(argv) + if flare is not None: + return flare engine = next(("vllm" if "vllm" in a else "sglang" for a in argv if a in ("vllm", "sglang", "sglang.launch_server", "vllm.entrypoints.openai.api_server", "vllm.entrypoints.cli.main") @@ -83,11 +183,11 @@ def launch_facts(argv): wrapper = "/opt/sparkring/toolchain/toolchain.py" if any(a == wrapper and argv[i + 1:i + 2] == ["serve"] for i, a in enumerate(argv)): engine = "vllm" - facts = {"engine": engine} + facts: dict = {"engine": engine} if engine == "unknown" or any(a in argv for a in ("bench", "preload", "--help", "--version")): return {"engine": "unknown"} # Keep only literal, recognized values. Shell variables are unknown, not facts. - values = {} + values: dict = {} for i, arg in enumerate(argv): key, sep, value = arg.partition("=") if not sep: @@ -173,24 +273,27 @@ def observations(facts, *, root="/", inspect=inspect_local): candidates.append(row) if len(candidates) == MAX_CONTAINERS: break - gpu_argv = {} - for pid in ((facts.get("gpu") or {}).get("compute_processes") or [])[:MAX_GPU_PIDS]: - if type(pid) is not int or pid <= 0: + gpu_argv, gpu_parsed, gpu_container = {}, {}, {} + for gpu_pid in ((facts.get("gpu") or {}).get("compute_processes") or [])[:MAX_GPU_PIDS]: + if type(gpu_pid) is not int or gpu_pid <= 0: continue + pid, parsed, container = gpu_pid, {"engine": "unknown"}, None try: for _ in range(8): proc = Path(root) / "proc" / str(pid) with (proc / "cgroup").open("rb") as stream: groups = stream.read(8192).decode("utf-8", "replace") - container = next((r.get("id") for r in candidates if r.get("id") and r["id"] in groups), None) - if not container: - break + found = next((r.get("id") for r in candidates if r.get("id") and r["id"] in groups), None) + if found: + container = found + # Read every hop, not only container members: a bare `flare run` + # process has no container cgroup to match. with (proc / "cmdline").open("rb") as stream: raw = stream.read(MAX_ARGV_BYTES + 1) - parsed = (launch_facts(raw.rstrip(b"\0").decode("utf-8", "replace").split("\0")) - if len(raw) <= MAX_ARGV_BYTES else {"engine": "unknown"}) - if container not in gpu_argv or parsed["engine"] != "unknown": - gpu_argv[container] = parsed + here = (launch_facts(raw.rstrip(b"\0").decode("utf-8", "replace").split("\0")) + if len(raw) <= MAX_ARGV_BYTES else {"engine": "unknown"}) + if parsed["engine"] == "unknown" or here["engine"] != "unknown": + parsed = here if parsed["engine"] != "unknown": break with (proc / "status").open("rb") as stream: @@ -200,6 +303,12 @@ def observations(facts, *, root="/", inspect=inspect_local): pid = int(parent[1]) except OSError: continue + if parsed["engine"] != "unknown": + gpu_parsed[gpu_pid] = parsed + if container: + gpu_container[gpu_pid] = container + if container not in gpu_argv or parsed["engine"] != "unknown": + gpu_argv[container] = parsed result = [] deadline = time.monotonic() + MAX_INSPECT_SECONDS for row in candidates: @@ -223,4 +332,12 @@ def observations(facts, *, root="/", inspect=inspect_local): "image_id": str(doc.get("Image") or "").removeprefix("sha256:")[:12], "managed": bool(MANAGED_NAME.fullmatch(name) or any(labels.get(k) for k in MANAGED_LABELS)), **parsed}) + bare = 0 + for pid, parsed in gpu_parsed.items(): + if bare >= MAX_FLARE_PROCESSES: + break + if pid in gpu_container or parsed.get("engine") != "flare": + continue + result.append({"name": flare_row_name(parsed), "image": "", "image_id": "", "managed": False, **parsed}) + bare += 1 return result diff --git a/runtime/host/test_dashboard.py b/runtime/host/test_dashboard.py index 8a7cdbdb..32293ce1 100644 --- a/runtime/host/test_dashboard.py +++ b/runtime/host/test_dashboard.py @@ -200,6 +200,22 @@ def test_the_client_sends_bodyless_get_requests_and_counts_them(): assert client.counters()["requests"] == {"/v1/models": 1} +def test_the_client_sends_the_expected_accept_header_for_every_permitted_path(): + accepts = {"/metrics": "text/plain", "/v1/models": "application/json", + "/v1/sparkring/status": "application/json"} + assert set(accepts) == dashboard.READ_PATHS + api = FakeAPI(Clock()) + client = dashboard.ReadOnlyClient(opener=api) + client.allow({"192.0.2.12"}) + for path, accept in accepts.items(): + assert client.get("192.0.2.12", 8015, path).ok + request = api.requests[-1] + assert request.get_header("Accept") == accept + assert request.get_header("User-agent") == "sparkring-dashboard" + assert (request.get_method(), request.data) == ("GET", None) + assert request.full_url == f"http://192.0.2.12:8015{path}" + + def test_collection_sends_only_get_requests_to_read_paths_and_starts_no_process(tmp_path, monkeypatch): """Two simulated hours: a viewer for half an hour, a model restart, a stopped model, a model that starts, an unreachable Spark and on-demand settings reads.""" diff --git a/runtime/host/test_dashboard_charts.py b/runtime/host/test_dashboard_charts.py index e562be1f..158439d5 100644 --- a/runtime/host/test_dashboard_charts.py +++ b/runtime/host/test_dashboard_charts.py @@ -145,6 +145,24 @@ def update_page(page): page.evaluate("async () => { await refresh(); await new Promise(requestAnimationFrame); }") +@pytest.mark.parametrize("external", [False, True]) +def test_flare_metrics_warning_is_visible_and_clears_in_both_card_types(smooth_page, external): + page, value, _ = smooth_page + member = value["groups"][0]["members"][0] + member.update(flare=True, unmanaged=True, external=external, kind="flare", + label="Flare endpoint", host="192.0.2.12", port=8000, + metrics_problem=dashboard.FLARE_METRICS_PROBLEM) + update_page(page) + card = page.locator(f'article.card[data-member="{member["id"]}"]') + badge = card.locator(".tag.warn", has_text="Metrics unavailable") + assert badge.is_visible() and badge.get_attribute("title") == dashboard.FLARE_METRICS_PROBLEM + note = card.get_by_role("status") + assert note.is_visible() and note.inner_text() == dashboard.FLARE_METRICS_PROBLEM + member["metrics_problem"] = None + update_page(page) + assert badge.count() == 0 and note.count() == 0 + + @pytest.mark.parametrize("width", [1400, 375]) def test_thinking_line_renders_and_updates_in_place(smooth_page, width): page, value, requests = smooth_page diff --git a/runtime/host/test_dashboard_external.py b/runtime/host/test_dashboard_external.py index 82a203e9..eb1841c5 100644 --- a/runtime/host/test_dashboard_external.py +++ b/runtime/host/test_dashboard_external.py @@ -75,6 +75,74 @@ def test_flare_metrics_are_read_by_short_name_and_summed_over_labels(): assert dashboard.parse_flare_metrics("") == {} +def test_flare_json_metrics_use_the_same_short_names(): + metrics = dashboard.parse_flare_metrics(json.dumps({ + "flare_generated_tokens_total": 12000, "flare_active_requests": 2, + "flare_total_kv_tokens": 4000, "flare_free_kv_tokens": 1000, + "flare_unknown_series": 7, + })) + assert metrics == {"generated_tokens": 12000, "active_requests": 2, + "kv_tokens_total": 4000, "kv_tokens_free": 1000} + + +def test_helios_json_snapshot_fields_are_mapped_without_inventing_absent_series(): + metrics = dashboard.parse_flare_metrics(json.dumps({ + "generated_tokens": 12000, "prefill_tokens": 48000, "active": 2, "pending": 0, + "completed": 150, "failed": 1, "accepted_drafts": 650, "verification_drafts": 800, + "total_pages": 100, "free_pages": 25, "cached_pages": 10, "page_tokens": 128, + "learned_drafter": True, "host_hits": 99, + })) + assert metrics == {"generated_tokens": 12000, "prefill_tokens": 48000, "active_requests": 2, + "pending_requests": 0, "requests_completed": 150, "requests_failed": 1, + "drafts_accepted": 650, "draft_tokens": 800, + "kv_tokens_total": 12800, "kv_tokens_free": 3200, "kv_tokens_cached": 1280} + + +@pytest.mark.parametrize("body", [ + "{}", "[]", "null", '{"unknown": 1}', '{"flare_generated_tokens_total": true}', + '{"generated_tokens": null, "active": "not a number"}', + '{"generated_tokens": NaN, "active": Infinity}', '{"generated_tokens":', +]) +def test_flare_json_without_usable_numbers_has_no_series(body): + assert dashboard.parse_flare_metrics(body) == {} + + +def test_json_flare_metrics_feed_the_external_card_and_token_rate(tmp_path): + found, api, clock = flare_collector(tmp_path, pairs=()) + payload = {"generated_tokens": 100, "prefill_tokens": 400, "active": 2, "pending": 0, + "completed": 10, "failed": 0, "total_pages": 100, "free_pages": 25, "page_tokens": 128} + api.bodies[(FLARE_HOST, FLARE_PORT, "/metrics")] = json.dumps(payload).encode() + found.mark_viewed() + found.cycle() + clock.now += dashboard.METRICS_VIEWED_SECONDS + api.bodies[(FLARE_HOST, FLARE_PORT, "/metrics")] = json.dumps({**payload, "generated_tokens": 160}).encode() + found.mark_viewed() + found.cycle() + [card] = found.snapshot()["external"] + assert card["state"] == "serving" and card["model"] == "qwen3.8-flash" + assert card["stats"]["output_tps"] == 30 + assert card["stats"]["active_requests"] == 2 and card["stats"]["kv_in_use"] == 0.75 + assert card["metrics_problem"] is None + + +@pytest.mark.parametrize("body", ["{}", '{"unknown": 1}', "[]", "", "# no Flare series\n"]) +def test_empty_flare_metrics_are_visible_and_recovery_clears_the_warning(tmp_path, body): + found, api, clock = flare_collector(tmp_path, pairs=()) + run_for(found, clock, 4, viewer=True) + api.bodies[(FLARE_HOST, FLARE_PORT, "/metrics")] = body.encode() + run_for(found, clock, 2, viewer=True) + [card] = found.snapshot()["external"] + assert card["state"] == "serving" and card["stats"] is None + assert card["metrics_problem"] == "No recognized numeric Flare metrics in /metrics response." + external = found.externals["flare-spark-6"] + assert external.sample is None and external.output_tps is None + api.bodies[(FLARE_HOST, FLARE_PORT, "/metrics")] = flare_metrics(generated=12400).encode() + run_for(found, clock, 2, viewer=True) + [card] = found.snapshot()["external"] + assert card["metrics_problem"] is None and card["stats"]["generated_tokens"] == 12400 + assert card["stats"]["output_tps"] is None + + def test_the_external_list_is_validated(tmp_path): assert dashboard.read_external(tmp_path / "missing.json") == [] rows = dashboard.read_external(external_file(tmp_path)) diff --git a/runtime/host/test_flare_dashboard.py b/runtime/host/test_flare_dashboard.py new file mode 100644 index 00000000..f5fab5b3 --- /dev/null +++ b/runtime/host/test_flare_dashboard.py @@ -0,0 +1,254 @@ +"""Flare processes that SparkRing did not install: launch facts, grouping and cards.""" +import json +from urllib.parse import urlparse + +from runtime.host import dashboard, node_containers as nc +from runtime.host.test_dashboard import collector + + +def flare_argv(rank=None, rendezvous=None, host=None, port=None, model=None): + argv = ["/srv/flare/artifacts/bin/flare", "run", "model.safetensors", "--gpus", "0", + "--dangerously-approve-key", "SECRET"] + if rank is not None: + argv += ["--rank", str(rank)] + if rendezvous is not None: + argv += ["--rendezvous", rendezvous] + if host is not None: + argv += ["--host", host] + if port is not None: + argv += ["--port", str(port)] + if model is not None: + argv += ["--served-model-name", model] + return argv + + +def flare_row(argv): + facts = nc.launch_facts(argv) + return {**facts, "managed": False, "name": nc.flare_row_name(facts), "image": "", "image_id": ""} + + +def flare_metrics(generated=500, running=2, total=1000, free=250, uptime=100): + return "\n".join([ + "# TYPE flare_active_requests gauge", + f"flare_active_requests {running}", + "flare_pending_requests 1", + f"flare_total_kv_tokens {total}", + f"flare_free_kv_tokens {free}", + "flare_cached_kv_tokens 100", + f"flare_generated_tokens_total {generated}", + "flare_prefill_tokens_total 2000", + "flare_requests_completed_total 10", + "flare_requests_failed_total 1", + "flare_requests_cancelled_total 2", + "flare_preemptions_total 3", + "flare_speculative_offered_drafts_total 40", + "flare_speculative_accepted_drafts_total 30", + "flare_verification_draft_tokens_total 50", + "flare_cache_lookups_total 10", + "flare_cache_hits_total 6", + f"flare_uptime_seconds {uptime}", + ]) + "\n" + + +def node_doc(fabric, position, rows, now): + return {"node_id": fabric["positions"][position]["node_id"], "observed_at": now, + "model_containers": rows} + + +def install_reports(found, documents, tmp_path): + path = tmp_path / "nodes.json" + path.write_text(json.dumps({"schema": "sparkring-dashboard-nodes/v1", "nodes": documents}), encoding="utf-8") + found.node_reports_path = path + found.next_files = 0 + + +def served(found, api, clock, documents, tmp_path, host, port, model="Flare-Model"): + install_reports(found, documents, tmp_path) + api.models[(host, port)] = model + api.bodies[(host, port, "/metrics")] = flare_metrics().encode() + found.mark_viewed() + found.cycle() + return found.snapshot()["groups"][0]["members"][0] + + +def test_flare_launch_facts_cover_orbit_manual_and_default_binds(): + rank0 = nc.launch_facts(flare_argv(0, "198.18.1.1:29836", "0.0.0.0", 18985, "DeepSeek-V4.1-Flash")) + assert rank0 == {"engine": "flare", "node_rank": 0, "headless": False, + "master_addr": "198.18.1.1", "master_port": 29836, + "api_bind": "0.0.0.0", "api_port": 18985, + "served_model": "DeepSeek-V4.1-Flash"} + rank1 = nc.launch_facts(flare_argv(1, "198.18.1.1:29836")) + assert rank1["node_rank"] == 1 and rank1["headless"] and "api_port" not in rank1 + single = nc.launch_facts(flare_argv(host="0.0.0.0", port=8001)) + assert (single["node_rank"], single["headless"], single["api_port"]) == (0, False, 8001) + default = nc.launch_facts(["flare-8ff0a231f54c37fccf3250d2", "run", "model.safetensors"]) + assert (default["api_bind"], default["api_port"]) == ("127.0.0.1", 8000) + assert "SECRET" not in json.dumps(rank0) + assert nc.launch_facts(["/x/flare", "agent", "--listen", "0.0.0.0:29900"]) == {"engine": "unknown"} + assert nc.launch_facts(["/x/flare", "orbit", "run", "model"]) == {"engine": "unknown"} + + +def test_unparsable_flare_host_is_not_published_as_loopback(): + facts = nc.launch_facts(flare_argv(host="spark-0.lan", port=8001)) + assert facts["engine"] == "flare" and "api_bind" not in facts and "api_port" not in facts + + +def test_bare_flare_process_is_observed_without_moving_its_argv(tmp_path): + proc = tmp_path / "proc" / "50" + proc.mkdir(parents=True) + (proc / "cgroup").write_text("0::/system.slice/sshd.service") + (proc / "status").write_text("Name:\tflare\nPPid:\t1\n") + argv = flare_argv(0, "198.18.1.1:29836", "0.0.0.0", 18985, "DeepSeek-V4.1-Flash") + (proc / "cmdline").write_bytes(b"\0".join(a.encode() for a in argv) + b"\0") + (proc / "environ").write_bytes(b"TOKEN=PRIVATE") + observed = nc.observations({"gpu": {"compute_processes": [50]}, "docker": {"containers": []}}, root=tmp_path) + assert len(observed) == 1 + row = observed[0] + assert row["engine"] == "flare" and row["api_port"] == 18985 and row["managed"] is False + assert row["name"].startswith("flare-r0-") + assert "SECRET" not in json.dumps(row) and "PRIVATE" not in json.dumps(row) + + +def test_flare_agent_and_orbit_commands_are_not_ranks(tmp_path): + proc = tmp_path / "proc" / "51" + proc.mkdir(parents=True) + (proc / "cgroup").write_text("0::/system.slice/sshd.service") + (proc / "status").write_text("Name:\tflare\nPPid:\t1\n") + (proc / "cmdline").write_bytes(b"\0".join(a.encode() for a in + ["/x/flare", "agent", "--listen", "0.0.0.0:29900"]) + b"\0") + assert nc.observations({"gpu": {"compute_processes": [51]}, "docker": {"containers": []}}, + root=tmp_path) == [] + + +def test_containerized_flare_is_observed_from_its_entrypoint(tmp_path): + cid = "a" * 64 + observed = nc.observations( + {"docker": {"containers": [{"id": cid, "state": "running", "name": "flare"}]}}, + root=tmp_path, + inspect=lambda *a, **kw: {"Path": "/usr/local/bin/flare", + "Args": flare_argv(0, "198.18.1.1:29836", "0.0.0.0", 8001, "Qwen"), + "Image": "sha256:" + "b" * 64, + "Config": {"Image": "flare:tag"}}) + assert observed[0]["engine"] == "flare" and observed[0]["image"] == "flare:tag" + assert observed[0]["api_port"] == 8001 and observed[0]["managed"] is False + + +def test_single_flare_group_without_rendezvous_is_one_spark(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + row = flare_row(flare_argv(host="0.0.0.0", port=8001)) + documents = [node_doc(fabric, 5, [row], clock.now)] + member = served(found, api, clock, documents, tmp_path, fabric["positions"][5]["address"], 8001, + "Qwen3.8-Flash") + assert member["flare"] and member["layout"]["label"] == "Flare" + assert member["arc"]["positions"] == [5] and member["head"]["position"] == 5 + assert member["state"] == "serving" and member["model"] == "Qwen3.8-Flash" + assert member["live"] and member["live"]["running"] == 2.0 and member["live"]["kv"] == 0.75 + assert member["totals"]["finished"] == {"stop": 10.0, "error": 1.0, "abort": 2.0} + assert member["group_note"] is None + assert len(found.history[member["id"]]) == 1 + assert {urlparse(r.full_url).port for r in api.requests} == {8001} + + +def test_multi_spark_flare_group_uses_rendezvous_and_only_head_api(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + documents = [node_doc(fabric, rank, [flare_row(flare_argv( + rank, "198.18.1.1:29836", "0.0.0.0" if rank == 0 else None, 18985 if rank == 0 else None, + "DeepSeek-V4.1-Flash" if rank == 0 else None))], clock.now) for rank in range(4)] + member = served(found, api, clock, documents, tmp_path, "192.0.2.10", 18985, + "DeepSeek-V4.1-Flash") + assert member["arc"]["positions"] == [0, 1, 2, 3] + assert member["head"]["position"] == 0 and member["head"]["port"] == 18985 + assert member["group_note"] is None and member["layout"]["count"] == 4 + assert member["workers"] == {"received": 4, "expected": 4} + assert {urlparse(r.full_url).port for r in api.requests} == {18985} + assert not any(urlparse(r.full_url).port == 8000 for r in api.requests) + + +def test_missing_flare_head_is_named_not_counted_as_complete(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + documents = [node_doc(fabric, rank, [flare_row(flare_argv(rank, "198.18.1.1:29836"))], clock.now) + for rank in (1, 2)] + install_reports(found, documents, tmp_path) + found.mark_viewed() + found.cycle() + member = found.snapshot()["groups"][0]["members"][0] + assert member["group_note"] == "head Spark not reporting" + assert member["workers"] == {"received": 2, "expected": 3} + assert member["arc"]["positions"] == [1, 2] and member["state"] == "busy" + assert api.requests == [] + + +def test_lone_flare_peer_names_the_missing_head(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + documents = [node_doc(fabric, 1, [flare_row(flare_argv(1, "198.18.1.1:29836"))], clock.now)] + install_reports(found, documents, tmp_path) + found.mark_viewed() + found.cycle() + member = found.snapshot()["groups"][0]["members"][0] + assert member["group_note"] == "head Spark not reporting" + assert member["workers"] == {"received": 1, "expected": 2} + assert member["state"] == "busy" and api.requests == [] + + +def test_loopback_bound_flare_group_shows_process_without_live_counters(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + documents = [node_doc(fabric, 6, [flare_row(flare_argv(host="127.0.0.1", port=8000))], clock.now)] + install_reports(found, documents, tmp_path) + found.mark_viewed() + found.cycle() + member = found.snapshot()["groups"][0]["members"][0] + assert member["api_local"] is True and member["endpoint"] is None + assert member["state"] == "busy" and member["head"]["port"] == 8000 + assert member["flare"] is True + assert api.requests == [] + + +def test_flare_metric_adapter_feeds_the_shared_history_keys(): + metrics = dashboard.parse_flare_as_metrics(flare_metrics()) + assert {k: metrics[k] for k in ("gen", "prompt", "running", "waiting", "kv", "accepted", + "drafts", "draft_tokens", "pc_hits", "pc_queries", + "preemptions", "uptime_seconds")} == { + "gen": 500.0, "prompt": 2000.0, "running": 2.0, "waiting": 1.0, "kv": 0.75, + "accepted": 30.0, "drafts": 40.0, "draft_tokens": 50.0, "pc_hits": 6.0, + "pc_queries": 10.0, "preemptions": 3.0, "uptime_seconds": 100.0} + assert metrics["finished"] == {"stop": 10.0, "error": 1.0, "abort": 2.0} + + +def test_discovered_flare_reports_empty_metrics_then_recovers_with_a_json_snapshot(tmp_path): + found, api, clock = collector(tmp_path, rows=()) + found.reload_sources() + fabric = found.sources[0].fabric + row = flare_row(flare_argv(host="0.0.0.0", port=8001)) + install_reports(found, [node_doc(fabric, 5, [row], clock.now)], tmp_path) + host = fabric["positions"][5]["address"] + api.models[(host, 8001)] = "Flare-Model" + api.bodies[(host, 8001, "/metrics")] = b"{}" + found.mark_viewed() + found.cycle() + member = found.snapshot()["groups"][0]["members"][0] + assert member["flare"] and member["state"] == "serving" + assert member["metrics_problem"] == "No recognized numeric Flare metrics in /metrics response." + assert member["counters"]["values"] == {} + clock.now += dashboard.METRICS_VIEWED_SECONDS + api.bodies[(host, 8001, "/metrics")] = json.dumps({ + "generated_tokens": 500, "prefill_tokens": 2000, "active": 2, "pending": 1, + "total_pages": 100, "free_pages": 25, "page_tokens": 128, "completed": 10, + }).encode() + found.mark_viewed() + found.cycle() + member = found.snapshot()["groups"][0]["members"][0] + assert member["metrics_problem"] is None and member["model"] == "Flare-Model" + assert member["counters"]["values"]["gen"] == 500 + assert member["live"]["running"] == 2 and member["live"]["kv"] == 0.75 + assert member["totals"]["finished"] == {"stop": 10} + assert {urlparse(r.full_url).path for r in api.requests} == {"/metrics", "/v1/models"}