diff --git a/docs/Generative-Mining.md b/docs/Generative-Mining.md index bfc2f8bb..1f543c8d 100644 --- a/docs/Generative-Mining.md +++ b/docs/Generative-Mining.md @@ -212,6 +212,60 @@ gascli generator restart gascli generator delete ``` +### Checking Your Performance + +```bash +gascli g perf # Image/video charts and verification stats +gascli g perf --modality image # Image only +gascli g perf --lookback-days 14 # Two weeks of chart history +gascli g perf --json # Raw API response, without terminal formatting +gascli g perf --no-chain # Skip chain RPC; keep benchmark charts +gascli g perf --netuid 379 --chain-endpoint test # Explicit testnet chain context +``` + +Rich panels and Plotext's high-resolution curves show **sample-weighted rolling +seven-day fool rates**, with muted threshold lines and dated axes. Each modality +needs at least 20 benchmark evaluations and a +rate strictly above its threshold: 2% for images, 1% for videos. Changing +`--lookback-days` changes the chart span (and the legacy aggregate window), not +the seven-day qualification window. + +Counts are evaluations, not unique media: the same media can be evaluated in +multiple benchmark runs. Missing data is shown as a gap, not a zero. Historical +points are reconstructed from currently available benchmark results; they are +not a record of what validators knew or paid at that time. + +Fool-rate eligibility is **not a guarantee of on-chain incentive**. Verified +activity, validator updates, and weight reveal/epoch timing also affect rewards. +Not participating in video does not disqualify an image miner. The CLI does not +infer participation from missing benchmark data. + +The on-chain panel reads the selected hotkey's current normalized incentive +score (not a token amount), current revealed weights toward it, and the latest +submission block/time for the five largest permitted validators by stake. +Amber vertical chart markers match those submission timestamps. These are the +**latest submission per validator, not a complete event history**. On commit-reveal +subnets, `LastUpdate` records commits, not reveals; the currently revealed weight +may belong to an earlier commit. No pending-payment or reveal ETA is inferred. + +Chain values are read at one metagraph block, with its timestamp shown separately +from the benchmark snapshot. Timestamp lookups older than 7,200 blocks are skipped +to avoid archive-node delays; their exact submission blocks remain visible. +Missing/failed RPC data is shown as unknown, not zero. `--chain-timeout` bounds the +lookup (default 30 seconds); partial snapshots remain useful on timeout. +`--netuid` / `--chain-endpoint` override `BT_NETUID` / `BT_CHAIN_ENDPOINT`, falling +back to subnet 34 on finney. Select the correct network when using a non-production +API. `--json` remains the raw API response and does not perform chain queries. + +The API supplies the chart history. Older API deployments still work, displaying +the aggregate summary until the history addition is deployed. Existing CLI +versions continue to read the unchanged verification and aggregate fields. + +Charts use the available terminal width; very narrow terminals show the numbers +without a plot. Redirected output remains free of color escape sequences, and +`NO_COLOR` disables colors. To preview the layout without a wallet or API call, +run `python -m scripts.preview_generator_performance --width 100` (demo data). + ### API Endpoints Your miner exposes these endpoints for validators: diff --git a/gas/cli.py b/gas/cli.py index 33af9fa7..584ce12c 100644 --- a/gas/cli.py +++ b/gas/cli.py @@ -896,10 +896,14 @@ def info(): help="Bittensor hotkey name (default: BT_WALLET_HOTKEY from .env.gen_miner, else 'default')", ) @click.option("--modality", type=click.Choice(["image", "video", "audio"]), default=None, help="Filter fool-rate aggregate by modality") -@click.option("--lookback-days", default=7, type=int, show_default=True, help="Fool-rate parquet lookback (days)") +@click.option("--lookback-days", default=7, type=click.IntRange(1, 90), show_default=True, help="Chart/aggregate lookback; qualification always uses 7 days") @click.option("--api-url", default=None, help="GAS API base URL (default: GAS_API_URL env or production)") +@click.option("--chain/--no-chain", default=True, help="Show current incentive and latest validator submissions (text output only)") +@click.option("--netuid", type=click.IntRange(0, 65535), default=None, help="Chain subnet (default: BT_NETUID or 34)") +@click.option("--chain-endpoint", default=None, help="Chain network/endpoint (default: BT_CHAIN_ENDPOINT or finney)") +@click.option("--chain-timeout", type=click.IntRange(1, 120), default=30, show_default=True, help="Maximum seconds for the optional chain lookup") @click.option("--json", "as_json", is_flag=True, help="Print raw API JSON") -def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_json): +def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, chain, netuid, chain_endpoint, chain_timeout, as_json): """Query your generator verification stats + aggregated fool rate (Epistula-authenticated GAS API). The request is signed with your hotkey (same Epistula flow as discriminator ``d perf``). @@ -920,10 +924,6 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js sys.exit(1) addr = wallet.hotkey.ss58_address - click.echo() - click.echo(f" ⛽ generator {addr}") - click.echo() - result = fetch_generator_performance( wallet, modality=modality, @@ -940,6 +940,24 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js click.echo(json_lib.dumps(data, indent=2, default=str)) return + chain_data = {"status": "disabled"} + if chain: + from gas.protocol.generator_chain import fetch_generator_chain + selected_netuid = netuid if netuid is not None else click.IntRange(0, 65535).convert( + os.environ.get("BT_NETUID", "34"), None, click.get_current_context()) + chain_data = fetch_generator_chain( + addr, network=chain_endpoint or os.environ.get("BT_CHAIN_ENDPOINT") or "finney", + netuid=selected_netuid, timeout=chain_timeout) + + from gas.utils.generator_performance import render_fool_history, render_chain_context + if render_fool_history(data, modality, hotkey=addr, lookback_days=lookback_days, chain=chain_data): + return + render_chain_context(chain_data) + click.echo() + click.echo(f" ⛽ generator {addr}") + click.echo(" History unavailable from this API; showing aggregate only.") + click.echo() + ver = data.get("verification") or {} fool = data.get("fool_aggregate") or {} @@ -967,7 +985,7 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js click.echo(" (no verification rows yet)") click.echo() - click.echo(f" Fool rate aggregate (benchmark parquets, last {lookback_days}d)") + click.echo(f" Fool rate aggregate (benchmark evaluations, last {lookback_days}d)") ts = fool.get("total_samples", 0) if ts: click.echo( diff --git a/gas/protocol/generator_chain.py b/gas/protocol/generator_chain.py new file mode 100644 index 00000000..a9e5bb34 --- /dev/null +++ b/gas/protocol/generator_chain.py @@ -0,0 +1,208 @@ +"""Bounded, read-only chain context for generator performance. + +LastUpdate tracks submissions (commits on commit-reveal subnets), not reveals. +No wallet is opened and no extrinsics are submitted. RPC work runs in a child +process so a stalled endpoint cannot hang the CLI or contaminate JSON output. +""" + +import json +import subprocess +import sys + +PREFIX = "GAS_CHAIN_SNAPSHOT " +MAX_VALIDATORS = 5 +# Avoid archive fallbacks for stale validators. Their exact block is still shown. +MAX_TIMESTAMP_AGE_BLOCKS = 7200 +REVEAL_INDEX_URL = "https://api.metagraph.sh/api/v1/chain-events" +MAX_REVEAL_PAGES = 8 + + +def add_recent_reveals(subtensor, snapshot, publish=lambda snapshot: None, get=None): + """Use a bounded public index as a locator; verify events/times on chain. + + This is latest *observed* reveal context, not a complete history or proof + that a particular commit has revealed. No wallet/miner address is sent to + the index. Only Finney's modern timelock events are indexed here. + """ + if not snapshot.get("commit_reveal_enabled") or not snapshot.get("validators"): + return + if snapshot["network"] != "finney": + snapshot["reveal_note"] = "Recent reveal lookup is available on Finney only." + return + import requests + get = get or requests.get + snapshot["reveal_note"] = "Recent indexed reveals only (Metagraphed), verified on chain; unknown ≠ not revealed." + publish(snapshot) + rows = {row["hotkey"]: row for row in snapshot["validators"]} + active = {key for key, row in rows.items() if row.get("submitted_block") and + snapshot["block"] - row["submitted_block"] <= MAX_TIMESTAMP_AGE_BLOCKS} + if not active: + return + params = dict(pallet="SubtensorModule", method="TimelockedWeightsRevealed", limit=100) + checked = set() + cursors = set() + try: + for _ in range(MAX_REVEAL_PAGES): + response = get(REVEAL_INDEX_URL, params=dict(params), timeout=3) + response.raise_for_status() + body = response.json() + if (response.headers.get("x-metagraph-degraded") or not body.get("ok") or + body.get("meta", {}).get("source") == "data-worker-unavailable"): + raise ValueError("Reveal index unavailable") + data = body["data"] + for event in data["events"]: + block = int(event["block_number"]) + if not 0 <= snapshot["block"] - block <= MAX_TIMESTAMP_AGE_BLOCKS: + continue + args = event.get("args") or [] + if (event.get("pallet") != "SubtensorModule" or + event.get("method") != "TimelockedWeightsRevealed" or len(args) != 2): + continue + netuid = args[0] + if isinstance(netuid, (list, tuple)) and len(netuid) == 1: + netuid = netuid[0] + if netuid != snapshot["netuid"] or args[1] not in rows or block in checked: + continue + checked.add(block) + # A locator cannot fabricate a reveal or substitute index ingestion + # time for the actual chain timestamp. Decode all reveals in this block. + events = subtensor.substrate.get_events(block_hash=subtensor.get_block_hash(block)) + verified = [] + for record in events: + raw = record.get("event", record) + values = raw.get("attributes") or [] + if (raw.get("module_id") == "SubtensorModule" and + raw.get("event_id") == "TimelockedWeightsRevealed" and + len(values) == 2 and values[0] == snapshot["netuid"] and values[1] in rows): + verified.append(values[1]) + if verified: + at = subtensor.get_timestamp(block=block).isoformat() + for key in verified: + if block > (rows[key].get("revealed_block") or 0): + rows[key].update(revealed_block=block, revealed_at=at) + publish(snapshot) + if all(rows[key].get("revealed_block") for key in active): + break + cursor = data.get("next_cursor") + if not cursor or cursor in cursors: + break + cursors.add(cursor) + params["cursor"] = cursor + except Exception: + snapshot["reveal_note"] = "Reveal lookup incomplete/unavailable; verified events retained. Unknown ≠ not revealed." + publish(snapshot) + + +def _last_snapshot(output): + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + for line in reversed((output or "").splitlines()): + if line.startswith(PREFIX): + try: + return json.loads(line[len(PREFIX):]) + except (ValueError, TypeError): + continue + return None + + +def fetch_generator_chain(hotkey, *, network="finney", netuid=34, timeout=30): + """Return unknown/partial on failure; never turn an RPC failure into zero.""" + base = dict(status="unavailable", network=network, netuid=netuid, + incentive=None, validators=[]) + try: + result = subprocess.run( + [sys.executable, "-m", __name__, hotkey, network, str(netuid)], + capture_output=True, text=True, timeout=timeout, check=False, + ) + snapshot = _last_snapshot(result.stdout) + if snapshot is not None: + return snapshot + except subprocess.TimeoutExpired as exc: + snapshot = _last_snapshot(exc.stdout) + if snapshot is not None: + snapshot.update(status="partial", warning="Chain lookup timed out; partial snapshot shown.") + return snapshot + base["warning"] = "Chain lookup timed out; incentive is unknown, not zero." + return base + except OSError: + pass + base["warning"] = "Chain lookup unavailable; incentive is unknown, not zero." + return base + + +def collect_snapshot(subtensor, hotkey, netuid, network, publish=lambda snapshot: None): + """All values are read at one metagraph block; timestamps come from chain.""" + graph = subtensor.metagraph(netuid, lite=True) + block = int(graph.block) + snapshot = dict(status="partial", network=network, netuid=netuid, block=block, + incentive=None, validators=[], as_of=None) + if hotkey not in graph.hotkeys: + snapshot.update(status="not_registered", warning="Hotkey is not registered on this subnet.") + return snapshot + uid = graph.hotkeys.index(hotkey) + snapshot.update(uid=uid, incentive=float(graph.I[uid])) + publish(snapshot) + try: + snapshot["as_of"] = subtensor.get_timestamp(block=block).isoformat() + snapshot["commit_reveal_enabled"] = bool(subtensor.commit_reveal_enabled(netuid, block=block)) + weights = dict(subtensor.weights(netuid, block=block)) + permitted = [i for i, permit in enumerate(graph.validator_permit) if bool(permit)] + snapshot["validator_count"] = len(permitted) + snapshot["positive_weight_count"] = sum( + any(int(target) == uid and int(value) > 0 for target, value in weights.get(i, [])) + for i in permitted) + chosen = sorted(permitted, key=lambda i: float(graph.S[i]), reverse=True)[:MAX_VALIDATORS] + for i in chosen: + entries = weights.get(i, []) + total = sum(int(value) for _, value in entries) + raw_weight = sum(int(value) for target, value in entries if int(target) == uid) + last = int(graph.last_update[i]) + snapshot["validators"].append(dict( + uid=i, hotkey=graph.hotkeys[i], + weight=raw_weight / total if total else 0.0, + submitted_block=last if 0 < last <= block else None, + submitted_at=None, + timing_note=(f">{MAX_TIMESTAMP_AGE_BLOCKS} blocks ago" + if last > 0 and block - last > MAX_TIMESTAMP_AGE_BLOCKS else None), + )) + publish(snapshot) + timestamps = {} + for validator in snapshot["validators"]: + last = validator["submitted_block"] + if last and block - last <= MAX_TIMESTAMP_AGE_BLOCKS: + if last not in timestamps: + timestamps[last] = subtensor.get_timestamp(block=last).isoformat() + validator["submitted_at"] = timestamps[last] + publish(snapshot) + snapshot["status"] = "ok" + except Exception: + snapshot["warning"] = "Some chain details are unavailable; missing timestamps are not estimated." + return snapshot + + +def _main(): + import bittensor as bt + hotkey, network, netuid = sys.argv[1:] + + def publish(snapshot): + # Child-process protocol, not user-facing logging. + sys.stdout.write(PREFIX + json.dumps(snapshot, allow_nan=False) + "\n") + sys.stdout.flush() + + chain = None + try: + chain = bt.Subtensor(network=network, retry_forever=False) + snapshot = collect_snapshot(chain, hotkey, int(netuid), network, publish) + publish(snapshot) + add_recent_reveals(chain, snapshot, publish) + publish(snapshot) + except Exception: + publish(dict(status="unavailable", network=network, netuid=int(netuid), + incentive=None, validators=[], warning="Chain lookup unavailable; incentive is unknown, not zero.")) + finally: + if chain is not None: + chain.close() + + +if __name__ == "__main__": + _main() diff --git a/gas/utils/generator_performance.py b/gas/utils/generator_performance.py new file mode 100644 index 00000000..636638fe --- /dev/null +++ b/gas/utils/generator_performance.py @@ -0,0 +1,277 @@ +"""Rich performance cards with Plotext's high-resolution terminal curves.""" + +import math +import os +from datetime import datetime, timezone + +import plotext as plt +from rich import box +from rich.console import Console, Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +ACCENTS = {"image": "#5eead4", "video": "#c4b5fd", "audio": "#7dd3fc"} +PLOT_COLORS = {"image": (94, 234, 212), "video": (196, 181, 253), "audio": (125, 211, 252)} +MUTED = "#94a3b8" +THRESHOLD = (148, 163, 184) +STATUS = { + "qualified": ("Fool-rate eligible", "#86efac"), + "below_threshold": ("Below threshold", "#fcd34d"), + "insufficient_samples": ("Insufficient samples", "#fcd34d"), + "no_data": ("No recent data", MUTED), + "not_applicable": ("No generator reward gate", MUTED), +} + + +def _datetime(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + + +def _date(value): + return _datetime(value).strftime("%b %d %H:%M UTC") if value else "none in available history" + + +def _segments(points): + """Never join across missing data, or reinterpret it as zero.""" + segment = [] + for point in points: + rate = point.get("fool_rate") + if rate is None: + if segment: + yield segment + segment = [] + else: + segment.append((_datetime(point["at"]).timestamp(), rate * 100)) + if segment: + yield segment + + +def line_chart(points, threshold, width=80, height=12, modality="image", submissions=()): + """Build colored text without printing ANSI codes directly to stdout.""" + segments = list(_segments(points)) + if not segments or width < 24: + return None + start, end = _datetime(points[0]["at"]), _datetime(points[-1]["at"]) + left, right = start.timestamp(), end.timestamp() + if left == right: + left, right = left - 3600, right + 3600 + max_rate = max(rate for segment in segments for _, rate in segment) + ceiling = max(1, max_rate * 1.15, (threshold or 0) * 125) + step = next(s for s in (.25, .5, 1, 2, 5, 10, 20, 25, 50) if s >= ceiling / 4) + ceiling = math.ceil(ceiling / step) * step + # Plotext uses global state. Each synchronous render starts/ends clean. + # Never call show(), which would bypass Rich's redirect/color handling. + plt.clear_figure() + try: + plt.limit_size(False, False) + plt.plot_size(width, height) + plt.theme("clear") + plt.canvas_color("default") + plt.axes_color("default") + plt.ticks_color(THRESHOLD) + plt.xaxes(True, False) + plt.yaxes(True, False) + plt.grid(False, False) + plt.xlim(left, right) + plt.ylim(0, ceiling) + ticks = [step * i for i in range(round(ceiling / step) + 1)] + plt.yticks(ticks, [f"{t:.1f}%" for t in ticks]) + count = min(8, max(2, (width - 9) // 10)) + positions = [left + (right - left) * i / (count - 1) for i in range(count)] + fmt = "%H:%M" if (end - start).total_seconds() <= 86400 else "%b %d" + plt.xticks(positions, [datetime.fromtimestamp(t, timezone.utc).strftime(fmt) for t in positions]) + if threshold is not None: + plt.horizontal_line(threshold * 100, color=(100, 116, 139)) + for segment in segments: + xs, ys = zip(*segment) + plt.plot(list(xs), list(ys), marker="braille", + color=PLOT_COLORS.get(modality, PLOT_COLORS["image"])) + chart = Text.from_ansi(plt.build().rstrip("\n")) + # Separate event tracks preserve both types even when their timestamps + # round to the same terminal column. Align to the rendered plot axis, + # whose left margin varies with the percentage tick labels. + axis = next((line for line in chart.plain.splitlines() if "└" in line), None) + if axis is not None: + first = axis.index("└") + 1 + last = len(axis.rstrip()) - 1 + for field, label, color in (("submitted_at", "S", "#fbbf24"), + ("revealed_at", "R", "#60a5fa")): + columns = set() + for row in submissions: + if row.get(field): + timestamp = _datetime(row[field]).timestamp() + if left <= timestamp <= right: + columns.add(round((timestamp - left) / (right - left) * (last - first))) + if columns: + track = "".join("│" if i in columns else " " for i in range(last - first + 1)) + chart.append("\n" + (label + " ").rjust(first) + track, style=color) + return chart + finally: + plt.clear_figure() + + +def _modality_panel(lane, entry, width, submissions=()): + accent = ACCENTS.get(lane, ACCENTS["image"]) + current = entry["current"] + rate = current.get("fool_rate") + label, status_color = STATUS.get(entry.get("status"), ("Unknown", MUTED)) + heading = Table.grid(expand=True) + heading.add_column(ratio=1) + heading.add_column(justify="right") + heading.add_row(Text(f"{rate:.2%}" if rate is not None else "—", style=f"bold {accent}"), + Text(f"● {label}", style=status_color)) + counts = Text(f"{current['fooled_count']:,} fooled / {current['total_samples']:,} evaluations" + f" · {current['benchmark_run_count']:,} runs", style=MUTED) + threshold = entry.get("threshold") + gate = (f"Gate >{threshold:.0%} + {entry['minimum_samples']} evaluations minimum" + if threshold is not None else "No generator reward gate for this modality") + points = entry.get("points") or [] + chart = line_chart(points, threshold, width=max(0, width - 6), modality=lane, submissions=submissions) + body = [heading, counts, Text("")] + if chart is not None: + body.extend([chart, Text("")]) + elif not any(p.get("fool_rate") is not None for p in points): + body.extend([Text("No benchmark observations in this history window.", style=MUTED), Text("")]) + else: + body.extend([Text("Widen the terminal to show the chart.", style=MUTED), Text("")]) + legend = Text("━ rolling 7-day fool rate", style=accent) + if threshold is not None: + legend.append(f" ─ {threshold:.0%} threshold", style=MUTED) + legend.append(" UTC", style=MUTED) + if chart is not None: + body.append(legend) + events = Text() + for field, label, color in (("submitted_at", "S latest submissions", "#fbbf24"), + ("revealed_at", "R recent reveals", "#60a5fa")): + if any(row.get(field) and _datetime(points[0]["at"]) <= _datetime(row[field]) <= _datetime(points[-1]["at"]) for row in submissions): + if events.plain: + events.append(" ") + events.append(label, style=color) + if events.plain: + body.extend([events, Text("Latest events only, not 7-day event history; separate tracks share the chart's time axis.", style=MUTED)]) + body.append(Text(gate, style=MUTED)) + return Panel(Group(*body), title=Text(f" {lane.upper()} ", style=f"bold {accent}"), + subtitle=Text(f"Last eval: {_date(entry.get('last_evaluated_at'))}", style=MUTED), + title_align="left", subtitle_align="right", box=box.ROUNDED, + border_style="#475569", padding=(1, 2), width=width) + + +def _verification_panel(verification): + rate = verification.get("aggregate_pass_rate") + summary = Text(f"{verification.get('validator_count', 0)} validators reporting", style=MUTED) + summary.append(f" · {rate:.1%} pass" if rate is not None else " · pass rate unavailable") + summary.append(f" · {verification.get('total_verified', 0):,} verified" + f" / {verification.get('total_failed', 0):,} failed" + f" / {verification.get('total_evaluated', 0):,} evaluated") + parts = [summary] + rows = verification.get("by_validator") or [] + if rows: + table = Table(box=None, expand=True, padding=(0, 1), header_style=MUTED) + for heading in ("Validator", "Pass", "Verified", "Failed", "Window"): + table.add_column(heading, overflow="fold") + for row in rows: + pr = row.get("pass_rate") + table.add_row(Text((row.get("validator_hotkey") or "")[:12] + "…"), + f"{pr:.1%}" if pr is not None else "—", + str(row.get("total_verified", 0)), str(row.get("total_failed", 0)), + f"{row.get('lookback_hours', '?')}h") + parts.extend([Text(""), table]) + return Panel(Group(*parts), title=" VERIFICATION ", title_align="left", + border_style="#475569", box=box.ROUNDED, padding=(0, 2)) + + +def _console(): + import click + context = click.get_current_context(silent=True) + color = context.color if context is not None else None + return Console(no_color="NO_COLOR" in os.environ or color is False, force_terminal=color) + + +def render_chain_context(chain, console=None): + console = console or _console() + chain = chain or {"status": "disabled"} + if chain.get("status") == "disabled": + return + incentive = chain.get("incentive") + parts = [] + if incentive is not None: + color = "#86efac" if incentive > 0 else "#fcd34d" + parts.append(Text(f"Incentive {incentive:.6g} ({incentive:.4%}) · UID {chain['uid']}", style=f"bold {color}")) + network = chain.get("network") if chain.get("network") in ("finney", "test", "local") else "custom RPC" + at = _date(chain["as_of"]) if chain.get("as_of") else "timestamp unavailable" + parts.append(Text(f"{network} / SN{chain['netuid']} · block {chain['block']:,} · {at}", style=MUTED)) + else: + parts.append(Text("Incentive — not registered" if chain.get("status") == "not_registered" + else "Incentive — unavailable", style="#fcd34d")) + if "positive_weight_count" in chain: + parts.append(Text(f"{chain['positive_weight_count']}/{chain['validator_count']} permitted validators have positive revealed weight for you.", style=MUTED)) + validators = chain.get("validators") or [] + if validators: + count = chain.get("validator_count", len(validators)) + parts.extend([Text(""), Text(f"Most recent observed events · showing {len(validators)}/{count} validators, largest by stake", style=MUTED)]) + table = Table(box=None, expand=True, padding=(0, 1), header_style=MUTED) + for heading in ("Validator UID", "Revealed weight", "Latest event UTC", "Block"): + table.add_column(heading, overflow="fold") + for row in validators: + submitted = _date(row["submitted_at"]) if row.get("submitted_at") else row.get("timing_note") or "unknown" + label = "Commit" if chain.get("commit_reveal_enabled") else "Submit" + table.add_row(str(row["uid"]), f"{row['weight']:.4%}", + Text(f"{label} {submitted}", style="#fbbf24"), + str(row.get("submitted_block") or "—")) + if chain.get("commit_reveal_enabled"): + revealed = _date(row["revealed_at"]) if row.get("revealed_at") else "unknown" + table.add_row("", "", Text(f"Reveal {revealed}", style="#60a5fa"), + str(row.get("revealed_block") or "—")) + parts.append(table) + parts.append(Text("Weights are current row-normalized shares, not incentive. Events are validator-wide, not miner-specific.", style=MUTED)) + if chain.get("commit_reveal_enabled"): + parts.append(Text("Submission ≠ reveal: these may belong to different commits. Neither marks an incentive change.", style="#fbbf24")) + if chain.get("reveal_note"): + parts.append(Text(chain["reveal_note"], style=MUTED)) + if chain.get("warning"): + parts.append(Text(chain["warning"], style="#fcd34d")) + console.print(Panel(Group(*parts), title=" ON-CHAIN SNAPSHOT ", title_align="left", + box=box.ROUNDED, border_style="#475569", padding=(1, 2))) + console.print() + + +def render_fool_history(data, modality=None, *, hotkey=None, lookback_days=7, console=None, chain=None): + """Old APIs return False; JSON callers bypass this renderer entirely.""" + history = data.get("history") or {} + if not history: + return False + console = console or _console() + console.print() + console.print(Text.assemble(("GAS", "bold #5eead4"), (" / GENERATOR PERFORMANCE", "bold"))) + if hotkey: + console.print(Text(hotkey, style=MUTED)) + console.print(Text(f"rolling 7-day · sample-weighted · snapshot {_date(data.get('as_of'))}", style=MUTED)) + lanes = [modality] if modality else ["image", "video"] + all_points = next((history[lane].get("points") for lane in lanes + if lane in history and history[lane].get("points")), []) + if all_points: + console.print(Text(f"History: {_date(all_points[0]['at'])} → {_date(all_points[-1]['at'])}", style=MUTED)) + console.print() + render_chain_context(chain, console) + submissions = (chain or {}).get("validators") or [] + for lane in lanes: + if lane in history: + console.print(_modality_panel(lane, history[lane], console.width, submissions)) + console.print() + console.print(_verification_panel(data.get("verification") or {})) + if lookback_days != 7: + aggregate = data.get("fool_aggregate") or {} + rate = aggregate.get("fool_rate") + display = f"{rate:.2%}" if rate is not None else "—" + console.print(Text(f"Requested aggregate · last {lookback_days}d: {display}" + f" · {aggregate.get('total_samples', 0):,} evaluations", style=MUTED)) + console.print() + console.print(Text("Counts are evaluations, not unique media. Missing data is not zero.", style=MUTED)) + console.print(Text("Eligibility is per modality; no video data does not block image rewards.", style=MUTED)) + note = ("Chain snapshot shown above; fool-rate eligibility does not guarantee payment." + if (chain or {}).get("incentive") is not None else + "On-chain incentive is not checked or unavailable; verified activity and weight reveal/epoch timing also matter.") + console.print(Text(note, style=MUTED)) + console.print() + return True diff --git a/pyproject.toml b/pyproject.toml index 6ab3d37d..00232007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,8 @@ dependencies = [ "python-dotenv==1.2.2", "accelerate==1.8.1", "click>=8.0.0", + "rich==15.0.0", + "plotext==5.3.2", "datasets==4.0.0", "async-substrate-interface==2.2.1", "ftfy==6.3.1", diff --git a/scripts/preview_generator_performance.py b/scripts/preview_generator_performance.py new file mode 100644 index 00000000..c6c626d0 --- /dev/null +++ b/scripts/preview_generator_performance.py @@ -0,0 +1,71 @@ +"""Render synthetic CLI performance data, without a wallet or API request. + +Run from the repo: python -m scripts.preview_generator_performance --width 100 +Optionally export the exact terminal render with --svg /tmp/gascli-preview.svg. +""" + +import argparse +from datetime import datetime, timedelta, timezone + +from rich.console import Console +from rich.terminal_theme import TerminalTheme + +from gas.utils.generator_performance import render_fool_history + + +def demo_data(): + now = datetime(2026, 9, 28, 16, tzinfo=timezone.utc) + history = {} + examples = [ + ("image", [.004, .006, .005, .009, .014, .012, .017, .019, .016, .021, .024, .022, 7 / 257], 7, 257, .02), + ("video", [.003, .004, .006, .005, .008, .011, .009, .012, .014, .011, .013, .014, .015], 3, 200, .01), + ] + for lane, rates, fooled, total, threshold in examples: + points = [dict(at=(now - timedelta(hours=(12-i)*14)).isoformat(), fool_rate=rate) + for i, rate in enumerate(rates)] + history[lane] = dict( + current=dict(fool_rate=fooled/total, fooled_count=fooled, total_samples=total, benchmark_run_count=20), + threshold=threshold, minimum_samples=20, status="qualified", + last_evaluated_at="2026-09-28T15:02:02Z", points=points) + return dict(as_of=now.isoformat(), history=history, verification=dict( + validator_count=3, aggregate_pass_rate=.975, total_verified=117, + total_failed=3, total_evaluated=120, by_validator=[])) + + +def demo_chain(): + return dict(status="ok", netuid=34, network="finney", block=9167400, + uid=239, incentive=.000183108, as_of="2026-09-28T16:00:00Z", + commit_reveal_enabled=True, validator_count=7, positive_weight_count=5, + reveal_note="Recent indexed reveals only (Metagraphed), verified on chain; unknown ≠ not revealed.", + validators=[ + dict(uid=135, weight=.0002966, submitted_block=9167100, + submitted_at="2026-09-28T15:00:00Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=134, weight=.0012634, submitted_block=9167090, + submitted_at="2026-09-28T14:58:00Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=19, weight=.0013733, submitted_block=9167089, + submitted_at="2026-09-28T14:57:48Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=72, weight=.0001978, submitted_block=9167160, + submitted_at="2026-09-28T15:12:00Z", revealed_block=9167040, + revealed_at="2026-09-28T14:48:00Z"), + dict(uid=5, weight=0, submitted_block=8656798, submitted_at=None, + timing_note=">7200 blocks ago"), + ]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--width", type=int, default=100) + parser.add_argument("--svg") + args = parser.parse_args() + console = Console(width=args.width, height=60, record=True, force_terminal=True, + color_system="truecolor", no_color=False) + console.print("$ gascli g perf [dim]# illustrative demo data[/dim]") + render_fool_history(demo_data(), hotkey="5GKGN7…DmWuGS · DEMO", console=console, chain=demo_chain()) + if args.svg: + theme = TerminalTheme((15, 23, 42), (226, 232, 240), + [(0, 0, 0), (255, 85, 85), (134, 239, 172), (252, 211, 77), + (125, 211, 252), (196, 181, 253), (94, 234, 212), (226, 232, 240)]) + console.save_svg(args.svg, title="gascli g perf · preview", theme=theme) diff --git a/tests/test_generator_chain.py b/tests/test_generator_chain.py new file mode 100644 index 00000000..e5163dd1 --- /dev/null +++ b/tests/test_generator_chain.py @@ -0,0 +1,230 @@ +import copy +import json +import subprocess +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from gas.protocol.generator_chain import (collect_snapshot, fetch_generator_chain, + add_recent_reveals, PREFIX, MAX_REVEAL_PAGES) + + +def chain(): + graph = SimpleNamespace(block=100, hotkeys=["miner", "val1", "val2", "nonval"], + I=[.001, 0, 0, 0], S=[0, 20, 10, 40], validator_permit=[False, True, True, False], + last_update=[0, 90, 95, 96]) + s = Mock() + s.metagraph.return_value = graph + s.get_timestamp.side_effect = lambda block: datetime.fromtimestamp(block * 12, timezone.utc) + s.commit_reveal_enabled.return_value = True + s.weights.return_value = [(1, [(0, 10), (2, 90)]), (2, [(1, 100)]), (3, [(0, 100)])] + return s + + +def test_snapshot_is_block_consistent_and_targets_miner_not_validator_update(): + s = chain() + result = collect_snapshot(s, "miner", 34, "finney") + assert result["incentive"] == .001 + assert result["positive_weight_count"] == 1 and result["validator_count"] == 2 + assert [r["submitted_block"] for r in result["validators"]] == [90, 95] + assert [r["weight"] for r in result["validators"]] == [.1, 0] + s.weights.assert_called_once_with(34, block=100) + s.commit_reveal_enabled.assert_called_once_with(34, block=100) + assert result["commit_reveal_enabled"] is True + assert result["status"] == "ok" + + +def test_missing_hotkey_is_not_zero_incentive(): + s = chain() + result = collect_snapshot(s, "not-registered", 34, "finney") + assert result["status"] == "not_registered" and result["incentive"] is None + s.weights.assert_not_called() + + +def test_rpc_failure_preserves_incentive_without_fake_zero_weights(): + s = chain() + s.weights.side_effect = RuntimeError("unavailable") + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "partial" and result["incentive"] == .001 + assert "positive_weight_count" not in result + + +def test_unknown_timestamp_has_no_estimated_event(): + s = chain() + s.get_timestamp.side_effect = [datetime.now(timezone.utc), RuntimeError("pruned")] + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "partial" + assert all(v["submitted_at"] is None for v in result["validators"]) + + +def test_never_submitted_or_future_block_is_not_plotted(): + s = chain() + s.metagraph.return_value.last_update = [0, 0, 101, 0] + result = collect_snapshot(s, "miner", 34, "finney") + assert all(v["submitted_block"] is None for v in result["validators"]) + assert s.get_timestamp.call_count == 1 + + +def test_limits_timestamp_queries_and_orders_validators_by_stake(): + s = chain() + g = s.metagraph.return_value + g.hotkeys = ["miner"] + [f"val{i}" for i in range(10)] + g.I = [0] * 11 + g.S = list(range(11)) + g.validator_permit = [False] + [True] * 10 + g.last_update = [0] + [90] * 10 + result = collect_snapshot(s, "miner", 34, "finney") + assert [v["uid"] for v in result["validators"]] == [10, 9, 8, 7, 6] + assert result["validator_count"] == 10 + assert s.get_timestamp.call_count == 2 # shared timestamps fetched once + + +def test_stale_validators_do_not_trigger_archive_timestamp_lookups(): + s = chain() + s.metagraph.return_value.block = 10000 + s.metagraph.return_value.last_update = [0, 9990, 50, 0] + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "ok" + assert result["validators"][1]["submitted_block"] == 50 + assert result["validators"][1]["submitted_at"] is None + assert result["validators"][1]["timing_note"] == ">7200 blocks ago" + assert s.get_timestamp.call_count == 2 + + +def test_publishes_partial_progress_before_optional_queries(): + snapshots = [] + collect_snapshot(chain(), "miner", 34, "finney", lambda x: snapshots.append(copy.deepcopy(x))) + assert snapshots[0]["incentive"] == .001 + assert snapshots[0]["validators"] == [] + assert snapshots[-1]["validators"][0]["submitted_at"] + + +def test_subprocess_is_bounded_and_stdout_is_isolated(monkeypatch): + runner = Mock(return_value=SimpleNamespace(stdout='SDK log\n'+PREFIX+'{"status":"ok","incentive":0}\n', returncode=0)) + monkeypatch.setattr(subprocess, "run", runner) + result = fetch_generator_chain("miner", network="test", netuid=379, timeout=12) + assert result["incentive"] == 0 and result["status"] == "ok" + assert runner.call_args.kwargs["timeout"] == 12 + assert runner.call_args.args[0][-3:] == ["miner", "test", "379"] + + +@pytest.mark.parametrize("output", [None, b"no result"]) +def test_timeout_is_unknown_not_zero(monkeypatch, output): + runner = Mock(side_effect=subprocess.TimeoutExpired("chain", 1, output=output)) + monkeypatch.setattr(subprocess, "run", runner) + result = fetch_generator_chain("miner") + assert result["incentive"] is None and result["status"] == "unavailable" + + +def test_timeout_retains_completed_snapshot_fields(monkeypatch): + output = (PREFIX + json.dumps(dict(status="partial", incentive=.1, validators=[]))).encode() + monkeypatch.setattr(subprocess, "run", Mock(side_effect=subprocess.TimeoutExpired("chain", 1, output=output))) + result = fetch_generator_chain("miner") + assert result["status"] == "partial" and result["incentive"] == .1 + + +@pytest.mark.parametrize("output", ["", PREFIX + "not json"]) +def test_broken_worker_does_not_crash_cli(monkeypatch, output): + monkeypatch.setattr(subprocess, "run", Mock(return_value=SimpleNamespace(stdout=output, returncode=1))) + assert fetch_generator_chain("miner")["incentive"] is None + + +def indexed(block=92, netuid=34, hotkey="val1"): + return dict(block_number=block, pallet="SubtensorModule", method="TimelockedWeightsRevealed", + args=[[netuid], hotkey], observed_at=0) + + +def page(events=(), cursor=None, degraded=False): + return Mock(headers={"x-metagraph-degraded": "tier_unavailable"} if degraded else {}, + json=Mock(return_value=dict(ok=True, data=dict(events=list(events), next_cursor=cursor)))) + + +def revealed(hotkey="val1", netuid=34): + return dict(event=dict(module_id="SubtensorModule", event_id="TimelockedWeightsRevealed", + attributes=(netuid, hotkey))) + + +def test_reveal_locator_is_verified_on_chain_and_never_uses_index_timestamp(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(return_value=page([indexed(), indexed(hotkey="val2")])) + s.get_block_hash.return_value = "block-hash" + s.substrate.get_events.return_value = [revealed(), revealed("val2"), revealed(netuid=99)] + add_recent_reveals(s, snapshot, get=get) + first, second = snapshot["validators"] + assert first["revealed_block"] == second["revealed_block"] == 92 + assert first["revealed_at"] == datetime.fromtimestamp(92 * 12, timezone.utc).isoformat() + # Latest val2 commit is newer than its reveal: don't invent a pairing. + assert second["submitted_block"] == 95 + s.substrate.get_events.assert_called_once_with(block_hash="block-hash") + assert "miner" not in str(get.call_args) + + +@pytest.mark.parametrize("events", [[], [revealed(netuid=99)], [revealed("unrelated")]]) +def test_index_cannot_fabricate_a_reveal(events): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + s.substrate.get_events.return_value = events + add_recent_reveals(s, snapshot, get=Mock(return_value=page([indexed()]))) + assert all(not r.get("revealed_at") for r in snapshot["validators"]) + + +def test_reveal_search_skips_future_stale_and_wrong_subnet_events(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + events = [indexed(101), indexed(-8000), indexed(netuid=99), indexed(hotkey="unrelated")] + add_recent_reveals(s, snapshot, get=Mock(return_value=page(events))) + s.substrate.get_events.assert_not_called() + + +def test_reveal_search_has_a_page_limit_and_uses_opaque_cursors(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(side_effect=[page(cursor=f"cursor-{i}") for i in range(MAX_REVEAL_PAGES)]) + add_recent_reveals(s, snapshot, get=get) + assert get.call_count == MAX_REVEAL_PAGES + assert get.call_args_list[1].kwargs["params"]["cursor"] == "cursor-0" + assert get.call_args.kwargs["timeout"] == 3 + + +def test_reveal_search_stops_repeated_cursor(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(return_value=page(cursor="same")) + add_recent_reveals(s, snapshot, get=get) + assert get.call_count == 2 + + +@pytest.mark.parametrize("response", [page(degraded=True), RuntimeError("offline")]) +def test_reveal_index_failure_is_unknown_and_preserves_chain_snapshot(response): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(side_effect=response) if isinstance(response, Exception) else Mock(return_value=response) + add_recent_reveals(s, snapshot, get=get) + assert snapshot["incentive"] == .001 + assert snapshot["validators"][0]["submitted_block"] == 90 + assert "incomplete/unavailable" in snapshot["reveal_note"] + + +@pytest.mark.parametrize("network, enabled", [("test", True), ("wss://custom", True), ("finney", False)]) +def test_index_not_used_for_other_networks_or_non_commit_reveal(network, enabled): + s = chain() + s.commit_reveal_enabled.return_value = enabled + snapshot = collect_snapshot(s, "miner", 34, network) + get = Mock() + add_recent_reveals(s, snapshot, get=get) + get.assert_not_called() + + +def test_partial_reveal_results_survive_later_index_failure(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + s.substrate.get_events.return_value = [revealed()] + get = Mock(side_effect=[page([indexed()], cursor="next"), RuntimeError("offline")]) + checkpoints = [] + add_recent_reveals(s, snapshot, lambda x: checkpoints.append(copy.deepcopy(x)), get=get) + assert snapshot["validators"][0]["revealed_block"] == 92 + assert checkpoints[-1]["validators"][0]["revealed_block"] == 92 + assert not snapshot["validators"][1].get("revealed_block") diff --git a/tests/test_generator_performance_cli.py b/tests/test_generator_performance_cli.py new file mode 100644 index 00000000..37536659 --- /dev/null +++ b/tests/test_generator_performance_cli.py @@ -0,0 +1,293 @@ +import json +from datetime import datetime, timedelta, timezone +from io import StringIO +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from gas.cli import cli +from rich.console import Console +from gas.utils.generator_performance import line_chart, render_fool_history, _segments + + +def payload(): + current = dict(fool_rate=.0272, total_samples=257, fooled_count=7, + not_fooled_count=250, benchmark_run_count=20) + empty = dict(fool_rate=None, total_samples=0, fooled_count=0, + not_fooled_count=0, benchmark_run_count=0) + at = "2026-09-28T16:00:00+00:00" + return dict(as_of=at, window_days=7, verification={}, fool_aggregate=current, + history={ + "image": dict(current=current, threshold=.02, minimum_samples=20, + status="qualified", last_evaluated_at=at, + points=[dict(at=at, **current)] * 29), + "video": dict(current=empty, threshold=.01, minimum_samples=20, + status="no_data", last_evaluated_at=None, + points=[dict(at=at, **empty)] * 29), + }) + + +@pytest.fixture +def run(monkeypatch): + import bittensor + from gas.protocol import miner_requests + monkeypatch.setattr("gas.cli.load_miner_env", lambda: None) + monkeypatch.setattr(bittensor, "Wallet", lambda **kwargs: SimpleNamespace( + hotkey=SimpleNamespace(ss58_address="test-hotkey"))) + fetch = Mock(return_value=dict(success=True, data=payload())) + monkeypatch.setattr(miner_requests, "fetch_generator_performance", fetch) + chain = Mock(return_value={"status": "disabled"}) + monkeypatch.setattr("gas.protocol.generator_chain.fetch_generator_chain", chain) + fetch.chain = chain + return lambda *args: CliRunner().invoke(cli, ["g", "perf", *args]), fetch + + +def test_default_charts_and_eligibility_caveat(run): + invoke, fetch = run + result = invoke() + assert result.exit_code == 0, result.output + assert "IMAGE" in result.output and "2.72%" in result.output + assert "Fool-rate eligible" in result.output + assert "VIDEO" in result.output and "—" in result.output + assert "No recent data" in result.output + assert "●" in result.output and "threshold" in result.output + assert "On-chain incentive is not checked" in result.output + assert fetch.call_args.kwargs["lookback_days"] == 7 + + +def test_json_is_pure_json_with_new_api(run): + invoke, fetch = run + result = invoke("--json") + assert result.exit_code == 0 + assert json.loads(result.output) == payload() + assert "⛽" not in result.output + fetch.chain.assert_not_called() + + +def test_old_api_keeps_aggregate_and_json(run): + invoke, fetch = run + old = dict(verification={}, fool_aggregate=payload()["fool_aggregate"]) + fetch.return_value = dict(success=True, data=old) + result = invoke() + assert result.exit_code == 0 + assert "History unavailable" in result.output + assert "samples=257" in result.output + assert json.loads(invoke("--json").output) == old + + +def test_modality_filter_and_custom_window(run): + invoke, fetch = run + result = invoke("--modality", "image", "--lookback-days", "3") + assert result.exit_code == 0 + assert "VIDEO" not in result.output + assert "rolling 7-day" in result.output + assert "last 3d" in result.output + + +@pytest.mark.parametrize("days", ["0", "91", "-1"]) +def test_invalid_window_never_calls_api(run, days): + invoke, fetch = run + assert invoke("--lookback-days", days).exit_code == 2 + fetch.assert_not_called() + + +def test_api_failure_has_no_fake_chart(run): + invoke, fetch = run + fetch.return_value = dict(success=False, error="unavailable") + result = invoke() + assert result.exit_code == 1 + assert "unavailable" in result.output + assert "eligible" not in result.output + + +def points(rates): + now = datetime(2026, 9, 28, 16, tzinfo=timezone.utc) + return [dict(at=(now-timedelta(days=len(rates)-i-1)).isoformat(), fool_rate=rate) + for i, rate in enumerate(rates)] + + +def test_missing_data_is_not_zero_and_splits_lines(): + sample = points([0, None, .01]) + assert line_chart(points([None] * 3), .02) is None + segments = list(_segments(sample)) + assert len(segments) == 2 + assert segments[0][0][1] == 0 + assert segments[1][0][1] == 1 + + +def test_plot_calls_do_not_bridge_missing_samples(monkeypatch): + from gas.utils import generator_performance as renderer + plot = Mock(wraps=renderer.plt.plot) + monkeypatch.setattr(renderer.plt, "plot", plot) + line_chart(points([.01, .02, None, .03, .04]), .02) + assert plot.call_count == 2 + assert plot.call_args_list[0].args[1] == [1, 2] + assert plot.call_args_list[1].args[1] == [3, 4] + assert all(call.kwargs["marker"] == "braille" for call in plot.call_args_list) + + +@pytest.mark.parametrize("width", [28, 68, 108, 148]) +def test_plot_fills_requested_width(width): + chart = line_chart(points([.01, .03]), .02, width=width) + lines = chart.plain.splitlines() + assert max(len(line) for line in lines) == width + assert all(len(line) <= width for line in lines) + assert "\x1b" not in chart.plain + + +@pytest.mark.parametrize("width", [20, 40, 80, 120]) +def test_dashboard_fits_terminal(width): + output = StringIO() + render_fool_history(payload(), console=Console(file=output, width=width, color_system=None)) + lines = output.getvalue().splitlines() + assert all(len(line) <= width for line in lines) + assert any(len(line) == width for line in lines) + + +def test_short_window_has_time_ticks_and_long_window_has_dates(): + sample = points([.01, .02]) + assert "16:00" in line_chart(sample, .02).plain + sample = points([.01, .02, .03, .02, .04, .02, .03, .04]) + chart = line_chart(sample, .02, width=100).plain + assert "Sep 21" in chart and "Sep 28" in chart + + +def test_repeated_render_has_no_state_leakage(): + first = line_chart(points([.01, .03]), .02).plain + line_chart(points([.5, .9]), .01, modality="video") + assert line_chart(points([.01, .03]), .02).plain == first + assert line_chart(points([.05]), .02) is not None + + +def test_redirected_cli_has_no_ansi(run): + result = run[0]() + assert result.exit_code == 0 + assert "\x1b" not in result.output + + +@pytest.mark.parametrize("status,label", [ + ("below_threshold", "Below threshold"), + ("insufficient_samples", "Insufficient samples"), + ("not_applicable", "No generator reward gate"), +]) +def test_status_labels(run, status, label): + invoke, fetch = run + fetch.return_value["data"]["history"]["image"]["status"] = status + assert label in invoke().output + + +def test_verification_details_are_preserved(run): + invoke, fetch = run + fetch.return_value["data"]["verification"] = dict( + validator_count=1, aggregate_pass_rate=.9, total_verified=9, + total_failed=1, total_evaluated=10, by_validator=[dict( + validator_hotkey="validator-123456", pass_rate=.9, + total_verified=9, total_failed=1, lookback_hours=24)]) + output = invoke().output + assert "VERIFICATION" in output and "90.0%" in output + assert "validator-12" in output and "24h" in output + + +def test_external_text_is_not_rich_markup(): + output = StringIO() + render_fool_history(payload(), hotkey="[red]not markup[/red]", + console=Console(file=output, width=100, color_system=None)) + assert "[red]not markup[/red]" in output.getvalue() + + +def test_chain_options_and_skip(run): + invoke, fetch = run + assert invoke("--netuid", "379", "--chain-endpoint", "test", "--chain-timeout", "10").exit_code == 0 + fetch.chain.assert_called_once_with("test-hotkey", network="test", netuid=379, timeout=10) + fetch.chain.reset_mock() + assert invoke("--no-chain").exit_code == 0 + fetch.chain.assert_not_called() + + +def test_chain_environment_defaults(run, monkeypatch): + invoke, fetch = run + monkeypatch.setenv("BT_NETUID", "379") + monkeypatch.setenv("BT_CHAIN_ENDPOINT", "test") + assert invoke().exit_code == 0 + assert fetch.chain.call_args.kwargs["netuid"] == 379 + assert fetch.chain.call_args.kwargs["network"] == "test" + + +def test_chain_failure_does_not_hide_charts(run): + invoke, fetch = run + fetch.chain.return_value = dict(status="unavailable", incentive=None, warning="Chain lookup timed out") + output = invoke().output + assert "Incentive — unavailable" in output + assert "2.72%" in output and "Chain lookup timed out" in output + + +def test_positive_incentive_is_separate_from_weight_and_submission(run): + invoke, fetch = run + fetch.chain.return_value = dict(status="ok", incentive=.000123, uid=239, netuid=34, + block=12345, as_of="2026-09-28T16:00:00Z", commit_reveal_enabled=True, + positive_weight_count=1, validator_count=2, validators=[dict(uid=135, + weight=.001, submitted_block=12300, submitted_at="2026-09-28T15:00:00Z")]) + output = invoke().output + assert "0.000123" in output and "0.0123%" in output + assert "Revealed weight" in output and "0.1000%" in output + assert "Submission ≠ reveal" in output + assert "showing 1/2 validators" in output + assert "Reveal unknown" in output + + +def test_submission_markers_are_deduplicated_and_clipped(): + sample = points([.01, .02, .03]) + markers = [dict(submitted_at=sample[1]["at"])] * 2 + [ + dict(submitted_at="2026-09-01T00:00:00Z"), dict(submitted_at=None)] + chart = line_chart(sample, .02, submissions=markers) + track = next(line for line in chart.plain.splitlines() if line.lstrip().startswith("S ")) + assert track.count("│") == 1 + + +def test_reveal_markers_are_distinct_deduplicated_and_clipped(): + sample = points([.01, .02, .03]) + markers = [dict(submitted_at=sample[1]["at"], revealed_at=sample[2]["at"])] * 2 + [ + dict(revealed_at="2026-09-01T00:00:00Z")] + chart = line_chart(sample, .02, submissions=markers) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert all(line.count("│") == 1 for line in tracks) + assert tracks[0].index("│") < tracks[1].index("│") + + +@pytest.mark.parametrize("width", [40, 80, 94, 160]) +def test_colliding_demo_markers_keep_both_colors_and_aligned_tracks(width): + from scripts.preview_generator_performance import demo_data, demo_chain + chart = line_chart(demo_data()["history"]["image"]["points"], .02, + width=width, submissions=demo_chain()["validators"]) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert all("│" in line for line in tracks) + styles = {str(span.style) for span in chart.spans} + assert {"#fbbf24", "#60a5fa"} <= styles + assert all(len(line) <= width for line in tracks) + + +def test_coincident_events_stay_at_the_same_time_in_separate_tracks(): + sample = points([.01, .02, .03]) + chart = line_chart(sample, .02, submissions=[dict(submitted_at=sample[1]["at"], revealed_at=sample[1]["at"])]) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert tracks[0].index("│") == tracks[1].index("│") + + +def test_demo_shows_recent_commit_and_reveal_without_pairing(): + import io + from rich.console import Console + from scripts.preview_generator_performance import demo_chain + from gas.utils.generator_performance import render_chain_context + out = io.StringIO() + render_chain_context(demo_chain(), Console(file=out, width=100, no_color=True)) + text = out.getvalue() + assert "showing 5/7 validators" in text + assert "Commit Sep 28 15:12 UTC" in text + assert "Reveal Sep 28 14:48 UTC" in text + assert "different commits" in text + assert "unknown ≠ not revealed" in text diff --git a/uv.lock b/uv.lock index 16188e76..acf9e528 100644 --- a/uv.lock +++ b/uv.lock @@ -1207,9 +1207,11 @@ dependencies = [ { name = "pandas" }, { name = "peft" }, { name = "pillow" }, + { name = "plotext" }, { name = "pyarrow" }, { name = "python-dotenv" }, { name = "qwen-vl-utils" }, + { name = "rich" }, { name = "runwayml" }, { name = "starlette" }, { name = "torch" }, @@ -1258,9 +1260,11 @@ requires-dist = [ { name = "pandas", specifier = "==2.3.0" }, { name = "peft", specifier = "==0.17.0" }, { name = "pillow", specifier = "==12.3.0" }, + { name = "plotext", specifier = "==5.3.2" }, { name = "pyarrow", specifier = "==23.0.1" }, { name = "python-dotenv", specifier = "==1.2.2" }, { name = "qwen-vl-utils", specifier = ">=0.0.14" }, + { name = "rich", specifier = "==15.0.0" }, { name = "runwayml", specifier = ">=4.13.0" }, { name = "starlette", specifier = "==1.6.0" }, { name = "torch", specifier = "==2.7.1" }, @@ -2318,6 +2322,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] +[[package]] +name = "plotext" +version = "5.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/d7/f75f397af966fe252d0d34ffd3cae765317fce2134f925f95e7d6725d1ce/plotext-5.3.2.tar.gz", hash = "sha256:52d1e932e67c177bf357a3f0fe6ce14d1a96f7f7d5679d7b455b929df517068e", size = 61967, upload-time = "2024-09-24T15:13:37.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/1e/12fe7c40cd2099a1f454518754ed229b01beaf3bbb343127f0cc13ce6c22/plotext-5.3.2-py3-none-any.whl", hash = "sha256:394362349c1ddbf319548cfac17ca65e6d5dfc03200c40dfdc0503b3e95a2283", size = 64047, upload-time = "2024-09-24T15:13:36.296Z" }, +] + [[package]] name = "plotille" version = "6.0.5"