diff --git a/VERSION b/VERSION index 51e67ba4..b07ada58 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.8 +5.0.10 diff --git a/docs/Generative-Mining.md b/docs/Generative-Mining.md index bfc2f8bb..1f543c8d 100644 --- a/docs/Generative-Mining.md +++ b/docs/Generative-Mining.md @@ -212,6 +212,60 @@ gascli generator restart gascli generator delete ``` +### Checking Your Performance + +```bash +gascli g perf # Image/video charts and verification stats +gascli g perf --modality image # Image only +gascli g perf --lookback-days 14 # Two weeks of chart history +gascli g perf --json # Raw API response, without terminal formatting +gascli g perf --no-chain # Skip chain RPC; keep benchmark charts +gascli g perf --netuid 379 --chain-endpoint test # Explicit testnet chain context +``` + +Rich panels and Plotext's high-resolution curves show **sample-weighted rolling +seven-day fool rates**, with muted threshold lines and dated axes. Each modality +needs at least 20 benchmark evaluations and a +rate strictly above its threshold: 2% for images, 1% for videos. Changing +`--lookback-days` changes the chart span (and the legacy aggregate window), not +the seven-day qualification window. + +Counts are evaluations, not unique media: the same media can be evaluated in +multiple benchmark runs. Missing data is shown as a gap, not a zero. Historical +points are reconstructed from currently available benchmark results; they are +not a record of what validators knew or paid at that time. + +Fool-rate eligibility is **not a guarantee of on-chain incentive**. Verified +activity, validator updates, and weight reveal/epoch timing also affect rewards. +Not participating in video does not disqualify an image miner. The CLI does not +infer participation from missing benchmark data. + +The on-chain panel reads the selected hotkey's current normalized incentive +score (not a token amount), current revealed weights toward it, and the latest +submission block/time for the five largest permitted validators by stake. +Amber vertical chart markers match those submission timestamps. These are the +**latest submission per validator, not a complete event history**. On commit-reveal +subnets, `LastUpdate` records commits, not reveals; the currently revealed weight +may belong to an earlier commit. No pending-payment or reveal ETA is inferred. + +Chain values are read at one metagraph block, with its timestamp shown separately +from the benchmark snapshot. Timestamp lookups older than 7,200 blocks are skipped +to avoid archive-node delays; their exact submission blocks remain visible. +Missing/failed RPC data is shown as unknown, not zero. `--chain-timeout` bounds the +lookup (default 30 seconds); partial snapshots remain useful on timeout. +`--netuid` / `--chain-endpoint` override `BT_NETUID` / `BT_CHAIN_ENDPOINT`, falling +back to subnet 34 on finney. Select the correct network when using a non-production +API. `--json` remains the raw API response and does not perform chain queries. + +The API supplies the chart history. Older API deployments still work, displaying +the aggregate summary until the history addition is deployed. Existing CLI +versions continue to read the unchanged verification and aggregate fields. + +Charts use the available terminal width; very narrow terminals show the numbers +without a plot. Redirected output remains free of color escape sequences, and +`NO_COLOR` disables colors. To preview the layout without a wallet or API call, +run `python -m scripts.preview_generator_performance --width 100` (demo data). + ### API Endpoints Your miner exposes these endpoints for validators: diff --git a/gas/__init__.py b/gas/__init__.py index e009b976..b807d88c 100644 --- a/gas/__init__.py +++ b/gas/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.0.8" +__version__ = "5.0.10" version_split = __version__.split(".") __spec_version__ = ( diff --git a/gas/cache/content_manager.py b/gas/cache/content_manager.py index 049fc206..f594ab7b 100644 --- a/gas/cache/content_manager.py +++ b/gas/cache/content_manager.py @@ -722,6 +722,10 @@ def get_verification_stats_last_n_hours( Returns: Dict mapping miner hotkey to verification stats, with per-modality breakdown. + + Raises: + Exception: If reward data cannot be read or aggregated. Callers must + not interpret unavailable data as a successfully read empty window. """ try: return self.challenges.get_outcome_stats_last_n_hours( @@ -731,7 +735,7 @@ def get_verification_stats_last_n_hours( bt.logging.error(f"Error getting verification stats for last {lookback_hours}h: {e}") import traceback bt.logging.error(traceback.format_exc()) - return {} + raise def upload_batch_to_huggingface( self, diff --git a/gas/cache/db/challenge_store.py b/gas/cache/db/challenge_store.py index 9978983e..7453fb54 100644 --- a/gas/cache/db/challenge_store.py +++ b/gas/cache/db/challenge_store.py @@ -153,6 +153,8 @@ def get_outcomes_last_n_hours( Reward scoring must not let other miners' newer outcomes displace eligible work within the requested time window. + + Read failures propagate: an unavailable window is not an empty window. """ try: cutoff = time.time() - (lookback_hours * 3600) @@ -197,7 +199,7 @@ def get_outcomes_last_n_hours( ] except Exception as e: bt.logging.error(f"Error getting recent challenge outcomes: {e}") - return [] + raise def get_outcome_stats_last_n_hours( self, lookback_hours: float = 2.0, limit: Optional[int] = None diff --git a/gas/cli.py b/gas/cli.py index 33af9fa7..584ce12c 100644 --- a/gas/cli.py +++ b/gas/cli.py @@ -896,10 +896,14 @@ def info(): help="Bittensor hotkey name (default: BT_WALLET_HOTKEY from .env.gen_miner, else 'default')", ) @click.option("--modality", type=click.Choice(["image", "video", "audio"]), default=None, help="Filter fool-rate aggregate by modality") -@click.option("--lookback-days", default=7, type=int, show_default=True, help="Fool-rate parquet lookback (days)") +@click.option("--lookback-days", default=7, type=click.IntRange(1, 90), show_default=True, help="Chart/aggregate lookback; qualification always uses 7 days") @click.option("--api-url", default=None, help="GAS API base URL (default: GAS_API_URL env or production)") +@click.option("--chain/--no-chain", default=True, help="Show current incentive and latest validator submissions (text output only)") +@click.option("--netuid", type=click.IntRange(0, 65535), default=None, help="Chain subnet (default: BT_NETUID or 34)") +@click.option("--chain-endpoint", default=None, help="Chain network/endpoint (default: BT_CHAIN_ENDPOINT or finney)") +@click.option("--chain-timeout", type=click.IntRange(1, 120), default=30, show_default=True, help="Maximum seconds for the optional chain lookup") @click.option("--json", "as_json", is_flag=True, help="Print raw API JSON") -def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_json): +def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, chain, netuid, chain_endpoint, chain_timeout, as_json): """Query your generator verification stats + aggregated fool rate (Epistula-authenticated GAS API). The request is signed with your hotkey (same Epistula flow as discriminator ``d perf``). @@ -920,10 +924,6 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js sys.exit(1) addr = wallet.hotkey.ss58_address - click.echo() - click.echo(f" ⛽ generator {addr}") - click.echo() - result = fetch_generator_performance( wallet, modality=modality, @@ -940,6 +940,24 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js click.echo(json_lib.dumps(data, indent=2, default=str)) return + chain_data = {"status": "disabled"} + if chain: + from gas.protocol.generator_chain import fetch_generator_chain + selected_netuid = netuid if netuid is not None else click.IntRange(0, 65535).convert( + os.environ.get("BT_NETUID", "34"), None, click.get_current_context()) + chain_data = fetch_generator_chain( + addr, network=chain_endpoint or os.environ.get("BT_CHAIN_ENDPOINT") or "finney", + netuid=selected_netuid, timeout=chain_timeout) + + from gas.utils.generator_performance import render_fool_history, render_chain_context + if render_fool_history(data, modality, hotkey=addr, lookback_days=lookback_days, chain=chain_data): + return + render_chain_context(chain_data) + click.echo() + click.echo(f" ⛽ generator {addr}") + click.echo(" History unavailable from this API; showing aggregate only.") + click.echo() + ver = data.get("verification") or {} fool = data.get("fool_aggregate") or {} @@ -967,7 +985,7 @@ def gen_perf(wallet_name, wallet_hotkey, modality, lookback_days, api_url, as_js click.echo(" (no verification rows yet)") click.echo() - click.echo(f" Fool rate aggregate (benchmark parquets, last {lookback_days}d)") + click.echo(f" Fool rate aggregate (benchmark evaluations, last {lookback_days}d)") ts = fool.get("total_samples", 0) if ts: click.echo( diff --git a/gas/evaluation/generator_scores.py b/gas/evaluation/generator_scores.py index 17854e4f..5405d215 100644 --- a/gas/evaluation/generator_scores.py +++ b/gas/evaluation/generator_scores.py @@ -10,11 +10,24 @@ class GeneratorScoreState: - """Persist modality EMAs and their hotkey qualification fallback together.""" + """Persist modality EMAs, qualification, and the last payable hotkey scores.""" def __init__(self): self.by_hotkey: Dict[str, Dict[str, float]] = {} self.qualification: Dict[str, GeneratorQualification] = {} + # None means no safe fallback exists; {} means a successful empty payout. + # Unlike the EMA history, this contains only miners eligible for payout. + self.last_payout: Optional[Dict[str, float]] = None + + def resolve_last_payout(self, hotkeys: Sequence[str]) -> Optional[Dict[int, float]]: + """Resolve the last good payout against current identities, never old UIDs.""" + if self.last_payout is None: + return None + return { + uid: self.last_payout[hotkey] + for uid, hotkey in enumerate(hotkeys) + if hotkey in self.last_payout + } def update( self, @@ -62,12 +75,14 @@ def save_state(self, save_dir: str, filename: str) -> None: "version": 1, "by_hotkey": self.by_hotkey, "qualification": {hotkey: asdict(q) for hotkey, q in self.qualification.items()}, + "last_payout": self.last_payout, }, stream, allow_nan=False) def load_state(self, save_dir: str, filename: str) -> bool: # Missing/invalid new-format state must never reuse legacy scalar EMA. self.by_hotkey = {} self.qualification = {} + self.last_payout = None path = Path(save_dir) / filename if not path.exists(): return False @@ -104,8 +119,19 @@ def load_state(self, save_dir: str, filename: str) -> bool: if not 0 <= fooled <= n or (qualified and n == 0): return False qualification[hotkey] = GeneratorQualification(**row) + # Older snapshots cannot establish payout eligibility from EMA alone. + payout = payload.get("last_payout") + if payout is not None: + if not isinstance(payout, dict): + return False + for hotkey, score in payout.items(): + if (not isinstance(hotkey, str) or not hotkey + or type(score) not in (int, float) + or not math.isfinite(score) or score <= 0): + return False self.by_hotkey = restored self.qualification = qualification + self.last_payout = payout return True except (OSError, ValueError, TypeError, KeyError, AttributeError): return False diff --git a/gas/protocol/generator_chain.py b/gas/protocol/generator_chain.py new file mode 100644 index 00000000..a9e5bb34 --- /dev/null +++ b/gas/protocol/generator_chain.py @@ -0,0 +1,208 @@ +"""Bounded, read-only chain context for generator performance. + +LastUpdate tracks submissions (commits on commit-reveal subnets), not reveals. +No wallet is opened and no extrinsics are submitted. RPC work runs in a child +process so a stalled endpoint cannot hang the CLI or contaminate JSON output. +""" + +import json +import subprocess +import sys + +PREFIX = "GAS_CHAIN_SNAPSHOT " +MAX_VALIDATORS = 5 +# Avoid archive fallbacks for stale validators. Their exact block is still shown. +MAX_TIMESTAMP_AGE_BLOCKS = 7200 +REVEAL_INDEX_URL = "https://api.metagraph.sh/api/v1/chain-events" +MAX_REVEAL_PAGES = 8 + + +def add_recent_reveals(subtensor, snapshot, publish=lambda snapshot: None, get=None): + """Use a bounded public index as a locator; verify events/times on chain. + + This is latest *observed* reveal context, not a complete history or proof + that a particular commit has revealed. No wallet/miner address is sent to + the index. Only Finney's modern timelock events are indexed here. + """ + if not snapshot.get("commit_reveal_enabled") or not snapshot.get("validators"): + return + if snapshot["network"] != "finney": + snapshot["reveal_note"] = "Recent reveal lookup is available on Finney only." + return + import requests + get = get or requests.get + snapshot["reveal_note"] = "Recent indexed reveals only (Metagraphed), verified on chain; unknown ≠ not revealed." + publish(snapshot) + rows = {row["hotkey"]: row for row in snapshot["validators"]} + active = {key for key, row in rows.items() if row.get("submitted_block") and + snapshot["block"] - row["submitted_block"] <= MAX_TIMESTAMP_AGE_BLOCKS} + if not active: + return + params = dict(pallet="SubtensorModule", method="TimelockedWeightsRevealed", limit=100) + checked = set() + cursors = set() + try: + for _ in range(MAX_REVEAL_PAGES): + response = get(REVEAL_INDEX_URL, params=dict(params), timeout=3) + response.raise_for_status() + body = response.json() + if (response.headers.get("x-metagraph-degraded") or not body.get("ok") or + body.get("meta", {}).get("source") == "data-worker-unavailable"): + raise ValueError("Reveal index unavailable") + data = body["data"] + for event in data["events"]: + block = int(event["block_number"]) + if not 0 <= snapshot["block"] - block <= MAX_TIMESTAMP_AGE_BLOCKS: + continue + args = event.get("args") or [] + if (event.get("pallet") != "SubtensorModule" or + event.get("method") != "TimelockedWeightsRevealed" or len(args) != 2): + continue + netuid = args[0] + if isinstance(netuid, (list, tuple)) and len(netuid) == 1: + netuid = netuid[0] + if netuid != snapshot["netuid"] or args[1] not in rows or block in checked: + continue + checked.add(block) + # A locator cannot fabricate a reveal or substitute index ingestion + # time for the actual chain timestamp. Decode all reveals in this block. + events = subtensor.substrate.get_events(block_hash=subtensor.get_block_hash(block)) + verified = [] + for record in events: + raw = record.get("event", record) + values = raw.get("attributes") or [] + if (raw.get("module_id") == "SubtensorModule" and + raw.get("event_id") == "TimelockedWeightsRevealed" and + len(values) == 2 and values[0] == snapshot["netuid"] and values[1] in rows): + verified.append(values[1]) + if verified: + at = subtensor.get_timestamp(block=block).isoformat() + for key in verified: + if block > (rows[key].get("revealed_block") or 0): + rows[key].update(revealed_block=block, revealed_at=at) + publish(snapshot) + if all(rows[key].get("revealed_block") for key in active): + break + cursor = data.get("next_cursor") + if not cursor or cursor in cursors: + break + cursors.add(cursor) + params["cursor"] = cursor + except Exception: + snapshot["reveal_note"] = "Reveal lookup incomplete/unavailable; verified events retained. Unknown ≠ not revealed." + publish(snapshot) + + +def _last_snapshot(output): + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + for line in reversed((output or "").splitlines()): + if line.startswith(PREFIX): + try: + return json.loads(line[len(PREFIX):]) + except (ValueError, TypeError): + continue + return None + + +def fetch_generator_chain(hotkey, *, network="finney", netuid=34, timeout=30): + """Return unknown/partial on failure; never turn an RPC failure into zero.""" + base = dict(status="unavailable", network=network, netuid=netuid, + incentive=None, validators=[]) + try: + result = subprocess.run( + [sys.executable, "-m", __name__, hotkey, network, str(netuid)], + capture_output=True, text=True, timeout=timeout, check=False, + ) + snapshot = _last_snapshot(result.stdout) + if snapshot is not None: + return snapshot + except subprocess.TimeoutExpired as exc: + snapshot = _last_snapshot(exc.stdout) + if snapshot is not None: + snapshot.update(status="partial", warning="Chain lookup timed out; partial snapshot shown.") + return snapshot + base["warning"] = "Chain lookup timed out; incentive is unknown, not zero." + return base + except OSError: + pass + base["warning"] = "Chain lookup unavailable; incentive is unknown, not zero." + return base + + +def collect_snapshot(subtensor, hotkey, netuid, network, publish=lambda snapshot: None): + """All values are read at one metagraph block; timestamps come from chain.""" + graph = subtensor.metagraph(netuid, lite=True) + block = int(graph.block) + snapshot = dict(status="partial", network=network, netuid=netuid, block=block, + incentive=None, validators=[], as_of=None) + if hotkey not in graph.hotkeys: + snapshot.update(status="not_registered", warning="Hotkey is not registered on this subnet.") + return snapshot + uid = graph.hotkeys.index(hotkey) + snapshot.update(uid=uid, incentive=float(graph.I[uid])) + publish(snapshot) + try: + snapshot["as_of"] = subtensor.get_timestamp(block=block).isoformat() + snapshot["commit_reveal_enabled"] = bool(subtensor.commit_reveal_enabled(netuid, block=block)) + weights = dict(subtensor.weights(netuid, block=block)) + permitted = [i for i, permit in enumerate(graph.validator_permit) if bool(permit)] + snapshot["validator_count"] = len(permitted) + snapshot["positive_weight_count"] = sum( + any(int(target) == uid and int(value) > 0 for target, value in weights.get(i, [])) + for i in permitted) + chosen = sorted(permitted, key=lambda i: float(graph.S[i]), reverse=True)[:MAX_VALIDATORS] + for i in chosen: + entries = weights.get(i, []) + total = sum(int(value) for _, value in entries) + raw_weight = sum(int(value) for target, value in entries if int(target) == uid) + last = int(graph.last_update[i]) + snapshot["validators"].append(dict( + uid=i, hotkey=graph.hotkeys[i], + weight=raw_weight / total if total else 0.0, + submitted_block=last if 0 < last <= block else None, + submitted_at=None, + timing_note=(f">{MAX_TIMESTAMP_AGE_BLOCKS} blocks ago" + if last > 0 and block - last > MAX_TIMESTAMP_AGE_BLOCKS else None), + )) + publish(snapshot) + timestamps = {} + for validator in snapshot["validators"]: + last = validator["submitted_block"] + if last and block - last <= MAX_TIMESTAMP_AGE_BLOCKS: + if last not in timestamps: + timestamps[last] = subtensor.get_timestamp(block=last).isoformat() + validator["submitted_at"] = timestamps[last] + publish(snapshot) + snapshot["status"] = "ok" + except Exception: + snapshot["warning"] = "Some chain details are unavailable; missing timestamps are not estimated." + return snapshot + + +def _main(): + import bittensor as bt + hotkey, network, netuid = sys.argv[1:] + + def publish(snapshot): + # Child-process protocol, not user-facing logging. + sys.stdout.write(PREFIX + json.dumps(snapshot, allow_nan=False) + "\n") + sys.stdout.flush() + + chain = None + try: + chain = bt.Subtensor(network=network, retry_forever=False) + snapshot = collect_snapshot(chain, hotkey, int(netuid), network, publish) + publish(snapshot) + add_recent_reveals(chain, snapshot, publish) + publish(snapshot) + except Exception: + publish(dict(status="unavailable", network=network, netuid=int(netuid), + incentive=None, validators=[], warning="Chain lookup unavailable; incentive is unknown, not zero.")) + finally: + if chain is not None: + chain.close() + + +if __name__ == "__main__": + _main() diff --git a/gas/utils/generator_performance.py b/gas/utils/generator_performance.py new file mode 100644 index 00000000..636638fe --- /dev/null +++ b/gas/utils/generator_performance.py @@ -0,0 +1,277 @@ +"""Rich performance cards with Plotext's high-resolution terminal curves.""" + +import math +import os +from datetime import datetime, timezone + +import plotext as plt +from rich import box +from rich.console import Console, Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +ACCENTS = {"image": "#5eead4", "video": "#c4b5fd", "audio": "#7dd3fc"} +PLOT_COLORS = {"image": (94, 234, 212), "video": (196, 181, 253), "audio": (125, 211, 252)} +MUTED = "#94a3b8" +THRESHOLD = (148, 163, 184) +STATUS = { + "qualified": ("Fool-rate eligible", "#86efac"), + "below_threshold": ("Below threshold", "#fcd34d"), + "insufficient_samples": ("Insufficient samples", "#fcd34d"), + "no_data": ("No recent data", MUTED), + "not_applicable": ("No generator reward gate", MUTED), +} + + +def _datetime(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + + +def _date(value): + return _datetime(value).strftime("%b %d %H:%M UTC") if value else "none in available history" + + +def _segments(points): + """Never join across missing data, or reinterpret it as zero.""" + segment = [] + for point in points: + rate = point.get("fool_rate") + if rate is None: + if segment: + yield segment + segment = [] + else: + segment.append((_datetime(point["at"]).timestamp(), rate * 100)) + if segment: + yield segment + + +def line_chart(points, threshold, width=80, height=12, modality="image", submissions=()): + """Build colored text without printing ANSI codes directly to stdout.""" + segments = list(_segments(points)) + if not segments or width < 24: + return None + start, end = _datetime(points[0]["at"]), _datetime(points[-1]["at"]) + left, right = start.timestamp(), end.timestamp() + if left == right: + left, right = left - 3600, right + 3600 + max_rate = max(rate for segment in segments for _, rate in segment) + ceiling = max(1, max_rate * 1.15, (threshold or 0) * 125) + step = next(s for s in (.25, .5, 1, 2, 5, 10, 20, 25, 50) if s >= ceiling / 4) + ceiling = math.ceil(ceiling / step) * step + # Plotext uses global state. Each synchronous render starts/ends clean. + # Never call show(), which would bypass Rich's redirect/color handling. + plt.clear_figure() + try: + plt.limit_size(False, False) + plt.plot_size(width, height) + plt.theme("clear") + plt.canvas_color("default") + plt.axes_color("default") + plt.ticks_color(THRESHOLD) + plt.xaxes(True, False) + plt.yaxes(True, False) + plt.grid(False, False) + plt.xlim(left, right) + plt.ylim(0, ceiling) + ticks = [step * i for i in range(round(ceiling / step) + 1)] + plt.yticks(ticks, [f"{t:.1f}%" for t in ticks]) + count = min(8, max(2, (width - 9) // 10)) + positions = [left + (right - left) * i / (count - 1) for i in range(count)] + fmt = "%H:%M" if (end - start).total_seconds() <= 86400 else "%b %d" + plt.xticks(positions, [datetime.fromtimestamp(t, timezone.utc).strftime(fmt) for t in positions]) + if threshold is not None: + plt.horizontal_line(threshold * 100, color=(100, 116, 139)) + for segment in segments: + xs, ys = zip(*segment) + plt.plot(list(xs), list(ys), marker="braille", + color=PLOT_COLORS.get(modality, PLOT_COLORS["image"])) + chart = Text.from_ansi(plt.build().rstrip("\n")) + # Separate event tracks preserve both types even when their timestamps + # round to the same terminal column. Align to the rendered plot axis, + # whose left margin varies with the percentage tick labels. + axis = next((line for line in chart.plain.splitlines() if "└" in line), None) + if axis is not None: + first = axis.index("└") + 1 + last = len(axis.rstrip()) - 1 + for field, label, color in (("submitted_at", "S", "#fbbf24"), + ("revealed_at", "R", "#60a5fa")): + columns = set() + for row in submissions: + if row.get(field): + timestamp = _datetime(row[field]).timestamp() + if left <= timestamp <= right: + columns.add(round((timestamp - left) / (right - left) * (last - first))) + if columns: + track = "".join("│" if i in columns else " " for i in range(last - first + 1)) + chart.append("\n" + (label + " ").rjust(first) + track, style=color) + return chart + finally: + plt.clear_figure() + + +def _modality_panel(lane, entry, width, submissions=()): + accent = ACCENTS.get(lane, ACCENTS["image"]) + current = entry["current"] + rate = current.get("fool_rate") + label, status_color = STATUS.get(entry.get("status"), ("Unknown", MUTED)) + heading = Table.grid(expand=True) + heading.add_column(ratio=1) + heading.add_column(justify="right") + heading.add_row(Text(f"{rate:.2%}" if rate is not None else "—", style=f"bold {accent}"), + Text(f"● {label}", style=status_color)) + counts = Text(f"{current['fooled_count']:,} fooled / {current['total_samples']:,} evaluations" + f" · {current['benchmark_run_count']:,} runs", style=MUTED) + threshold = entry.get("threshold") + gate = (f"Gate >{threshold:.0%} + {entry['minimum_samples']} evaluations minimum" + if threshold is not None else "No generator reward gate for this modality") + points = entry.get("points") or [] + chart = line_chart(points, threshold, width=max(0, width - 6), modality=lane, submissions=submissions) + body = [heading, counts, Text("")] + if chart is not None: + body.extend([chart, Text("")]) + elif not any(p.get("fool_rate") is not None for p in points): + body.extend([Text("No benchmark observations in this history window.", style=MUTED), Text("")]) + else: + body.extend([Text("Widen the terminal to show the chart.", style=MUTED), Text("")]) + legend = Text("━ rolling 7-day fool rate", style=accent) + if threshold is not None: + legend.append(f" ─ {threshold:.0%} threshold", style=MUTED) + legend.append(" UTC", style=MUTED) + if chart is not None: + body.append(legend) + events = Text() + for field, label, color in (("submitted_at", "S latest submissions", "#fbbf24"), + ("revealed_at", "R recent reveals", "#60a5fa")): + if any(row.get(field) and _datetime(points[0]["at"]) <= _datetime(row[field]) <= _datetime(points[-1]["at"]) for row in submissions): + if events.plain: + events.append(" ") + events.append(label, style=color) + if events.plain: + body.extend([events, Text("Latest events only, not 7-day event history; separate tracks share the chart's time axis.", style=MUTED)]) + body.append(Text(gate, style=MUTED)) + return Panel(Group(*body), title=Text(f" {lane.upper()} ", style=f"bold {accent}"), + subtitle=Text(f"Last eval: {_date(entry.get('last_evaluated_at'))}", style=MUTED), + title_align="left", subtitle_align="right", box=box.ROUNDED, + border_style="#475569", padding=(1, 2), width=width) + + +def _verification_panel(verification): + rate = verification.get("aggregate_pass_rate") + summary = Text(f"{verification.get('validator_count', 0)} validators reporting", style=MUTED) + summary.append(f" · {rate:.1%} pass" if rate is not None else " · pass rate unavailable") + summary.append(f" · {verification.get('total_verified', 0):,} verified" + f" / {verification.get('total_failed', 0):,} failed" + f" / {verification.get('total_evaluated', 0):,} evaluated") + parts = [summary] + rows = verification.get("by_validator") or [] + if rows: + table = Table(box=None, expand=True, padding=(0, 1), header_style=MUTED) + for heading in ("Validator", "Pass", "Verified", "Failed", "Window"): + table.add_column(heading, overflow="fold") + for row in rows: + pr = row.get("pass_rate") + table.add_row(Text((row.get("validator_hotkey") or "")[:12] + "…"), + f"{pr:.1%}" if pr is not None else "—", + str(row.get("total_verified", 0)), str(row.get("total_failed", 0)), + f"{row.get('lookback_hours', '?')}h") + parts.extend([Text(""), table]) + return Panel(Group(*parts), title=" VERIFICATION ", title_align="left", + border_style="#475569", box=box.ROUNDED, padding=(0, 2)) + + +def _console(): + import click + context = click.get_current_context(silent=True) + color = context.color if context is not None else None + return Console(no_color="NO_COLOR" in os.environ or color is False, force_terminal=color) + + +def render_chain_context(chain, console=None): + console = console or _console() + chain = chain or {"status": "disabled"} + if chain.get("status") == "disabled": + return + incentive = chain.get("incentive") + parts = [] + if incentive is not None: + color = "#86efac" if incentive > 0 else "#fcd34d" + parts.append(Text(f"Incentive {incentive:.6g} ({incentive:.4%}) · UID {chain['uid']}", style=f"bold {color}")) + network = chain.get("network") if chain.get("network") in ("finney", "test", "local") else "custom RPC" + at = _date(chain["as_of"]) if chain.get("as_of") else "timestamp unavailable" + parts.append(Text(f"{network} / SN{chain['netuid']} · block {chain['block']:,} · {at}", style=MUTED)) + else: + parts.append(Text("Incentive — not registered" if chain.get("status") == "not_registered" + else "Incentive — unavailable", style="#fcd34d")) + if "positive_weight_count" in chain: + parts.append(Text(f"{chain['positive_weight_count']}/{chain['validator_count']} permitted validators have positive revealed weight for you.", style=MUTED)) + validators = chain.get("validators") or [] + if validators: + count = chain.get("validator_count", len(validators)) + parts.extend([Text(""), Text(f"Most recent observed events · showing {len(validators)}/{count} validators, largest by stake", style=MUTED)]) + table = Table(box=None, expand=True, padding=(0, 1), header_style=MUTED) + for heading in ("Validator UID", "Revealed weight", "Latest event UTC", "Block"): + table.add_column(heading, overflow="fold") + for row in validators: + submitted = _date(row["submitted_at"]) if row.get("submitted_at") else row.get("timing_note") or "unknown" + label = "Commit" if chain.get("commit_reveal_enabled") else "Submit" + table.add_row(str(row["uid"]), f"{row['weight']:.4%}", + Text(f"{label} {submitted}", style="#fbbf24"), + str(row.get("submitted_block") or "—")) + if chain.get("commit_reveal_enabled"): + revealed = _date(row["revealed_at"]) if row.get("revealed_at") else "unknown" + table.add_row("", "", Text(f"Reveal {revealed}", style="#60a5fa"), + str(row.get("revealed_block") or "—")) + parts.append(table) + parts.append(Text("Weights are current row-normalized shares, not incentive. Events are validator-wide, not miner-specific.", style=MUTED)) + if chain.get("commit_reveal_enabled"): + parts.append(Text("Submission ≠ reveal: these may belong to different commits. Neither marks an incentive change.", style="#fbbf24")) + if chain.get("reveal_note"): + parts.append(Text(chain["reveal_note"], style=MUTED)) + if chain.get("warning"): + parts.append(Text(chain["warning"], style="#fcd34d")) + console.print(Panel(Group(*parts), title=" ON-CHAIN SNAPSHOT ", title_align="left", + box=box.ROUNDED, border_style="#475569", padding=(1, 2))) + console.print() + + +def render_fool_history(data, modality=None, *, hotkey=None, lookback_days=7, console=None, chain=None): + """Old APIs return False; JSON callers bypass this renderer entirely.""" + history = data.get("history") or {} + if not history: + return False + console = console or _console() + console.print() + console.print(Text.assemble(("GAS", "bold #5eead4"), (" / GENERATOR PERFORMANCE", "bold"))) + if hotkey: + console.print(Text(hotkey, style=MUTED)) + console.print(Text(f"rolling 7-day · sample-weighted · snapshot {_date(data.get('as_of'))}", style=MUTED)) + lanes = [modality] if modality else ["image", "video"] + all_points = next((history[lane].get("points") for lane in lanes + if lane in history and history[lane].get("points")), []) + if all_points: + console.print(Text(f"History: {_date(all_points[0]['at'])} → {_date(all_points[-1]['at'])}", style=MUTED)) + console.print() + render_chain_context(chain, console) + submissions = (chain or {}).get("validators") or [] + for lane in lanes: + if lane in history: + console.print(_modality_panel(lane, history[lane], console.width, submissions)) + console.print() + console.print(_verification_panel(data.get("verification") or {})) + if lookback_days != 7: + aggregate = data.get("fool_aggregate") or {} + rate = aggregate.get("fool_rate") + display = f"{rate:.2%}" if rate is not None else "—" + console.print(Text(f"Requested aggregate · last {lookback_days}d: {display}" + f" · {aggregate.get('total_samples', 0):,} evaluations", style=MUTED)) + console.print() + console.print(Text("Counts are evaluations, not unique media. Missing data is not zero.", style=MUTED)) + console.print(Text("Eligibility is per modality; no video data does not block image rewards.", style=MUTED)) + note = ("Chain snapshot shown above; fool-rate eligibility does not guarantee payment." + if (chain or {}).get("incentive") is not None else + "On-chain incentive is not checked or unavailable; verified activity and weight reveal/epoch timing also matter.") + console.print(Text(note, style=MUTED)) + console.print() + return True diff --git a/neurons/validator/validator.py b/neurons/validator/validator.py index fde8030d..21fea5c5 100644 --- a/neurons/validator/validator.py +++ b/neurons/validator/validator.py @@ -203,11 +203,27 @@ async def set_weights(self, block): """Set burn weights, or calculate rewards when full burn is disabled.""" if BURN_PERCENTAGE < 1.0: bt.logging.info(f"Updating scores at block {block}") - generator_uids = await self.update_scores() - + generator_uids = None + try: + generator_uids = await self.update_scores() + except Exception: + bt.logging.error( + "Generator scoring failed; attempting last-good payout fallback.\n" + + traceback.format_exc() + ) + if generator_uids is None: - generator_uids = [] - bt.logging.warning("No generator rewards available; using empty generator_uids") + bt.logging.warning( + "Generator scoring unavailable; keeping scores unchanged and " + "using the last-good payout snapshot." + ) + else: + # Persist eligibility with scores so restart-time outages can use it. + await self.save_state() + + if self.generator_score_state.last_payout is None: + bt.logging.error("Skipping weight submission: no last-good generator payout snapshot") + return False kings_payload = await get_current_kings( self.wallet.hotkey, base_url=self.config.benchmark_api_url @@ -261,10 +277,19 @@ def uid_for_hotkey(hotkey_ss58: str): if BURN_PERCENTAGE == 1.0: normed_weights = np.zeros(int(self.metagraph.n), dtype=np.float64) else: + # Re-resolve identities even after a successful update: UID slots + # may have changed. Do not reuse positive but ineligible EMA tails. + payout = self.generator_score_state.resolve_last_payout(self.metagraph.hotkeys) + if payout is None: + bt.logging.error("Skipping weight submission: generator payout snapshot unavailable") + return False + payout_scores = np.zeros(int(self.metagraph.n), dtype=np.float64) + for uid, score in payout.items(): + payout_scores[uid] = score normed_weights = build_koth_weights( n=int(self.metagraph.n), - scores=self.scores, - generator_uids=generator_uids, + scores=payout_scores, + generator_uids=list(payout), kings=kings, uid_for_hotkey=uid_for_hotkey, burn_uid=burn_uid, @@ -377,6 +402,11 @@ async def update_scores(self): self.scores = np.zeros(len(hotkeys), dtype=np.float64) for uid, score in scores.items(): self.scores[uid] = score + self.generator_score_state.last_payout = { + hotkeys[uid]: scores[uid] + for uid in rewards + if scores.get(uid, 0.0) > 0 + } bt.logging.info( f"Updated scores for {len(rewards)} miners with EMA (alpha={alpha})" @@ -468,8 +498,8 @@ async def load_state(self): self.generator_qualification = self.generator_score_state.qualification # scores.npy is retained for snapshot compatibility, not as EMA # input: legacy scalars cannot be split by modality or hotkey. - # Rebuild the payout vector after qualification/liveness checks - # in update_scores; a restored gate is fallback during outages. + # Fresh scoring rebuilds this vector; database outages use the + # separate last_payout snapshot, not unfiltered EMA history. self.scores = np.zeros(len(self.metagraph.hotkeys), dtype=np.float64) bt.logging.info( f"Loaded modality EMA histories for {len(self.generator_score_state.by_hotkey)} hotkeys; " diff --git a/pyproject.toml b/pyproject.toml index 6ab3d37d..00232007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,8 @@ dependencies = [ "python-dotenv==1.2.2", "accelerate==1.8.1", "click>=8.0.0", + "rich==15.0.0", + "plotext==5.3.2", "datasets==4.0.0", "async-substrate-interface==2.2.1", "ftfy==6.3.1", diff --git a/scripts/preview_generator_performance.py b/scripts/preview_generator_performance.py new file mode 100644 index 00000000..c6c626d0 --- /dev/null +++ b/scripts/preview_generator_performance.py @@ -0,0 +1,71 @@ +"""Render synthetic CLI performance data, without a wallet or API request. + +Run from the repo: python -m scripts.preview_generator_performance --width 100 +Optionally export the exact terminal render with --svg /tmp/gascli-preview.svg. +""" + +import argparse +from datetime import datetime, timedelta, timezone + +from rich.console import Console +from rich.terminal_theme import TerminalTheme + +from gas.utils.generator_performance import render_fool_history + + +def demo_data(): + now = datetime(2026, 9, 28, 16, tzinfo=timezone.utc) + history = {} + examples = [ + ("image", [.004, .006, .005, .009, .014, .012, .017, .019, .016, .021, .024, .022, 7 / 257], 7, 257, .02), + ("video", [.003, .004, .006, .005, .008, .011, .009, .012, .014, .011, .013, .014, .015], 3, 200, .01), + ] + for lane, rates, fooled, total, threshold in examples: + points = [dict(at=(now - timedelta(hours=(12-i)*14)).isoformat(), fool_rate=rate) + for i, rate in enumerate(rates)] + history[lane] = dict( + current=dict(fool_rate=fooled/total, fooled_count=fooled, total_samples=total, benchmark_run_count=20), + threshold=threshold, minimum_samples=20, status="qualified", + last_evaluated_at="2026-09-28T15:02:02Z", points=points) + return dict(as_of=now.isoformat(), history=history, verification=dict( + validator_count=3, aggregate_pass_rate=.975, total_verified=117, + total_failed=3, total_evaluated=120, by_validator=[])) + + +def demo_chain(): + return dict(status="ok", netuid=34, network="finney", block=9167400, + uid=239, incentive=.000183108, as_of="2026-09-28T16:00:00Z", + commit_reveal_enabled=True, validator_count=7, positive_weight_count=5, + reveal_note="Recent indexed reveals only (Metagraphed), verified on chain; unknown ≠ not revealed.", + validators=[ + dict(uid=135, weight=.0002966, submitted_block=9167100, + submitted_at="2026-09-28T15:00:00Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=134, weight=.0012634, submitted_block=9167090, + submitted_at="2026-09-28T14:58:00Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=19, weight=.0013733, submitted_block=9167089, + submitted_at="2026-09-28T14:57:48Z", revealed_block=9167250, + revealed_at="2026-09-28T15:30:00Z"), + dict(uid=72, weight=.0001978, submitted_block=9167160, + submitted_at="2026-09-28T15:12:00Z", revealed_block=9167040, + revealed_at="2026-09-28T14:48:00Z"), + dict(uid=5, weight=0, submitted_block=8656798, submitted_at=None, + timing_note=">7200 blocks ago"), + ]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--width", type=int, default=100) + parser.add_argument("--svg") + args = parser.parse_args() + console = Console(width=args.width, height=60, record=True, force_terminal=True, + color_system="truecolor", no_color=False) + console.print("$ gascli g perf [dim]# illustrative demo data[/dim]") + render_fool_history(demo_data(), hotkey="5GKGN7…DmWuGS · DEMO", console=console, chain=demo_chain()) + if args.svg: + theme = TerminalTheme((15, 23, 42), (226, 232, 240), + [(0, 0, 0), (255, 85, 85), (134, 239, 172), (252, 211, 77), + (125, 211, 252), (196, 181, 253), (94, 234, 212), (226, 232, 240)]) + console.save_svg(args.svg, title="gascli g perf · preview", theme=theme) diff --git a/tests/test_full_window_rewards.py b/tests/test_full_window_rewards.py index eb741d5e..d1a4f59a 100644 --- a/tests/test_full_window_rewards.py +++ b/tests/test_full_window_rewards.py @@ -1,6 +1,7 @@ """The reward lookback is time-bounded, not a global newest-N sample.""" import ast +import sqlite3 from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock @@ -120,3 +121,38 @@ def test_shorter_lookback_still_excludes_older_work(store): stats = manager_for(store).get_verification_stats_last_n_hours(lookback_hours=2) assert set(stats) == {"busy"} assert stats["busy"]["total_verified"] == 1100 + + +@pytest.mark.parametrize("message", [ + "database disk image is malformed", "disk I/O error", "database is locked", +]) +def test_unavailable_database_is_not_an_empty_reward_window(store, monkeypatch, message): + error = sqlite3.DatabaseError(message) + monkeypatch.setattr(store.db, "connect", Mock(side_effect=error)) + for query in ( + store.get_outcomes_last_n_hours, + store.get_outcome_stats_last_n_hours, + manager_for(store).get_verification_stats_last_n_hours, + ): + with pytest.raises(sqlite3.DatabaseError, match=message): + query(24) + + +def test_partial_cursor_failure_does_not_return_partial_rewards(store, monkeypatch): + def broken_cursor(): + yield dict(task_id="partial", uid=1, hotkey="busy", prompt_id="prompt", + modality="video", status="verified", failure_reason=None, + media_id=None, media_resolution=None, media_has_audio=None, + created_at=NOW, updated_at=NOW) + raise sqlite3.DatabaseError("disk I/O error") + + from unittest.mock import MagicMock + connection = MagicMock() + connection.__enter__.return_value.execute.return_value = broken_cursor() + monkeypatch.setattr(store.db, "connect", Mock(return_value=connection)) + with pytest.raises(sqlite3.DatabaseError, match="disk I/O error"): + manager_for(store).get_verification_stats_last_n_hours(24) + + +def test_successfully_read_empty_window_is_still_empty(store): + assert manager_for(store).get_verification_stats_last_n_hours(0) == {} diff --git a/tests/test_generator_chain.py b/tests/test_generator_chain.py new file mode 100644 index 00000000..e5163dd1 --- /dev/null +++ b/tests/test_generator_chain.py @@ -0,0 +1,230 @@ +import copy +import json +import subprocess +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from gas.protocol.generator_chain import (collect_snapshot, fetch_generator_chain, + add_recent_reveals, PREFIX, MAX_REVEAL_PAGES) + + +def chain(): + graph = SimpleNamespace(block=100, hotkeys=["miner", "val1", "val2", "nonval"], + I=[.001, 0, 0, 0], S=[0, 20, 10, 40], validator_permit=[False, True, True, False], + last_update=[0, 90, 95, 96]) + s = Mock() + s.metagraph.return_value = graph + s.get_timestamp.side_effect = lambda block: datetime.fromtimestamp(block * 12, timezone.utc) + s.commit_reveal_enabled.return_value = True + s.weights.return_value = [(1, [(0, 10), (2, 90)]), (2, [(1, 100)]), (3, [(0, 100)])] + return s + + +def test_snapshot_is_block_consistent_and_targets_miner_not_validator_update(): + s = chain() + result = collect_snapshot(s, "miner", 34, "finney") + assert result["incentive"] == .001 + assert result["positive_weight_count"] == 1 and result["validator_count"] == 2 + assert [r["submitted_block"] for r in result["validators"]] == [90, 95] + assert [r["weight"] for r in result["validators"]] == [.1, 0] + s.weights.assert_called_once_with(34, block=100) + s.commit_reveal_enabled.assert_called_once_with(34, block=100) + assert result["commit_reveal_enabled"] is True + assert result["status"] == "ok" + + +def test_missing_hotkey_is_not_zero_incentive(): + s = chain() + result = collect_snapshot(s, "not-registered", 34, "finney") + assert result["status"] == "not_registered" and result["incentive"] is None + s.weights.assert_not_called() + + +def test_rpc_failure_preserves_incentive_without_fake_zero_weights(): + s = chain() + s.weights.side_effect = RuntimeError("unavailable") + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "partial" and result["incentive"] == .001 + assert "positive_weight_count" not in result + + +def test_unknown_timestamp_has_no_estimated_event(): + s = chain() + s.get_timestamp.side_effect = [datetime.now(timezone.utc), RuntimeError("pruned")] + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "partial" + assert all(v["submitted_at"] is None for v in result["validators"]) + + +def test_never_submitted_or_future_block_is_not_plotted(): + s = chain() + s.metagraph.return_value.last_update = [0, 0, 101, 0] + result = collect_snapshot(s, "miner", 34, "finney") + assert all(v["submitted_block"] is None for v in result["validators"]) + assert s.get_timestamp.call_count == 1 + + +def test_limits_timestamp_queries_and_orders_validators_by_stake(): + s = chain() + g = s.metagraph.return_value + g.hotkeys = ["miner"] + [f"val{i}" for i in range(10)] + g.I = [0] * 11 + g.S = list(range(11)) + g.validator_permit = [False] + [True] * 10 + g.last_update = [0] + [90] * 10 + result = collect_snapshot(s, "miner", 34, "finney") + assert [v["uid"] for v in result["validators"]] == [10, 9, 8, 7, 6] + assert result["validator_count"] == 10 + assert s.get_timestamp.call_count == 2 # shared timestamps fetched once + + +def test_stale_validators_do_not_trigger_archive_timestamp_lookups(): + s = chain() + s.metagraph.return_value.block = 10000 + s.metagraph.return_value.last_update = [0, 9990, 50, 0] + result = collect_snapshot(s, "miner", 34, "finney") + assert result["status"] == "ok" + assert result["validators"][1]["submitted_block"] == 50 + assert result["validators"][1]["submitted_at"] is None + assert result["validators"][1]["timing_note"] == ">7200 blocks ago" + assert s.get_timestamp.call_count == 2 + + +def test_publishes_partial_progress_before_optional_queries(): + snapshots = [] + collect_snapshot(chain(), "miner", 34, "finney", lambda x: snapshots.append(copy.deepcopy(x))) + assert snapshots[0]["incentive"] == .001 + assert snapshots[0]["validators"] == [] + assert snapshots[-1]["validators"][0]["submitted_at"] + + +def test_subprocess_is_bounded_and_stdout_is_isolated(monkeypatch): + runner = Mock(return_value=SimpleNamespace(stdout='SDK log\n'+PREFIX+'{"status":"ok","incentive":0}\n', returncode=0)) + monkeypatch.setattr(subprocess, "run", runner) + result = fetch_generator_chain("miner", network="test", netuid=379, timeout=12) + assert result["incentive"] == 0 and result["status"] == "ok" + assert runner.call_args.kwargs["timeout"] == 12 + assert runner.call_args.args[0][-3:] == ["miner", "test", "379"] + + +@pytest.mark.parametrize("output", [None, b"no result"]) +def test_timeout_is_unknown_not_zero(monkeypatch, output): + runner = Mock(side_effect=subprocess.TimeoutExpired("chain", 1, output=output)) + monkeypatch.setattr(subprocess, "run", runner) + result = fetch_generator_chain("miner") + assert result["incentive"] is None and result["status"] == "unavailable" + + +def test_timeout_retains_completed_snapshot_fields(monkeypatch): + output = (PREFIX + json.dumps(dict(status="partial", incentive=.1, validators=[]))).encode() + monkeypatch.setattr(subprocess, "run", Mock(side_effect=subprocess.TimeoutExpired("chain", 1, output=output))) + result = fetch_generator_chain("miner") + assert result["status"] == "partial" and result["incentive"] == .1 + + +@pytest.mark.parametrize("output", ["", PREFIX + "not json"]) +def test_broken_worker_does_not_crash_cli(monkeypatch, output): + monkeypatch.setattr(subprocess, "run", Mock(return_value=SimpleNamespace(stdout=output, returncode=1))) + assert fetch_generator_chain("miner")["incentive"] is None + + +def indexed(block=92, netuid=34, hotkey="val1"): + return dict(block_number=block, pallet="SubtensorModule", method="TimelockedWeightsRevealed", + args=[[netuid], hotkey], observed_at=0) + + +def page(events=(), cursor=None, degraded=False): + return Mock(headers={"x-metagraph-degraded": "tier_unavailable"} if degraded else {}, + json=Mock(return_value=dict(ok=True, data=dict(events=list(events), next_cursor=cursor)))) + + +def revealed(hotkey="val1", netuid=34): + return dict(event=dict(module_id="SubtensorModule", event_id="TimelockedWeightsRevealed", + attributes=(netuid, hotkey))) + + +def test_reveal_locator_is_verified_on_chain_and_never_uses_index_timestamp(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(return_value=page([indexed(), indexed(hotkey="val2")])) + s.get_block_hash.return_value = "block-hash" + s.substrate.get_events.return_value = [revealed(), revealed("val2"), revealed(netuid=99)] + add_recent_reveals(s, snapshot, get=get) + first, second = snapshot["validators"] + assert first["revealed_block"] == second["revealed_block"] == 92 + assert first["revealed_at"] == datetime.fromtimestamp(92 * 12, timezone.utc).isoformat() + # Latest val2 commit is newer than its reveal: don't invent a pairing. + assert second["submitted_block"] == 95 + s.substrate.get_events.assert_called_once_with(block_hash="block-hash") + assert "miner" not in str(get.call_args) + + +@pytest.mark.parametrize("events", [[], [revealed(netuid=99)], [revealed("unrelated")]]) +def test_index_cannot_fabricate_a_reveal(events): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + s.substrate.get_events.return_value = events + add_recent_reveals(s, snapshot, get=Mock(return_value=page([indexed()]))) + assert all(not r.get("revealed_at") for r in snapshot["validators"]) + + +def test_reveal_search_skips_future_stale_and_wrong_subnet_events(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + events = [indexed(101), indexed(-8000), indexed(netuid=99), indexed(hotkey="unrelated")] + add_recent_reveals(s, snapshot, get=Mock(return_value=page(events))) + s.substrate.get_events.assert_not_called() + + +def test_reveal_search_has_a_page_limit_and_uses_opaque_cursors(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(side_effect=[page(cursor=f"cursor-{i}") for i in range(MAX_REVEAL_PAGES)]) + add_recent_reveals(s, snapshot, get=get) + assert get.call_count == MAX_REVEAL_PAGES + assert get.call_args_list[1].kwargs["params"]["cursor"] == "cursor-0" + assert get.call_args.kwargs["timeout"] == 3 + + +def test_reveal_search_stops_repeated_cursor(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(return_value=page(cursor="same")) + add_recent_reveals(s, snapshot, get=get) + assert get.call_count == 2 + + +@pytest.mark.parametrize("response", [page(degraded=True), RuntimeError("offline")]) +def test_reveal_index_failure_is_unknown_and_preserves_chain_snapshot(response): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + get = Mock(side_effect=response) if isinstance(response, Exception) else Mock(return_value=response) + add_recent_reveals(s, snapshot, get=get) + assert snapshot["incentive"] == .001 + assert snapshot["validators"][0]["submitted_block"] == 90 + assert "incomplete/unavailable" in snapshot["reveal_note"] + + +@pytest.mark.parametrize("network, enabled", [("test", True), ("wss://custom", True), ("finney", False)]) +def test_index_not_used_for_other_networks_or_non_commit_reveal(network, enabled): + s = chain() + s.commit_reveal_enabled.return_value = enabled + snapshot = collect_snapshot(s, "miner", 34, network) + get = Mock() + add_recent_reveals(s, snapshot, get=get) + get.assert_not_called() + + +def test_partial_reveal_results_survive_later_index_failure(): + s = chain() + snapshot = collect_snapshot(s, "miner", 34, "finney") + s.substrate.get_events.return_value = [revealed()] + get = Mock(side_effect=[page([indexed()], cursor="next"), RuntimeError("offline")]) + checkpoints = [] + add_recent_reveals(s, snapshot, lambda x: checkpoints.append(copy.deepcopy(x)), get=get) + assert snapshot["validators"][0]["revealed_block"] == 92 + assert checkpoints[-1]["validators"][0]["revealed_block"] == 92 + assert not snapshot["validators"][1].get("revealed_block") diff --git a/tests/test_generator_performance_cli.py b/tests/test_generator_performance_cli.py new file mode 100644 index 00000000..37536659 --- /dev/null +++ b/tests/test_generator_performance_cli.py @@ -0,0 +1,293 @@ +import json +from datetime import datetime, timedelta, timezone +from io import StringIO +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from gas.cli import cli +from rich.console import Console +from gas.utils.generator_performance import line_chart, render_fool_history, _segments + + +def payload(): + current = dict(fool_rate=.0272, total_samples=257, fooled_count=7, + not_fooled_count=250, benchmark_run_count=20) + empty = dict(fool_rate=None, total_samples=0, fooled_count=0, + not_fooled_count=0, benchmark_run_count=0) + at = "2026-09-28T16:00:00+00:00" + return dict(as_of=at, window_days=7, verification={}, fool_aggregate=current, + history={ + "image": dict(current=current, threshold=.02, minimum_samples=20, + status="qualified", last_evaluated_at=at, + points=[dict(at=at, **current)] * 29), + "video": dict(current=empty, threshold=.01, minimum_samples=20, + status="no_data", last_evaluated_at=None, + points=[dict(at=at, **empty)] * 29), + }) + + +@pytest.fixture +def run(monkeypatch): + import bittensor + from gas.protocol import miner_requests + monkeypatch.setattr("gas.cli.load_miner_env", lambda: None) + monkeypatch.setattr(bittensor, "Wallet", lambda **kwargs: SimpleNamespace( + hotkey=SimpleNamespace(ss58_address="test-hotkey"))) + fetch = Mock(return_value=dict(success=True, data=payload())) + monkeypatch.setattr(miner_requests, "fetch_generator_performance", fetch) + chain = Mock(return_value={"status": "disabled"}) + monkeypatch.setattr("gas.protocol.generator_chain.fetch_generator_chain", chain) + fetch.chain = chain + return lambda *args: CliRunner().invoke(cli, ["g", "perf", *args]), fetch + + +def test_default_charts_and_eligibility_caveat(run): + invoke, fetch = run + result = invoke() + assert result.exit_code == 0, result.output + assert "IMAGE" in result.output and "2.72%" in result.output + assert "Fool-rate eligible" in result.output + assert "VIDEO" in result.output and "—" in result.output + assert "No recent data" in result.output + assert "●" in result.output and "threshold" in result.output + assert "On-chain incentive is not checked" in result.output + assert fetch.call_args.kwargs["lookback_days"] == 7 + + +def test_json_is_pure_json_with_new_api(run): + invoke, fetch = run + result = invoke("--json") + assert result.exit_code == 0 + assert json.loads(result.output) == payload() + assert "⛽" not in result.output + fetch.chain.assert_not_called() + + +def test_old_api_keeps_aggregate_and_json(run): + invoke, fetch = run + old = dict(verification={}, fool_aggregate=payload()["fool_aggregate"]) + fetch.return_value = dict(success=True, data=old) + result = invoke() + assert result.exit_code == 0 + assert "History unavailable" in result.output + assert "samples=257" in result.output + assert json.loads(invoke("--json").output) == old + + +def test_modality_filter_and_custom_window(run): + invoke, fetch = run + result = invoke("--modality", "image", "--lookback-days", "3") + assert result.exit_code == 0 + assert "VIDEO" not in result.output + assert "rolling 7-day" in result.output + assert "last 3d" in result.output + + +@pytest.mark.parametrize("days", ["0", "91", "-1"]) +def test_invalid_window_never_calls_api(run, days): + invoke, fetch = run + assert invoke("--lookback-days", days).exit_code == 2 + fetch.assert_not_called() + + +def test_api_failure_has_no_fake_chart(run): + invoke, fetch = run + fetch.return_value = dict(success=False, error="unavailable") + result = invoke() + assert result.exit_code == 1 + assert "unavailable" in result.output + assert "eligible" not in result.output + + +def points(rates): + now = datetime(2026, 9, 28, 16, tzinfo=timezone.utc) + return [dict(at=(now-timedelta(days=len(rates)-i-1)).isoformat(), fool_rate=rate) + for i, rate in enumerate(rates)] + + +def test_missing_data_is_not_zero_and_splits_lines(): + sample = points([0, None, .01]) + assert line_chart(points([None] * 3), .02) is None + segments = list(_segments(sample)) + assert len(segments) == 2 + assert segments[0][0][1] == 0 + assert segments[1][0][1] == 1 + + +def test_plot_calls_do_not_bridge_missing_samples(monkeypatch): + from gas.utils import generator_performance as renderer + plot = Mock(wraps=renderer.plt.plot) + monkeypatch.setattr(renderer.plt, "plot", plot) + line_chart(points([.01, .02, None, .03, .04]), .02) + assert plot.call_count == 2 + assert plot.call_args_list[0].args[1] == [1, 2] + assert plot.call_args_list[1].args[1] == [3, 4] + assert all(call.kwargs["marker"] == "braille" for call in plot.call_args_list) + + +@pytest.mark.parametrize("width", [28, 68, 108, 148]) +def test_plot_fills_requested_width(width): + chart = line_chart(points([.01, .03]), .02, width=width) + lines = chart.plain.splitlines() + assert max(len(line) for line in lines) == width + assert all(len(line) <= width for line in lines) + assert "\x1b" not in chart.plain + + +@pytest.mark.parametrize("width", [20, 40, 80, 120]) +def test_dashboard_fits_terminal(width): + output = StringIO() + render_fool_history(payload(), console=Console(file=output, width=width, color_system=None)) + lines = output.getvalue().splitlines() + assert all(len(line) <= width for line in lines) + assert any(len(line) == width for line in lines) + + +def test_short_window_has_time_ticks_and_long_window_has_dates(): + sample = points([.01, .02]) + assert "16:00" in line_chart(sample, .02).plain + sample = points([.01, .02, .03, .02, .04, .02, .03, .04]) + chart = line_chart(sample, .02, width=100).plain + assert "Sep 21" in chart and "Sep 28" in chart + + +def test_repeated_render_has_no_state_leakage(): + first = line_chart(points([.01, .03]), .02).plain + line_chart(points([.5, .9]), .01, modality="video") + assert line_chart(points([.01, .03]), .02).plain == first + assert line_chart(points([.05]), .02) is not None + + +def test_redirected_cli_has_no_ansi(run): + result = run[0]() + assert result.exit_code == 0 + assert "\x1b" not in result.output + + +@pytest.mark.parametrize("status,label", [ + ("below_threshold", "Below threshold"), + ("insufficient_samples", "Insufficient samples"), + ("not_applicable", "No generator reward gate"), +]) +def test_status_labels(run, status, label): + invoke, fetch = run + fetch.return_value["data"]["history"]["image"]["status"] = status + assert label in invoke().output + + +def test_verification_details_are_preserved(run): + invoke, fetch = run + fetch.return_value["data"]["verification"] = dict( + validator_count=1, aggregate_pass_rate=.9, total_verified=9, + total_failed=1, total_evaluated=10, by_validator=[dict( + validator_hotkey="validator-123456", pass_rate=.9, + total_verified=9, total_failed=1, lookback_hours=24)]) + output = invoke().output + assert "VERIFICATION" in output and "90.0%" in output + assert "validator-12" in output and "24h" in output + + +def test_external_text_is_not_rich_markup(): + output = StringIO() + render_fool_history(payload(), hotkey="[red]not markup[/red]", + console=Console(file=output, width=100, color_system=None)) + assert "[red]not markup[/red]" in output.getvalue() + + +def test_chain_options_and_skip(run): + invoke, fetch = run + assert invoke("--netuid", "379", "--chain-endpoint", "test", "--chain-timeout", "10").exit_code == 0 + fetch.chain.assert_called_once_with("test-hotkey", network="test", netuid=379, timeout=10) + fetch.chain.reset_mock() + assert invoke("--no-chain").exit_code == 0 + fetch.chain.assert_not_called() + + +def test_chain_environment_defaults(run, monkeypatch): + invoke, fetch = run + monkeypatch.setenv("BT_NETUID", "379") + monkeypatch.setenv("BT_CHAIN_ENDPOINT", "test") + assert invoke().exit_code == 0 + assert fetch.chain.call_args.kwargs["netuid"] == 379 + assert fetch.chain.call_args.kwargs["network"] == "test" + + +def test_chain_failure_does_not_hide_charts(run): + invoke, fetch = run + fetch.chain.return_value = dict(status="unavailable", incentive=None, warning="Chain lookup timed out") + output = invoke().output + assert "Incentive — unavailable" in output + assert "2.72%" in output and "Chain lookup timed out" in output + + +def test_positive_incentive_is_separate_from_weight_and_submission(run): + invoke, fetch = run + fetch.chain.return_value = dict(status="ok", incentive=.000123, uid=239, netuid=34, + block=12345, as_of="2026-09-28T16:00:00Z", commit_reveal_enabled=True, + positive_weight_count=1, validator_count=2, validators=[dict(uid=135, + weight=.001, submitted_block=12300, submitted_at="2026-09-28T15:00:00Z")]) + output = invoke().output + assert "0.000123" in output and "0.0123%" in output + assert "Revealed weight" in output and "0.1000%" in output + assert "Submission ≠ reveal" in output + assert "showing 1/2 validators" in output + assert "Reveal unknown" in output + + +def test_submission_markers_are_deduplicated_and_clipped(): + sample = points([.01, .02, .03]) + markers = [dict(submitted_at=sample[1]["at"])] * 2 + [ + dict(submitted_at="2026-09-01T00:00:00Z"), dict(submitted_at=None)] + chart = line_chart(sample, .02, submissions=markers) + track = next(line for line in chart.plain.splitlines() if line.lstrip().startswith("S ")) + assert track.count("│") == 1 + + +def test_reveal_markers_are_distinct_deduplicated_and_clipped(): + sample = points([.01, .02, .03]) + markers = [dict(submitted_at=sample[1]["at"], revealed_at=sample[2]["at"])] * 2 + [ + dict(revealed_at="2026-09-01T00:00:00Z")] + chart = line_chart(sample, .02, submissions=markers) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert all(line.count("│") == 1 for line in tracks) + assert tracks[0].index("│") < tracks[1].index("│") + + +@pytest.mark.parametrize("width", [40, 80, 94, 160]) +def test_colliding_demo_markers_keep_both_colors_and_aligned_tracks(width): + from scripts.preview_generator_performance import demo_data, demo_chain + chart = line_chart(demo_data()["history"]["image"]["points"], .02, + width=width, submissions=demo_chain()["validators"]) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert all("│" in line for line in tracks) + styles = {str(span.style) for span in chart.spans} + assert {"#fbbf24", "#60a5fa"} <= styles + assert all(len(line) <= width for line in tracks) + + +def test_coincident_events_stay_at_the_same_time_in_separate_tracks(): + sample = points([.01, .02, .03]) + chart = line_chart(sample, .02, submissions=[dict(submitted_at=sample[1]["at"], revealed_at=sample[1]["at"])]) + tracks = [line for line in chart.plain.splitlines() if line.lstrip().startswith(("S ", "R "))] + assert len(tracks) == 2 + assert tracks[0].index("│") == tracks[1].index("│") + + +def test_demo_shows_recent_commit_and_reveal_without_pairing(): + import io + from rich.console import Console + from scripts.preview_generator_performance import demo_chain + from gas.utils.generator_performance import render_chain_context + out = io.StringIO() + render_chain_context(demo_chain(), Console(file=out, width=100, no_color=True)) + text = out.getvalue() + assert "showing 5/7 validators" in text + assert "Commit Sep 28 15:12 UTC" in text + assert "Reveal Sep 28 14:48 UTC" in text + assert "different commits" in text + assert "unknown ≠ not revealed" in text diff --git a/tests/test_generator_scores.py b/tests/test_generator_scores.py index 724ffa6a..75058ce1 100644 --- a/tests/test_generator_scores.py +++ b/tests/test_generator_scores.py @@ -4,6 +4,7 @@ import asyncio from dataclasses import asdict import json +import sqlite3 from pathlib import Path import time import traceback @@ -21,7 +22,10 @@ resolve_generator_qualification, ) from gas.utils.state_manager import load_validator_state, save_validator_state -from gas.koth_weights import build_koth_weights +from gas.koth_weights import ( + build_koth_weights, chains_by_modality, discriminator_emissions_enabled, + kings_by_modality, +) def _qualified(image=True, video=True): @@ -115,10 +119,18 @@ def validator(tmp_path): tree = ast.parse(source.read_text()) cls = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Validator") methods = [node for node in cls.body if isinstance(node, ast.AsyncFunctionDef) - and node.name in {"update_scores", "save_state", "load_state"}] + and node.name in {"update_scores", "save_state", "load_state", "set_weights"}] + for method in methods: + method.decorator_list = [] namespace = { "np": np, "time": time, "traceback": traceback, - "bt": SimpleNamespace(logging=Mock()), + "bt": SimpleNamespace(logging=Mock(), Subtensor=Mock()), + "BURN_PERCENTAGE": 0.0, "BURN_SS58": "burn", + "get_current_kings": AsyncMock(return_value={"kings": []}), + "kings_by_modality": kings_by_modality, + "chains_by_modality": chains_by_modality, + "discriminator_emissions_enabled": discriminator_emissions_enabled, + "build_koth_weights": build_koth_weights, "get_benchmark_results": AsyncMock(), "get_generator_base_rewards": lambda stats: (stats, []), "get_generator_qualification": get_generator_qualification, @@ -147,6 +159,8 @@ def validator(tmp_path): instance.generative_challenge_manager.get_all_generator_last_seen.return_value = {} instance.kings_state = Mock() instance.api = namespace["get_benchmark_results"] + instance.weight_globals = namespace + instance.set_weights_fn = Mock() instance.api.return_value = [ {"ss58_address": "A", "modality": modality, "fooled_count": 10, "not_fooled_count": 10} for modality in ("image", "video") @@ -154,6 +168,219 @@ def validator(tmp_path): return instance +def prepare_weight_submission(validator): + validator.metagraph.hotkeys = ["A", "burn"] + validator.metagraph.n = 2 + validator.config.netuid = 34 + validator.config.subtensor = SimpleNamespace(chain_endpoint="unused") + validator.weight_globals["bt"].Subtensor.return_value.get_uid_for_hotkey_on_subnet.return_value = 1 + + +@pytest.mark.parametrize("message", [ + "database disk image is malformed", "disk I/O error", "database is locked", +]) +def test_database_outage_resubmits_payout_preserves_state_and_recovers(validator, message): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + previous_scores = validator.scores.copy() + previous_history = json.loads(json.dumps(validator.generator_score_state.by_hotkey)) + previous_qualification = dict(validator.generator_qualification) + previous_payout = dict(validator.generator_score_state.last_payout) + previous_weights = validator.set_weights_fn.call_args.args[3][1].copy() + validator.set_weights_fn.reset_mock() + validator.api.reset_mock() + kings = validator.weight_globals["get_current_kings"] + kings.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError(message) + + assert asyncio.run(validator.set_weights(460)) is True + validator.set_weights_fn.assert_called_once() + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], previous_weights) + validator.api.assert_not_awaited() + kings.assert_awaited_once() + assert np.array_equal(validator.scores, previous_scores) + assert validator.generator_score_state.by_hotkey == previous_history + assert validator.generator_qualification == previous_qualification + assert validator.generator_score_state.last_payout == previous_payout + + validator.set_weights_fn.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = None + assert asyncio.run(validator.set_weights(820)) is True + validator.set_weights_fn.assert_called_once() + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == pytest.approx(0.16) + + +def test_missing_score_result_skips_weights(validator): + validator.update_scores = AsyncMock(return_value=None) + assert asyncio.run(validator.set_weights(100)) is False + validator.set_weights_fn.assert_not_called() + validator.weight_globals["get_current_kings"].assert_not_awaited() + + +def test_repeated_missing_score_results_resubmit_without_decaying_snapshot(validator): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + previous_scores = validator.scores.copy() + previous_payout = dict(validator.generator_score_state.last_payout) + previous_weights = validator.set_weights_fn.call_args.args[3][1].copy() + validator.update_scores = AsyncMock(return_value=None) + validator.set_weights_fn.reset_mock() + for block in (460, 820, 1180): + assert asyncio.run(validator.set_weights(block)) is True + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], previous_weights) + assert np.array_equal(validator.scores, previous_scores) + assert validator.generator_score_state.last_payout == previous_payout + assert validator.set_weights_fn.call_count == 3 + + +def test_real_empty_window_retains_intended_burn_behavior(validator): + prepare_weight_submission(validator) + validator.content_manager.get_verification_stats_last_n_hours.return_value = {} + assert asyncio.run(validator.set_weights(100)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == [0.0, 1.0] + + +def test_full_burn_does_not_require_generator_database(validator): + prepare_weight_submission(validator) + validator.weight_globals["BURN_PERCENTAGE"] = 1.0 + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(100)) is True + validator.content_manager.get_verification_stats_last_n_hours.assert_not_called() + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == [0.0, 1.0] + + +def test_cold_start_database_outage_has_no_safe_fallback(validator): + prepare_weight_submission(validator) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(100)) is False + validator.set_weights_fn.assert_not_called() + assert validator.generator_score_state.last_payout is None + + +@pytest.mark.parametrize("empty", [False, True]) +def test_payout_fallback_survives_restart_including_known_empty(validator, empty): + prepare_weight_submission(validator) + if empty: + validator.content_manager.get_verification_stats_last_n_hours.return_value = {} + assert asyncio.run(validator.set_weights(100)) is True + expected = validator.set_weights_fn.call_args.args[3][1].copy() + # set_weights itself must persist the snapshot, without a later challenge save. + _restart(validator) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], expected) + assert validator.scores.tolist() == [0.0, 0.0] # Fallback does not rebuild EMA. + + +@pytest.mark.parametrize("still_registered", [False, True]) +def test_payout_fallback_never_transfers_to_reused_uid(validator, still_registered): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + validator.metagraph.hotkeys = ["replacement", "burn"] + (["A"] if still_registered else []) + validator.metagraph.n = len(validator.metagraph.hotkeys) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == 0 + assert weights[1] == pytest.approx(0.84 if still_registered else 1.0) + if still_registered: + assert weights[2] == pytest.approx(0.16) + + +def test_ineligible_positive_ema_is_not_in_fallback(validator): + prepare_weight_submission(validator) + validator.metagraph.hotkeys.append("B") + validator.metagraph.n = 3 + stats = validator.content_manager.get_verification_stats_last_n_hours.return_value + stats[2] = {"image": 10, "video": 10} + validator.api.return_value.extend([ + {"ss58_address": "B", "modality": m, "fooled_count": 10, "not_fooled_count": 10} + for m in ("image", "video") + ]) + assert asyncio.run(validator.set_weights(100)) is True + del stats[0] + assert asyncio.run(validator.set_weights(460)) is True + assert validator.scores[0] > 0 + assert set(validator.generator_score_state.last_payout) == {"B"} + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(820)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == 0 + assert weights[2] == pytest.approx(0.16) + + +def test_successful_disqualification_replaces_fallback_with_empty(validator): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + for row in validator.api.return_value: + row["fooled_count"] = 0 + assert asyncio.run(validator.set_weights(460)) is True + assert validator.generator_score_state.last_payout == {} + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(820)) is True + assert validator.set_weights_fn.call_args.args[3][1].tolist() == [0.0, 1.0] + + +def test_fallback_refreshes_kings_instead_of_replaying_old_final_weights(validator): + prepare_weight_submission(validator) + validator.metagraph.hotkeys.append("king") + validator.metagraph.n = 3 + assert asyncio.run(validator.set_weights(100)) is True + validator.weight_globals["get_current_kings"].return_value = { + "kings": [{"modality": "image", "ss58_address": "king"}], + "emissions_enabled": True, "emissions_start_at": "2020-01-01T00:00:00Z", + } + validator.weight_globals["bt"].Subtensor.return_value.get_uid_for_hotkey_on_subnet.side_effect = ( + lambda hotkey_ss58, netuid: validator.metagraph.hotkeys.index(hotkey_ss58) + ) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == pytest.approx([0.16, 0.44, 0.4]) + + +def test_legacy_snapshot_needs_successful_scoring_before_outage_fallback(validator, tmp_path): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + path = tmp_path / "state_current" / "generator_scores.json" + payload = json.loads(path.read_text()) + del payload["last_payout"] + path.write_text(json.dumps(payload)) + _restart(validator) + assert validator.generator_score_state.last_payout is None + assert validator.generator_score_state.by_hotkey # Existing EMA is preserved. + validator.set_weights_fn.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is False + validator.set_weights_fn.assert_not_called() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = None + assert asyncio.run(validator.set_weights(820)) is True + assert validator.generator_score_state.last_payout + + +@pytest.mark.parametrize("payout", [[], {"": 1}, {"A": True}, {"A": "1"}, + {"A": -1}, {"A": 0}, {"A": float("nan")}, + {"A": float("inf")}]) +def test_invalid_payout_snapshot_cannot_leave_stale_fallback(tmp_path, payout): + state = GeneratorScoreState() + state.last_payout = {"old": 5.0} + (tmp_path / "ema.json").write_text(json.dumps({ + "version": 1, "by_hotkey": {}, "qualification": {}, "last_payout": payout, + })) + assert not state.load_state(tmp_path, "ema.json") + assert state.last_payout is None + + +def test_missing_snapshot_clears_old_payout(tmp_path): + state = GeneratorScoreState() + state.last_payout = {"old": 5.0} + assert not state.load_state(tmp_path, "missing.json") + assert state.last_payout is None + + def test_validator_outage_uses_cached_gates_then_clears_lost_lane(validator): assert asyncio.run(validator.update_scores()) == [0] assert validator.scores[0] == 5 diff --git a/uv.lock b/uv.lock index 16188e76..acf9e528 100644 --- a/uv.lock +++ b/uv.lock @@ -1207,9 +1207,11 @@ dependencies = [ { name = "pandas" }, { name = "peft" }, { name = "pillow" }, + { name = "plotext" }, { name = "pyarrow" }, { name = "python-dotenv" }, { name = "qwen-vl-utils" }, + { name = "rich" }, { name = "runwayml" }, { name = "starlette" }, { name = "torch" }, @@ -1258,9 +1260,11 @@ requires-dist = [ { name = "pandas", specifier = "==2.3.0" }, { name = "peft", specifier = "==0.17.0" }, { name = "pillow", specifier = "==12.3.0" }, + { name = "plotext", specifier = "==5.3.2" }, { name = "pyarrow", specifier = "==23.0.1" }, { name = "python-dotenv", specifier = "==1.2.2" }, { name = "qwen-vl-utils", specifier = ">=0.0.14" }, + { name = "rich", specifier = "==15.0.0" }, { name = "runwayml", specifier = ">=4.13.0" }, { name = "starlette", specifier = "==1.6.0" }, { name = "torch", specifier = "==2.7.1" }, @@ -2318,6 +2322,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] +[[package]] +name = "plotext" +version = "5.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/d7/f75f397af966fe252d0d34ffd3cae765317fce2134f925f95e7d6725d1ce/plotext-5.3.2.tar.gz", hash = "sha256:52d1e932e67c177bf357a3f0fe6ce14d1a96f7f7d5679d7b455b929df517068e", size = 61967, upload-time = "2024-09-24T15:13:37.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/1e/12fe7c40cd2099a1f454518754ed229b01beaf3bbb343127f0cc13ce6c22/plotext-5.3.2-py3-none-any.whl", hash = "sha256:394362349c1ddbf319548cfac17ca65e6d5dfc03200c40dfdc0503b3e95a2283", size = 64047, upload-time = "2024-09-24T15:13:36.296Z" }, +] + [[package]] name = "plotille" version = "6.0.5"