diff --git a/gas/cache/content_manager.py b/gas/cache/content_manager.py index 049fc206..f594ab7b 100644 --- a/gas/cache/content_manager.py +++ b/gas/cache/content_manager.py @@ -722,6 +722,10 @@ def get_verification_stats_last_n_hours( Returns: Dict mapping miner hotkey to verification stats, with per-modality breakdown. + + Raises: + Exception: If reward data cannot be read or aggregated. Callers must + not interpret unavailable data as a successfully read empty window. """ try: return self.challenges.get_outcome_stats_last_n_hours( @@ -731,7 +735,7 @@ def get_verification_stats_last_n_hours( bt.logging.error(f"Error getting verification stats for last {lookback_hours}h: {e}") import traceback bt.logging.error(traceback.format_exc()) - return {} + raise def upload_batch_to_huggingface( self, diff --git a/gas/cache/db/challenge_store.py b/gas/cache/db/challenge_store.py index 9978983e..7453fb54 100644 --- a/gas/cache/db/challenge_store.py +++ b/gas/cache/db/challenge_store.py @@ -153,6 +153,8 @@ def get_outcomes_last_n_hours( Reward scoring must not let other miners' newer outcomes displace eligible work within the requested time window. + + Read failures propagate: an unavailable window is not an empty window. """ try: cutoff = time.time() - (lookback_hours * 3600) @@ -197,7 +199,7 @@ def get_outcomes_last_n_hours( ] except Exception as e: bt.logging.error(f"Error getting recent challenge outcomes: {e}") - return [] + raise def get_outcome_stats_last_n_hours( self, lookback_hours: float = 2.0, limit: Optional[int] = None diff --git a/gas/evaluation/generator_scores.py b/gas/evaluation/generator_scores.py index 17854e4f..5405d215 100644 --- a/gas/evaluation/generator_scores.py +++ b/gas/evaluation/generator_scores.py @@ -10,11 +10,24 @@ class GeneratorScoreState: - """Persist modality EMAs and their hotkey qualification fallback together.""" + """Persist modality EMAs, qualification, and the last payable hotkey scores.""" def __init__(self): self.by_hotkey: Dict[str, Dict[str, float]] = {} self.qualification: Dict[str, GeneratorQualification] = {} + # None means no safe fallback exists; {} means a successful empty payout. + # Unlike the EMA history, this contains only miners eligible for payout. + self.last_payout: Optional[Dict[str, float]] = None + + def resolve_last_payout(self, hotkeys: Sequence[str]) -> Optional[Dict[int, float]]: + """Resolve the last good payout against current identities, never old UIDs.""" + if self.last_payout is None: + return None + return { + uid: self.last_payout[hotkey] + for uid, hotkey in enumerate(hotkeys) + if hotkey in self.last_payout + } def update( self, @@ -62,12 +75,14 @@ def save_state(self, save_dir: str, filename: str) -> None: "version": 1, "by_hotkey": self.by_hotkey, "qualification": {hotkey: asdict(q) for hotkey, q in self.qualification.items()}, + "last_payout": self.last_payout, }, stream, allow_nan=False) def load_state(self, save_dir: str, filename: str) -> bool: # Missing/invalid new-format state must never reuse legacy scalar EMA. self.by_hotkey = {} self.qualification = {} + self.last_payout = None path = Path(save_dir) / filename if not path.exists(): return False @@ -104,8 +119,19 @@ def load_state(self, save_dir: str, filename: str) -> bool: if not 0 <= fooled <= n or (qualified and n == 0): return False qualification[hotkey] = GeneratorQualification(**row) + # Older snapshots cannot establish payout eligibility from EMA alone. + payout = payload.get("last_payout") + if payout is not None: + if not isinstance(payout, dict): + return False + for hotkey, score in payout.items(): + if (not isinstance(hotkey, str) or not hotkey + or type(score) not in (int, float) + or not math.isfinite(score) or score <= 0): + return False self.by_hotkey = restored self.qualification = qualification + self.last_payout = payout return True except (OSError, ValueError, TypeError, KeyError, AttributeError): return False diff --git a/neurons/validator/validator.py b/neurons/validator/validator.py index fde8030d..21fea5c5 100644 --- a/neurons/validator/validator.py +++ b/neurons/validator/validator.py @@ -203,11 +203,27 @@ async def set_weights(self, block): """Set burn weights, or calculate rewards when full burn is disabled.""" if BURN_PERCENTAGE < 1.0: bt.logging.info(f"Updating scores at block {block}") - generator_uids = await self.update_scores() - + generator_uids = None + try: + generator_uids = await self.update_scores() + except Exception: + bt.logging.error( + "Generator scoring failed; attempting last-good payout fallback.\n" + + traceback.format_exc() + ) + if generator_uids is None: - generator_uids = [] - bt.logging.warning("No generator rewards available; using empty generator_uids") + bt.logging.warning( + "Generator scoring unavailable; keeping scores unchanged and " + "using the last-good payout snapshot." + ) + else: + # Persist eligibility with scores so restart-time outages can use it. + await self.save_state() + + if self.generator_score_state.last_payout is None: + bt.logging.error("Skipping weight submission: no last-good generator payout snapshot") + return False kings_payload = await get_current_kings( self.wallet.hotkey, base_url=self.config.benchmark_api_url @@ -261,10 +277,19 @@ def uid_for_hotkey(hotkey_ss58: str): if BURN_PERCENTAGE == 1.0: normed_weights = np.zeros(int(self.metagraph.n), dtype=np.float64) else: + # Re-resolve identities even after a successful update: UID slots + # may have changed. Do not reuse positive but ineligible EMA tails. + payout = self.generator_score_state.resolve_last_payout(self.metagraph.hotkeys) + if payout is None: + bt.logging.error("Skipping weight submission: generator payout snapshot unavailable") + return False + payout_scores = np.zeros(int(self.metagraph.n), dtype=np.float64) + for uid, score in payout.items(): + payout_scores[uid] = score normed_weights = build_koth_weights( n=int(self.metagraph.n), - scores=self.scores, - generator_uids=generator_uids, + scores=payout_scores, + generator_uids=list(payout), kings=kings, uid_for_hotkey=uid_for_hotkey, burn_uid=burn_uid, @@ -377,6 +402,11 @@ async def update_scores(self): self.scores = np.zeros(len(hotkeys), dtype=np.float64) for uid, score in scores.items(): self.scores[uid] = score + self.generator_score_state.last_payout = { + hotkeys[uid]: scores[uid] + for uid in rewards + if scores.get(uid, 0.0) > 0 + } bt.logging.info( f"Updated scores for {len(rewards)} miners with EMA (alpha={alpha})" @@ -468,8 +498,8 @@ async def load_state(self): self.generator_qualification = self.generator_score_state.qualification # scores.npy is retained for snapshot compatibility, not as EMA # input: legacy scalars cannot be split by modality or hotkey. - # Rebuild the payout vector after qualification/liveness checks - # in update_scores; a restored gate is fallback during outages. + # Fresh scoring rebuilds this vector; database outages use the + # separate last_payout snapshot, not unfiltered EMA history. self.scores = np.zeros(len(self.metagraph.hotkeys), dtype=np.float64) bt.logging.info( f"Loaded modality EMA histories for {len(self.generator_score_state.by_hotkey)} hotkeys; " diff --git a/tests/test_full_window_rewards.py b/tests/test_full_window_rewards.py index eb741d5e..d1a4f59a 100644 --- a/tests/test_full_window_rewards.py +++ b/tests/test_full_window_rewards.py @@ -1,6 +1,7 @@ """The reward lookback is time-bounded, not a global newest-N sample.""" import ast +import sqlite3 from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock @@ -120,3 +121,38 @@ def test_shorter_lookback_still_excludes_older_work(store): stats = manager_for(store).get_verification_stats_last_n_hours(lookback_hours=2) assert set(stats) == {"busy"} assert stats["busy"]["total_verified"] == 1100 + + +@pytest.mark.parametrize("message", [ + "database disk image is malformed", "disk I/O error", "database is locked", +]) +def test_unavailable_database_is_not_an_empty_reward_window(store, monkeypatch, message): + error = sqlite3.DatabaseError(message) + monkeypatch.setattr(store.db, "connect", Mock(side_effect=error)) + for query in ( + store.get_outcomes_last_n_hours, + store.get_outcome_stats_last_n_hours, + manager_for(store).get_verification_stats_last_n_hours, + ): + with pytest.raises(sqlite3.DatabaseError, match=message): + query(24) + + +def test_partial_cursor_failure_does_not_return_partial_rewards(store, monkeypatch): + def broken_cursor(): + yield dict(task_id="partial", uid=1, hotkey="busy", prompt_id="prompt", + modality="video", status="verified", failure_reason=None, + media_id=None, media_resolution=None, media_has_audio=None, + created_at=NOW, updated_at=NOW) + raise sqlite3.DatabaseError("disk I/O error") + + from unittest.mock import MagicMock + connection = MagicMock() + connection.__enter__.return_value.execute.return_value = broken_cursor() + monkeypatch.setattr(store.db, "connect", Mock(return_value=connection)) + with pytest.raises(sqlite3.DatabaseError, match="disk I/O error"): + manager_for(store).get_verification_stats_last_n_hours(24) + + +def test_successfully_read_empty_window_is_still_empty(store): + assert manager_for(store).get_verification_stats_last_n_hours(0) == {} diff --git a/tests/test_generator_scores.py b/tests/test_generator_scores.py index 724ffa6a..75058ce1 100644 --- a/tests/test_generator_scores.py +++ b/tests/test_generator_scores.py @@ -4,6 +4,7 @@ import asyncio from dataclasses import asdict import json +import sqlite3 from pathlib import Path import time import traceback @@ -21,7 +22,10 @@ resolve_generator_qualification, ) from gas.utils.state_manager import load_validator_state, save_validator_state -from gas.koth_weights import build_koth_weights +from gas.koth_weights import ( + build_koth_weights, chains_by_modality, discriminator_emissions_enabled, + kings_by_modality, +) def _qualified(image=True, video=True): @@ -115,10 +119,18 @@ def validator(tmp_path): tree = ast.parse(source.read_text()) cls = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Validator") methods = [node for node in cls.body if isinstance(node, ast.AsyncFunctionDef) - and node.name in {"update_scores", "save_state", "load_state"}] + and node.name in {"update_scores", "save_state", "load_state", "set_weights"}] + for method in methods: + method.decorator_list = [] namespace = { "np": np, "time": time, "traceback": traceback, - "bt": SimpleNamespace(logging=Mock()), + "bt": SimpleNamespace(logging=Mock(), Subtensor=Mock()), + "BURN_PERCENTAGE": 0.0, "BURN_SS58": "burn", + "get_current_kings": AsyncMock(return_value={"kings": []}), + "kings_by_modality": kings_by_modality, + "chains_by_modality": chains_by_modality, + "discriminator_emissions_enabled": discriminator_emissions_enabled, + "build_koth_weights": build_koth_weights, "get_benchmark_results": AsyncMock(), "get_generator_base_rewards": lambda stats: (stats, []), "get_generator_qualification": get_generator_qualification, @@ -147,6 +159,8 @@ def validator(tmp_path): instance.generative_challenge_manager.get_all_generator_last_seen.return_value = {} instance.kings_state = Mock() instance.api = namespace["get_benchmark_results"] + instance.weight_globals = namespace + instance.set_weights_fn = Mock() instance.api.return_value = [ {"ss58_address": "A", "modality": modality, "fooled_count": 10, "not_fooled_count": 10} for modality in ("image", "video") @@ -154,6 +168,219 @@ def validator(tmp_path): return instance +def prepare_weight_submission(validator): + validator.metagraph.hotkeys = ["A", "burn"] + validator.metagraph.n = 2 + validator.config.netuid = 34 + validator.config.subtensor = SimpleNamespace(chain_endpoint="unused") + validator.weight_globals["bt"].Subtensor.return_value.get_uid_for_hotkey_on_subnet.return_value = 1 + + +@pytest.mark.parametrize("message", [ + "database disk image is malformed", "disk I/O error", "database is locked", +]) +def test_database_outage_resubmits_payout_preserves_state_and_recovers(validator, message): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + previous_scores = validator.scores.copy() + previous_history = json.loads(json.dumps(validator.generator_score_state.by_hotkey)) + previous_qualification = dict(validator.generator_qualification) + previous_payout = dict(validator.generator_score_state.last_payout) + previous_weights = validator.set_weights_fn.call_args.args[3][1].copy() + validator.set_weights_fn.reset_mock() + validator.api.reset_mock() + kings = validator.weight_globals["get_current_kings"] + kings.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError(message) + + assert asyncio.run(validator.set_weights(460)) is True + validator.set_weights_fn.assert_called_once() + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], previous_weights) + validator.api.assert_not_awaited() + kings.assert_awaited_once() + assert np.array_equal(validator.scores, previous_scores) + assert validator.generator_score_state.by_hotkey == previous_history + assert validator.generator_qualification == previous_qualification + assert validator.generator_score_state.last_payout == previous_payout + + validator.set_weights_fn.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = None + assert asyncio.run(validator.set_weights(820)) is True + validator.set_weights_fn.assert_called_once() + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == pytest.approx(0.16) + + +def test_missing_score_result_skips_weights(validator): + validator.update_scores = AsyncMock(return_value=None) + assert asyncio.run(validator.set_weights(100)) is False + validator.set_weights_fn.assert_not_called() + validator.weight_globals["get_current_kings"].assert_not_awaited() + + +def test_repeated_missing_score_results_resubmit_without_decaying_snapshot(validator): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + previous_scores = validator.scores.copy() + previous_payout = dict(validator.generator_score_state.last_payout) + previous_weights = validator.set_weights_fn.call_args.args[3][1].copy() + validator.update_scores = AsyncMock(return_value=None) + validator.set_weights_fn.reset_mock() + for block in (460, 820, 1180): + assert asyncio.run(validator.set_weights(block)) is True + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], previous_weights) + assert np.array_equal(validator.scores, previous_scores) + assert validator.generator_score_state.last_payout == previous_payout + assert validator.set_weights_fn.call_count == 3 + + +def test_real_empty_window_retains_intended_burn_behavior(validator): + prepare_weight_submission(validator) + validator.content_manager.get_verification_stats_last_n_hours.return_value = {} + assert asyncio.run(validator.set_weights(100)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == [0.0, 1.0] + + +def test_full_burn_does_not_require_generator_database(validator): + prepare_weight_submission(validator) + validator.weight_globals["BURN_PERCENTAGE"] = 1.0 + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(100)) is True + validator.content_manager.get_verification_stats_last_n_hours.assert_not_called() + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == [0.0, 1.0] + + +def test_cold_start_database_outage_has_no_safe_fallback(validator): + prepare_weight_submission(validator) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(100)) is False + validator.set_weights_fn.assert_not_called() + assert validator.generator_score_state.last_payout is None + + +@pytest.mark.parametrize("empty", [False, True]) +def test_payout_fallback_survives_restart_including_known_empty(validator, empty): + prepare_weight_submission(validator) + if empty: + validator.content_manager.get_verification_stats_last_n_hours.return_value = {} + assert asyncio.run(validator.set_weights(100)) is True + expected = validator.set_weights_fn.call_args.args[3][1].copy() + # set_weights itself must persist the snapshot, without a later challenge save. + _restart(validator) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + assert np.array_equal(validator.set_weights_fn.call_args.args[3][1], expected) + assert validator.scores.tolist() == [0.0, 0.0] # Fallback does not rebuild EMA. + + +@pytest.mark.parametrize("still_registered", [False, True]) +def test_payout_fallback_never_transfers_to_reused_uid(validator, still_registered): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + validator.metagraph.hotkeys = ["replacement", "burn"] + (["A"] if still_registered else []) + validator.metagraph.n = len(validator.metagraph.hotkeys) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == 0 + assert weights[1] == pytest.approx(0.84 if still_registered else 1.0) + if still_registered: + assert weights[2] == pytest.approx(0.16) + + +def test_ineligible_positive_ema_is_not_in_fallback(validator): + prepare_weight_submission(validator) + validator.metagraph.hotkeys.append("B") + validator.metagraph.n = 3 + stats = validator.content_manager.get_verification_stats_last_n_hours.return_value + stats[2] = {"image": 10, "video": 10} + validator.api.return_value.extend([ + {"ss58_address": "B", "modality": m, "fooled_count": 10, "not_fooled_count": 10} + for m in ("image", "video") + ]) + assert asyncio.run(validator.set_weights(100)) is True + del stats[0] + assert asyncio.run(validator.set_weights(460)) is True + assert validator.scores[0] > 0 + assert set(validator.generator_score_state.last_payout) == {"B"} + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(820)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights[0] == 0 + assert weights[2] == pytest.approx(0.16) + + +def test_successful_disqualification_replaces_fallback_with_empty(validator): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + for row in validator.api.return_value: + row["fooled_count"] = 0 + assert asyncio.run(validator.set_weights(460)) is True + assert validator.generator_score_state.last_payout == {} + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(820)) is True + assert validator.set_weights_fn.call_args.args[3][1].tolist() == [0.0, 1.0] + + +def test_fallback_refreshes_kings_instead_of_replaying_old_final_weights(validator): + prepare_weight_submission(validator) + validator.metagraph.hotkeys.append("king") + validator.metagraph.n = 3 + assert asyncio.run(validator.set_weights(100)) is True + validator.weight_globals["get_current_kings"].return_value = { + "kings": [{"modality": "image", "ss58_address": "king"}], + "emissions_enabled": True, "emissions_start_at": "2020-01-01T00:00:00Z", + } + validator.weight_globals["bt"].Subtensor.return_value.get_uid_for_hotkey_on_subnet.side_effect = ( + lambda hotkey_ss58, netuid: validator.metagraph.hotkeys.index(hotkey_ss58) + ) + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is True + _, weights = validator.set_weights_fn.call_args.args[3] + assert weights.tolist() == pytest.approx([0.16, 0.44, 0.4]) + + +def test_legacy_snapshot_needs_successful_scoring_before_outage_fallback(validator, tmp_path): + prepare_weight_submission(validator) + assert asyncio.run(validator.set_weights(100)) is True + path = tmp_path / "state_current" / "generator_scores.json" + payload = json.loads(path.read_text()) + del payload["last_payout"] + path.write_text(json.dumps(payload)) + _restart(validator) + assert validator.generator_score_state.last_payout is None + assert validator.generator_score_state.by_hotkey # Existing EMA is preserved. + validator.set_weights_fn.reset_mock() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = sqlite3.DatabaseError("disk I/O error") + assert asyncio.run(validator.set_weights(460)) is False + validator.set_weights_fn.assert_not_called() + validator.content_manager.get_verification_stats_last_n_hours.side_effect = None + assert asyncio.run(validator.set_weights(820)) is True + assert validator.generator_score_state.last_payout + + +@pytest.mark.parametrize("payout", [[], {"": 1}, {"A": True}, {"A": "1"}, + {"A": -1}, {"A": 0}, {"A": float("nan")}, + {"A": float("inf")}]) +def test_invalid_payout_snapshot_cannot_leave_stale_fallback(tmp_path, payout): + state = GeneratorScoreState() + state.last_payout = {"old": 5.0} + (tmp_path / "ema.json").write_text(json.dumps({ + "version": 1, "by_hotkey": {}, "qualification": {}, "last_payout": payout, + })) + assert not state.load_state(tmp_path, "ema.json") + assert state.last_payout is None + + +def test_missing_snapshot_clears_old_payout(tmp_path): + state = GeneratorScoreState() + state.last_payout = {"old": 5.0} + assert not state.load_state(tmp_path, "missing.json") + assert state.last_payout is None + + def test_validator_outage_uses_cached_gates_then_clears_lost_lane(validator): assert asyncio.run(validator.update_scores()) == [0] assert validator.scores[0] == 5