Keep submitting last-good payouts during reward database outages

gas/cache/content_manager.py

@@ -722,6 +722,10 @@ def get_verification_stats_last_n_hours(
 
         Returns:
             Dict mapping miner hotkey to verification stats, with per-modality breakdown.
+
+        Raises:
+            Exception: If reward data cannot be read or aggregated. Callers must
+                not interpret unavailable data as a successfully read empty window.
         """
         try:
             return self.challenges.get_outcome_stats_last_n_hours(
@@ -731,7 +735,7 @@ def get_verification_stats_last_n_hours(
             bt.logging.error(f"Error getting verification stats for last {lookback_hours}h: {e}")
             import traceback
             bt.logging.error(traceback.format_exc())
-            return {}
+            raise
 
     def upload_batch_to_huggingface(
         self, 

gas/cache/db/challenge_store.py

@@ -153,6 +153,8 @@ def get_outcomes_last_n_hours(
 
         Reward scoring must not let other miners' newer outcomes displace
         eligible work within the requested time window.
+
+        Read failures propagate: an unavailable window is not an empty window.
         """
         try:
             cutoff = time.time() - (lookback_hours * 3600)
@@ -197,7 +199,7 @@ def get_outcomes_last_n_hours(
                 ]
         except Exception as e:
             bt.logging.error(f"Error getting recent challenge outcomes: {e}")
-            return []
+            raise
 
     def get_outcome_stats_last_n_hours(
         self, lookback_hours: float = 2.0, limit: Optional[int] = None

gas/evaluation/generator_scores.py

@@ -10,11 +10,24 @@
 
 
 class GeneratorScoreState:
-    """Persist modality EMAs and their hotkey qualification fallback together."""
+    """Persist modality EMAs, qualification, and the last payable hotkey scores."""
 
     def __init__(self):
         self.by_hotkey: Dict[str, Dict[str, float]] = {}
         self.qualification: Dict[str, GeneratorQualification] = {}
+        # None means no safe fallback exists; {} means a successful empty payout.
+        # Unlike the EMA history, this contains only miners eligible for payout.
+        self.last_payout: Optional[Dict[str, float]] = None
+
+    def resolve_last_payout(self, hotkeys: Sequence[str]) -> Optional[Dict[int, float]]:
+        """Resolve the last good payout against current identities, never old UIDs."""
+        if self.last_payout is None:
+            return None
+        return {
+            uid: self.last_payout[hotkey]
+            for uid, hotkey in enumerate(hotkeys)
+            if hotkey in self.last_payout
+        }
 
     def update(
         self,
@@ -62,12 +75,14 @@ def save_state(self, save_dir: str, filename: str) -> None:
                 "version": 1,
                 "by_hotkey": self.by_hotkey,
                 "qualification": {hotkey: asdict(q) for hotkey, q in self.qualification.items()},
+                "last_payout": self.last_payout,
             }, stream, allow_nan=False)
 
     def load_state(self, save_dir: str, filename: str) -> bool:
         # Missing/invalid new-format state must never reuse legacy scalar EMA.
         self.by_hotkey = {}
         self.qualification = {}
+        self.last_payout = None
         path = Path(save_dir) / filename
         if not path.exists():
             return False
@@ -104,8 +119,19 @@ def load_state(self, save_dir: str, filename: str) -> bool:
                     if not 0 <= fooled <= n or (qualified and n == 0):
                         return False
                 qualification[hotkey] = GeneratorQualification(**row)
+            # Older snapshots cannot establish payout eligibility from EMA alone.
+            payout = payload.get("last_payout")
+            if payout is not None:
+                if not isinstance(payout, dict):
+                    return False
+                for hotkey, score in payout.items():
+                    if (not isinstance(hotkey, str) or not hotkey
+                            or type(score) not in (int, float)
+                            or not math.isfinite(score) or score <= 0):
+                        return False
             self.by_hotkey = restored
             self.qualification = qualification
+            self.last_payout = payout
             return True
         except (OSError, ValueError, TypeError, KeyError, AttributeError):
             return False

neurons/validator/validator.py

@@ -203,11 +203,27 @@ async def set_weights(self, block):
         """Set burn weights, or calculate rewards when full burn is disabled."""
         if BURN_PERCENTAGE < 1.0:
             bt.logging.info(f"Updating scores at block {block}")
-            generator_uids = await self.update_scores()
-
+            generator_uids = None
+            try:
+                generator_uids = await self.update_scores()
+            except Exception:
+                bt.logging.error(
+                    "Generator scoring failed; attempting last-good payout fallback.\n"
+                    + traceback.format_exc()
+                )
+
             if generator_uids is None:
-                generator_uids = []
-                bt.logging.warning("No generator rewards available; using empty generator_uids")
+                bt.logging.warning(
+                    "Generator scoring unavailable; keeping scores unchanged and "
+                    "using the last-good payout snapshot."
+                )
+            else:
+                # Persist eligibility with scores so restart-time outages can use it.
+                await self.save_state()
+
+            if self.generator_score_state.last_payout is None:
+                bt.logging.error("Skipping weight submission: no last-good generator payout snapshot")
+                return False
 
             kings_payload = await get_current_kings(
                 self.wallet.hotkey, base_url=self.config.benchmark_api_url
@@ -261,10 +277,19 @@ def uid_for_hotkey(hotkey_ss58: str):
                 if BURN_PERCENTAGE == 1.0:
                     normed_weights = np.zeros(int(self.metagraph.n), dtype=np.float64)
                 else:
+                    # Re-resolve identities even after a successful update: UID slots
+                    # may have changed. Do not reuse positive but ineligible EMA tails.
+                    payout = self.generator_score_state.resolve_last_payout(self.metagraph.hotkeys)
+                    if payout is None:
+                        bt.logging.error("Skipping weight submission: generator payout snapshot unavailable")
+                        return False
+                    payout_scores = np.zeros(int(self.metagraph.n), dtype=np.float64)
+                    for uid, score in payout.items():
+                        payout_scores[uid] = score
                     normed_weights = build_koth_weights(
                         n=int(self.metagraph.n),
-                        scores=self.scores,
-                        generator_uids=generator_uids,
+                        scores=payout_scores,
+                        generator_uids=list(payout),
                         kings=kings,
                         uid_for_hotkey=uid_for_hotkey,
                         burn_uid=burn_uid,
@@ -377,6 +402,11 @@ async def update_scores(self):
             self.scores = np.zeros(len(hotkeys), dtype=np.float64)
             for uid, score in scores.items():
                 self.scores[uid] = score
+            self.generator_score_state.last_payout = {
+                hotkeys[uid]: scores[uid]
+                for uid in rewards
+                if scores.get(uid, 0.0) > 0
+            }
 
         bt.logging.info(
             f"Updated scores for {len(rewards)} miners with EMA (alpha={alpha})"
@@ -468,8 +498,8 @@ async def load_state(self):
                 self.generator_qualification = self.generator_score_state.qualification
                 # scores.npy is retained for snapshot compatibility, not as EMA
                 # input: legacy scalars cannot be split by modality or hotkey.
-                # Rebuild the payout vector after qualification/liveness checks
-                # in update_scores; a restored gate is fallback during outages.
+                # Fresh scoring rebuilds this vector; database outages use the
+                # separate last_payout snapshot, not unfiltered EMA history.
                 self.scores = np.zeros(len(self.metagraph.hotkeys), dtype=np.float64)
                 bt.logging.info(
                     f"Loaded modality EMA histories for {len(self.generator_score_state.by_hotkey)} hotkeys; "

tests/test_full_window_rewards.py

@@ -1,6 +1,7 @@
 """The reward lookback is time-bounded, not a global newest-N sample."""
 
 import ast
+import sqlite3
 from pathlib import Path
 from types import SimpleNamespace
 from unittest.mock import Mock
@@ -120,3 +121,38 @@ def test_shorter_lookback_still_excludes_older_work(store):
     stats = manager_for(store).get_verification_stats_last_n_hours(lookback_hours=2)
     assert set(stats) == {"busy"}
     assert stats["busy"]["total_verified"] == 1100
+
+
+@pytest.mark.parametrize("message", [
+    "database disk image is malformed", "disk I/O error", "database is locked",
+])
+def test_unavailable_database_is_not_an_empty_reward_window(store, monkeypatch, message):
+    error = sqlite3.DatabaseError(message)
+    monkeypatch.setattr(store.db, "connect", Mock(side_effect=error))
+    for query in (
+        store.get_outcomes_last_n_hours,
+        store.get_outcome_stats_last_n_hours,
+        manager_for(store).get_verification_stats_last_n_hours,
+    ):
+        with pytest.raises(sqlite3.DatabaseError, match=message):
+            query(24)
+
+
+def test_partial_cursor_failure_does_not_return_partial_rewards(store, monkeypatch):
+    def broken_cursor():
+        yield dict(task_id="partial", uid=1, hotkey="busy", prompt_id="prompt",
+                   modality="video", status="verified", failure_reason=None,
+                   media_id=None, media_resolution=None, media_has_audio=None,
+                   created_at=NOW, updated_at=NOW)
+        raise sqlite3.DatabaseError("disk I/O error")
+
+    from unittest.mock import MagicMock
+    connection = MagicMock()
+    connection.__enter__.return_value.execute.return_value = broken_cursor()
+    monkeypatch.setattr(store.db, "connect", Mock(return_value=connection))
+    with pytest.raises(sqlite3.DatabaseError, match="disk I/O error"):
+        manager_for(store).get_verification_stats_last_n_hours(24)
+
+
+def test_successfully_read_empty_window_is_still_empty(store):
+    assert manager_for(store).get_verification_stats_last_n_hours(0) == {}