From 1209f15499e9d57cf11cd01ff16cedd3a7928e86 Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 19 Sep 2026 16:27:02 +0000 Subject: [PATCH 1/2] fix: score generator rewards over the full lookback window --- gas/cache/content_manager.py | 8 +- gas/cache/db/challenge_store.py | 42 +++++----- tests/test_full_window_rewards.py | 122 ++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 23 deletions(-) create mode 100644 tests/test_full_window_rewards.py diff --git a/gas/cache/content_manager.py b/gas/cache/content_manager.py index 0372b10c..049fc206 100644 --- a/gas/cache/content_manager.py +++ b/gas/cache/content_manager.py @@ -709,7 +709,7 @@ def get_recent_failed_miner_media(self, lookback_hours: float = 2.0, limit: int return self.media.get_recent_failed_miner_media(lookback_hours=lookback_hours, limit=limit) def get_verification_stats_last_n_hours( - self, lookback_hours: float = 2.0, limit: int = None + self, lookback_hours: float = 2.0, limit: Optional[int] = None ) -> Dict[str, Dict[str, Any]]: """ Get verification statistics from challenge outcomes table only (hard cutover). @@ -717,15 +717,15 @@ def get_verification_stats_last_n_hours( Args: lookback_hours: Number of hours to look back (default 2.0). - limit: Maximum number of entries per type to consider (default 1000). + limit: Optional maximum number of terminal outcomes across all miners. + None (default) includes the full window for reward scoring. Returns: Dict mapping miner hotkey to verification stats, with per-modality breakdown. """ try: - limit_val = limit or 1000 return self.challenges.get_outcome_stats_last_n_hours( - lookback_hours=lookback_hours, limit=limit_val + lookback_hours=lookback_hours, limit=limit ) except Exception as e: bt.logging.error(f"Error getting verification stats for last {lookback_hours}h: {e}") diff --git a/gas/cache/db/challenge_store.py b/gas/cache/db/challenge_store.py index fe1e7d1a..9978983e 100644 --- a/gas/cache/db/challenge_store.py +++ b/gas/cache/db/challenge_store.py @@ -147,27 +147,31 @@ def mark_outcome_for_media( # ------------------------------------------------------------------ def get_outcomes_last_n_hours( - self, lookback_hours: float = 2.0, limit: int = 1000 + self, lookback_hours: float = 2.0, limit: Optional[int] = None ) -> List[ChallengeOutcome]: - """Get terminal generation challenge outcomes from the last N hours.""" + """Get all terminal outcomes in the window unless explicitly limited. + + Reward scoring must not let other miners' newer outcomes displace + eligible work within the requested time window. + """ try: cutoff = time.time() - (lookback_hours * 3600) - if limit is None: - limit = 1000 + query = """ + SELECT o.*, m.resolution AS media_resolution, m.has_audio AS media_has_audio + FROM generator_challenge_outcomes o + LEFT JOIN media m ON o.media_id = m.id + WHERE o.status IN ('verified', 'failed') + AND o.updated_at >= ? + AND COALESCE(o.failure_reason, '') != 'no_answer' + ORDER BY o.updated_at DESC + """ + params = [cutoff] + if limit is not None: + query += " LIMIT ?" + params.append(int(limit)) with self.db.connect() as conn: conn.row_factory = sqlite3.Row - cursor = conn.execute( - """ - SELECT o.*, m.resolution AS media_resolution, m.has_audio AS media_has_audio - FROM generator_challenge_outcomes o - LEFT JOIN media m ON o.media_id = m.id - WHERE o.status IN ('verified', 'failed') - AND o.updated_at >= ? - AND COALESCE(o.failure_reason, '') != 'no_answer' - ORDER BY o.updated_at DESC LIMIT ? - """, - (cutoff, int(limit)), - ) + cursor = conn.execute(query, params) return [ ChallengeOutcome( task_id=row["task_id"], @@ -189,16 +193,16 @@ def get_outcomes_last_n_hours( created_at=row["created_at"], updated_at=row["updated_at"], ) - for row in cursor.fetchall() + for row in cursor ] except Exception as e: bt.logging.error(f"Error getting recent challenge outcomes: {e}") return [] def get_outcome_stats_last_n_hours( - self, lookback_hours: float = 2.0, limit: int = 1000 + self, lookback_hours: float = 2.0, limit: Optional[int] = None ) -> Dict[str, Dict[str, Any]]: - """Build reward stats from terminal challenge outcomes, split by modality.""" + """Build reward stats over the full time window, unless explicitly limited.""" outcomes = self.get_outcomes_last_n_hours(lookback_hours, limit) miner_stats: Dict[str, Dict[str, Any]] = {} for outcome in outcomes: diff --git a/tests/test_full_window_rewards.py b/tests/test_full_window_rewards.py new file mode 100644 index 00000000..eb741d5e --- /dev/null +++ b/tests/test_full_window_rewards.py @@ -0,0 +1,122 @@ +"""The reward lookback is time-bounded, not a global newest-N sample.""" + +import ast +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from gas.cache.db import ChallengeStore, ConnectionManager +from gas.cache.db.connection import create_schema +from gas.evaluation.rewards import ( + GeneratorQualification, + combine_generator_rewards, + get_generator_base_rewards, +) + + +NOW = 2_000_000_000.0 + + +@pytest.fixture +def store(tmp_path, monkeypatch): + monkeypatch.setattr("gas.cache.db.challenge_store.time.time", lambda: NOW) + db = ConnectionManager(tmp_path / "prompts.db") + with db.connect() as conn: + create_schema(conn) + conn.execute( + "INSERT INTO prompts (id, content, content_type, modality, created_at) " + "VALUES ('prompt', 'test', 'prompt', 'image', ?)", (NOW,), + ) + # Newer work from another miner must not crowd out the target's image. + rows = [ + (f"busy-{i}", 1, "busy", "video", "verified", None, NOW - 60 - i) + for i in range(1100) + ] + [ + ("older-image", 200, "target", "image", "verified", None, NOW - 23 * 3600), + ("failed-image", 200, "target", "image", "failed", "C2PA verification failed", NOW - 22 * 3600), + ("older-video", 200, "target", "video", "verified", None, NOW - 21 * 3600), + ("failed-video", 200, "target", "video", "failed", "challenge_timeout", NOW - 20 * 3600), + ("too-old", 200, "target", "image", "verified", None, NOW - 24 * 3600 - 1), + ("no-answer", 200, "target", "image", "failed", "no_answer", NOW), + ("pending", 200, "target", "image", "pending", None, NOW), + ("stored", 200, "target", "image", "stored", None, NOW), + ("at-cutoff", 2, "boundary", "image", "verified", None, NOW - 24 * 3600), + ] + conn.executemany( + "INSERT INTO generator_challenge_outcomes " + "(task_id,uid,hotkey,modality,status,failure_reason,updated_at,prompt_id,created_at) " + "VALUES (?,?,?,?,?,?,?,'prompt',?)", + # Scoring uses completion/update time, not creation time. + [(*row, NOW - 25 * 3600) for row in rows], + ) + conn.commit() + return ChallengeStore(db) + + +def manager_for(store): + # Execute the actual thin wrapper without importing optional GPU/HF modules, + # matching the lightweight method harness used by test_generator_scores. + source = Path(__file__).parents[1] / "gas/cache/content_manager.py" + tree = ast.parse(source.read_text()) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "ContentManager") + method = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "get_verification_stats_last_n_hours") + namespace = {"bt": SimpleNamespace(logging=Mock())} + module = ast.Module(body=[ast.ImportFrom(module="__future__", names=[ast.alias(name="annotations")], level=0), method], type_ignores=[]) + exec(compile(ast.fix_missing_locations(module), str(source), "exec"), namespace) + harness = type("ContentManagerHarness", (), {method.name: namespace[method.name]}) + manager = harness() + manager.challenges = store + return manager + + +@pytest.mark.parametrize("kwargs", [{}, {"limit": None}]) +def test_default_and_none_include_every_terminal_outcome_in_window(store, kwargs): + outcomes = store.get_outcomes_last_n_hours(24, **kwargs) + assert len(outcomes) == 1105 + assert outcomes[-1].task_id == "at-cutoff" + assert all(outcome.updated_at >= NOW - 86400 for outcome in outcomes) + assert {"too-old", "no-answer", "pending", "stored"}.isdisjoint( + outcome.task_id for outcome in outcomes + ) + stats = store.get_outcome_stats_last_n_hours(24, **kwargs) + assert stats["busy"]["video_verified"] == 1100 + assert stats["target"]["image_verified"] == 1 + assert stats["target"]["image_failed"] == 1 + assert stats["target"]["video_verified"] == 1 + assert stats["target"]["video_failed"] == 1 + assert stats["target"]["image_pass_rate"] == 0.5 + assert stats["target"]["video_pass_rate"] == 0.5 + assert stats["boundary"]["image_verified"] == 1 + + +@pytest.mark.parametrize("kwargs", [{}, {"limit": None}]) +def test_reward_facing_wrapper_keeps_older_qualified_images_payable(store, kwargs): + manager = manager_for(store) + stats = manager.get_verification_stats_last_n_hours(lookback_hours=24, **kwargs) + assert stats["target"]["total_evaluated"] == 4 + base, _ = get_generator_base_rewards(stats) + paid = combine_generator_rewards( + base, {200: GeneratorQualification(qualified_image=True, qualified_video=False)}, + ) + assert paid[200] == pytest.approx(0.3 * 0.5) + # Reproduce the former cap: all of this miner's eligible work disappears. + truncated = manager.get_verification_stats_last_n_hours(lookback_hours=24, limit=1000) + assert "target" not in truncated + + +@pytest.mark.parametrize("limit", [0, 1, 1000]) +def test_explicit_limits_are_preserved(store, limit): + outcomes = store.get_outcomes_last_n_hours(24, limit=limit) + assert len(outcomes) == limit + if limit: + assert outcomes[0].task_id == "busy-0" + stats = manager_for(store).get_verification_stats_last_n_hours(24, limit=limit) + assert sum(s["total_evaluated"] for s in stats.values()) == limit + + +def test_shorter_lookback_still_excludes_older_work(store): + stats = manager_for(store).get_verification_stats_last_n_hours(lookback_hours=2) + assert set(stats) == {"busy"} + assert stats["busy"]["total_verified"] == 1100 From 24134db95b4e184c888dc02e7ed76acbae2c47f8 Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 19 Sep 2026 16:29:09 +0000 Subject: [PATCH 2/2] chore: bump subnet version to 5.0.8 --- VERSION | 2 +- gas/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c20c645d..51e67ba4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.6 +5.0.8 diff --git a/gas/__init__.py b/gas/__init__.py index 6bbd1085..e009b976 100644 --- a/gas/__init__.py +++ b/gas/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.0.6" +__version__ = "5.0.8" version_split = __version__.split(".") __spec_version__ = (