From 7e0303544af2edd062890c61a828fff4c48292b4 Mon Sep 17 00:00:00 2001 From: Thykof Date: Wed, 18 Nov 2026 19:19:27 +0100 Subject: [PATCH] Serving: a champion can prewarm its market data before registration; crypto-1h warms first A ChampionMiner fetches its minute history and the event calendar at each start, and refuses an asset's prompts until that asset is in. On a fresh volume that happens after registration, so the first prompts fail. ChampionMiner.prewarm() runs the same warm-up once, then one calendar refresh, with no wallet and no chain. The generated miner.py and entrypoint.sh take --prewarm for it. A miner started afterwards on that volume fetches only the partitions not yet settled and serves the stored calendar, so it answers almost at once. servable_assets() now lists the shortest horizon's assets first, so on every start the crypto-1h assets, prompted most often, are served first. Co-Authored-By: Claude Opus 5.5 --- synth_lib/serving/champion_miner.py | 39 ++++++++++++++------ synth_lib/serving/serve.py | 5 ++- synth_lib/serving/unpack_champion.py | 9 +++++ tests/serving/test_champion_miner_startup.py | 14 +++++++ tests/serving/test_champion_serving.py | 38 +++++++++++++++++++ 5 files changed, 91 insertions(+), 14 deletions(-) diff --git a/synth_lib/serving/champion_miner.py b/synth_lib/serving/champion_miner.py index cd4bfce..499c6b4 100644 --- a/synth_lib/serving/champion_miner.py +++ b/synth_lib/serving/champion_miner.py @@ -11,6 +11,9 @@ campaign. A calendar already stored on the volume, no older than `CALENDAR_MAX_AGE_SECONDS`, is served until the first refresh replaces it. +`prewarm()` fills the same stores and calendar once, with no wallet and no chain. A miner started +afterwards on that volume fetches only the partitions not yet settled, and serves the stored calendar. + No guards: an unservable asset, a data hole, or an exploding path crashes the request so it shows up in monitoring instead of silently degrading. @@ -27,6 +30,7 @@ import threading import time from datetime import timedelta +from pathlib import Path from typing import Callable import pandas as pd @@ -68,6 +72,21 @@ CALENDAR_WAIT_SECONDS = 10 * 60 +def refresh_calendar(root: Path) -> None: + """One update of the calendar: a source that fails keeps its rows, and the champion keeps the last view.""" + now = pd.Timestamp.now(tz="UTC") + today = now.date() + try: + calendar_build.update( + root, + today - timedelta(days=CALENDAR_HISTORY_DAYS), + today + timedelta(days=calendar_build.DAYS_AHEAD), + now, + ) + except Exception as exc: + logger.warning("calendar refresh failed: %s", exc) + + class ChampionMiner(Miner): """Serves one champion's simulate() over all venue-routed competition assets.""" @@ -80,6 +99,13 @@ def __init__(self, *args, **kwargs): raise TypeError("subclass must set simulate_fn (see synth_lib/serving/unpack_champion.py)") self._start(servable_assets()) + @classmethod + def prewarm(cls) -> None: + """One warm-up of every servable asset, then one calendar refresh, into MARKET_DATA_DIR.""" + warm_up(servable_assets(), cls.context_days) + refresh_calendar(MARKET_DATA_DIR) + logger.info("prewarm complete") + def _start(self, assets: list[str]) -> None: """Binds the stores and starts the background thread. Returns before any venue call, so the axon comes up while the data is still being fetched.""" @@ -128,18 +154,7 @@ def _refresh(self, assets: list[str], days: int) -> None: self._contexts.store_refreshed() def _refresh_calendar(self) -> None: - """One update of the calendar: a source that fails keeps its rows, and the champion keeps the last view.""" - now = pd.Timestamp.now(tz="UTC") - today = now.date() - try: - calendar_build.update( - self._calendar_root, - today - timedelta(days=CALENDAR_HISTORY_DAYS), - today + timedelta(days=calendar_build.DAYS_AHEAD), - now, - ) - except Exception as exc: - logger.warning("calendar refresh failed: %s", exc) + refresh_calendar(self._calendar_root) def _refresh_calendar_daily(self) -> None: while True: diff --git a/synth_lib/serving/serve.py b/synth_lib/serving/serve.py index 348e0b5..dd7f941 100644 --- a/synth_lib/serving/serve.py +++ b/synth_lib/serving/serve.py @@ -77,9 +77,10 @@ def venue_store(asset: str) -> MinutePriceStore: def servable_assets() -> list[str]: - """Every competition asset that has a live venue, in competition order.""" + """Every competition asset that has a live venue, the shortest horizon's first: the miner warms them up + in this order, and the 1h competition's assets are prompted most often.""" assets: list[str] = [] - for comp in ALL_COMPETITIONS: + for comp in sorted(ALL_COMPETITIONS, key=lambda comp: comp.time_length): for asset in comp.asset_list: if asset in assets: continue diff --git a/synth_lib/serving/unpack_champion.py b/synth_lib/serving/unpack_champion.py index 0d680a3..21e129d 100644 --- a/synth_lib/serving/unpack_champion.py +++ b/synth_lib/serving/unpack_champion.py @@ -59,6 +59,7 @@ from __future__ import annotations import logging +import sys import time from synth_lib.benchmark.generate_predictions import context_days_of @@ -76,6 +77,9 @@ class $class_name(ChampionMiner): if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + if sys.argv[1:] == ["--prewarm"]: + $class_name.prewarm() + sys.exit(0) with $class_name() as miner: while True: miner.print_info() @@ -102,6 +106,11 @@ class $class_name(ChampionMiner): export SYNTH_BENCHMARK_SEED="$${SYNTH_BENCHMARK_SEED:-0}" export PYTHONHASHSEED="$${PYTHONHASHSEED:-0}" +# --prewarm fills market_data/ once, with no wallet and no chain, then exits. +if [ "$${1:-}" = "--prewarm" ]; then + exec uv run python3.12 "$${SCRIPT_DIR}/miner.py" --prewarm +fi + exec uv run python3.12 "$${SCRIPT_DIR}/miner.py" \\ --subtensor.network "$$NETWORK" \\ --netuid "$$NETUID" \\ diff --git a/tests/serving/test_champion_miner_startup.py b/tests/serving/test_champion_miner_startup.py index afb13e7..9ad57cc 100644 --- a/tests/serving/test_champion_miner_startup.py +++ b/tests/serving/test_champion_miner_startup.py @@ -78,6 +78,20 @@ def test_an_asset_is_served_as_soon_as_its_own_warm_up_is_done(miner): assert _forward(miner, "XAU").simulation_output == ("store:XAU",) +def test_prewarm_fills_every_store_then_the_calendar_with_no_chain(monkeypatch, tmp_path): + calls: list[tuple] = [] + monkeypatch.setattr(cm, "MARKET_DATA_DIR", tmp_path / "market_data") + monkeypatch.setattr(cm, "servable_assets", lambda: ["BTC", "XAU"]) + monkeypatch.setattr(cm, "warm_up", lambda assets, context_days: calls.append(("warm_up", assets, context_days))) + monkeypatch.setattr(cm.calendar_build, "update", lambda root, start, end, now: calls.append(("calendar", root))) + + class FourteenDays(ChampionMiner): + context_days = 14 + + FourteenDays.prewarm() # a classmethod: Miner.__init__, its wallet and its chain never run + assert calls == [("warm_up", ["BTC", "XAU"], 14), ("calendar", tmp_path / "market_data")] + + def test_an_asset_with_no_live_venue_still_fails_loudly(miner): miner._start(["BTC"]) with pytest.raises(KeyError): diff --git a/tests/serving/test_champion_serving.py b/tests/serving/test_champion_serving.py index 77c0bf9..b2334df 100644 --- a/tests/serving/test_champion_serving.py +++ b/tests/serving/test_champion_serving.py @@ -12,10 +12,12 @@ import pytest from synth.simulation_input import SimulationInput # type: ignore[import-untyped] from synth.validator import response_validation_v2 # type: ignore[import-untyped] +from synth.validator.competition_config import ALL_COMPETITIONS # type: ignore[import-untyped] import synth_lib.serving.serve as serve from synth_lib.benchmark.generate_predictions import load_simulate from synth_lib.benchmark.verdict.run_verdict import DEFAULT_SEED, HASH_SEED_ENV, SEED_ENV, generation_env +from synth_lib.serving.champion_miner import ChampionMiner from synth_lib.serving.serve import ( ContextCache, build_context, @@ -271,6 +273,11 @@ def test_venue_routing_never_pyth(): assert "BTC" in servable_assets() +def test_the_1h_competitions_assets_warm_up_first(): + one_hour = min(ALL_COMPETITIONS, key=lambda comp: comp.time_length) + assert servable_assets()[: len(one_hour.asset_list)] == one_hour.asset_list + + # -- the unpacker, end to end on a synthetic bundle ------------------------------ @@ -340,6 +347,37 @@ def test_the_entrypoint_pins_the_verdicts_seeds(tmp_path): assert seeds == [expected[SEED_ENV], expected[HASH_SEED_ENV]] +@pytest.mark.parametrize("args, chain", [(["--prewarm"], False), ([], True)]) +def test_the_entrypoint_prewarms_without_the_chain(tmp_path, args, chain): + dest = unpack("test-camp", "fake", None, _make_bundle(tmp_path), tmp_path / "champions") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "uv").write_text('#!/bin/sh\necho "$@"\n') + (bin_dir / "uv").chmod(0o755) + command = subprocess.run( + ["bash", str(dest / "entrypoint.sh"), *args], + env={"PATH": f"{bin_dir}:{os.environ['PATH']}"}, + capture_output=True, + text=True, + check=True, + ).stdout.split() + assert command[:3] == ["run", "python3.12", str(dest / "miner.py")] + assert ("--netuid" in command) is chain and ("--prewarm" in command) is not chain + + +def test_the_generated_miner_prewarms_without_a_wallet(monkeypatch): + called: list[str] = [] + monkeypatch.setattr(ChampionMiner, "prewarm", classmethod(lambda cls: called.append(cls.__name__))) + modeling = ModuleType("modeling") + modeling.simulate = _recording([]) + monkeypatch.setitem(sys.modules, "modeling", modeling) + monkeypatch.setattr(sys, "argv", ["miner.py", "--prewarm"]) + source = MINER_TEMPLATE.substitute(name="test", campaign="c", leg="l", class_name="TestMiner") + with pytest.raises(SystemExit) as exit_info: + exec(compile(source, "miner.py", "exec"), {"__name__": "__main__"}) + assert exit_info.value.code == 0 and called == ["TestMiner"] + + def test_unpack_refuses_a_champion_that_fails_the_gate(tmp_path): results_dir = _make_bundle(tmp_path) leg_dir = results_dir / "test-camp" / "fake"