diff --git a/src/spindle/configs/gpt_oss_20b_lora_64k.py b/src/spindle/configs/gpt_oss_20b_lora_64k.py index 73e55f0..270f05f 100644 --- a/src/spindle/configs/gpt_oss_20b_lora_64k.py +++ b/src/spindle/configs/gpt_oss_20b_lora_64k.py @@ -4,6 +4,8 @@ class Config(BaseConfig): name = "gpt-oss-20b-lora-64k" model = "openai/gpt-oss-20b" + # SGLang LoRA MoE needs unquantized experts; this is the MXFP4 release upcast to BF16. + model_weights = "lmsys/gpt-oss-20b-bf16" max_context_length = 65536 trainer_gpu = "H200" trainer_gpus_per_node = 8 @@ -52,6 +54,7 @@ class Config(BaseConfig): "max_loaded_loras": 64, "max_loras_per_batch": 8, "schedule_policy": "lpm", + "moe_runner_backend": "triton", } diff --git a/src/spindle/configuration.py b/src/spindle/configuration.py index 338dbbf..207deb0 100644 --- a/src/spindle/configuration.py +++ b/src/spindle/configuration.py @@ -30,6 +30,8 @@ class BaseConfig: name = "" model = "" + # Hugging Face repo for base weights; defaults to ``model``. + model_weights = "" max_context_length = 16384 parameterization = "lora" backend = "miles" diff --git a/src/spindle/control_plane/http.py b/src/spindle/control_plane/http.py index 9413ec5..4f09d18 100644 --- a/src/spindle/control_plane/http.py +++ b/src/spindle/control_plane/http.py @@ -273,6 +273,7 @@ async def list_deployments(): "name": d.name, "definition_id": d.definition_id, "base_model": d.model, + "weights_repo": d.weights_repo, "parameterization": d.parameterization, "max_context_length": d.max_context_length, } diff --git a/src/spindle/deployments.py b/src/spindle/deployments.py index 4bd0d6d..0ef638e 100644 --- a/src/spindle/deployments.py +++ b/src/spindle/deployments.py @@ -42,7 +42,7 @@ def create(cls, recipe: BaseConfig) -> DeploymentConfig: """Copy the recipe and attach backend settings.""" pinned = BaseConfig(**_jsonable(vars(recipe))) trainer_settings, inference_settings = resolve_backend_settings( - pinned, f"/assets/{pinned.model}" + pinned, f"/assets/{pinned.model_weights or pinned.model}" ) return cls( recipe=pinned, @@ -78,9 +78,13 @@ def trainer_app_name(self) -> str: def inference_app_name(self) -> str: return f"spindle-inference-{self.recipe.name}" + @property + def weights_repo(self) -> str: + return self.recipe.model_weights or self.recipe.model + @property def asset_path(self) -> str: - return f"/assets/{self.recipe.model}" + return f"/assets/{self.weights_repo}" @property def rollout_tensor_parallel_size(self) -> int: @@ -94,6 +98,7 @@ def trainer_identity(self) -> tuple: return ( recipe.name, recipe.model, + recipe.model_weights, recipe.parameterization, recipe.max_context_length, recipe.backend, @@ -117,6 +122,7 @@ def inference_identity(self) -> tuple: return ( recipe.name, recipe.model, + recipe.model_weights, recipe.parameterization, recipe.max_context_length, recipe.sglang_cfg, diff --git a/src/spindle/providers/modal/app.py b/src/spindle/providers/modal/app.py index b4097c7..010a694 100644 --- a/src/spindle/providers/modal/app.py +++ b/src/spindle/providers/modal/app.py @@ -181,7 +181,7 @@ def prepare_model_assets(definition_id: str) -> None: ): raise ValueError(f"invalid model asset path: {checkpoint}") snapshot_download( - repo_id=definition.model, + repo_id=definition.weights_repo, local_dir=checkpoint, ) model_assets.commit() diff --git a/tests/control_plane/test_http.py b/tests/control_plane/test_http.py index 4b49992..779298b 100644 --- a/tests/control_plane/test_http.py +++ b/tests/control_plane/test_http.py @@ -19,6 +19,7 @@ definition_id=DEFINITION, name=DEFINITION, model=BASE_MODEL, + weights_repo=BASE_MODEL, parameterization="lora", max_context_length=16_384, ), @@ -26,6 +27,7 @@ definition_id=f"{DEFINITION}_full", name=f"{DEFINITION}_full", model=BASE_MODEL, + weights_repo=BASE_MODEL, parameterization="full", max_context_length=65_536, ), @@ -471,6 +473,7 @@ async def run(): definition_id="isolated", name="isolated", model=BASE_MODEL, + weights_repo=f"{BASE_MODEL}-bf16", parameterization="lora", max_context_length=16384, ) @@ -489,6 +492,8 @@ async def run(): assert [row["definition_id"] for row in listed] == [ d.definition_id for d in (*DEFINITIONS, explicit) ] + assert listed[-1]["base_model"] == BASE_MODEL + assert listed[-1]["weights_repo"] == f"{BASE_MODEL}-bf16" session = ( await client.post( "/api/v1/create_session", json={"tags": [], "sdk_version": "0.5.0"} diff --git a/tests/test_deployments.py b/tests/test_deployments.py index d89db7c..8ca467d 100644 --- a/tests/test_deployments.py +++ b/tests/test_deployments.py @@ -104,6 +104,16 @@ def test_asset_paths_follow_the_model(): assert a.asset_path != b.asset_path +def test_model_weights_override_assets_but_not_the_served_model(): + row = resolved(recipe(model_weights="org/model-bf16")) + assert row.model == row.recipe.model + assert row.weights_repo == "org/model-bf16" + assert row.asset_path == "/assets/org/model-bf16" + assert row.trainer_settings["miles"]["hf_checkpoint"] == row.asset_path + assert row.trainer_identity() != resolved().trainer_identity() + assert row.inference_identity() != resolved().inference_identity() + + def test_routing_uses_deployment_order(): small = resolved() large = resolved(recipe("qwen35-9b-lora-64k"))