Serve gpt-oss LoRA from BF16 weights with the Triton MoE runner

src/spindle/configs/gpt_oss_20b_lora_64k.py

@@ -4,6 +4,8 @@
 class Config(BaseConfig):
     name = "gpt-oss-20b-lora-64k"
     model = "openai/gpt-oss-20b"
+    # SGLang LoRA MoE needs unquantized experts; this is the MXFP4 release upcast to BF16.
+    model_weights = "lmsys/gpt-oss-20b-bf16"
     max_context_length = 65536
     trainer_gpu = "H200"
     trainer_gpus_per_node = 8
@@ -52,6 +54,7 @@ class Config(BaseConfig):
         "max_loaded_loras": 64,
         "max_loras_per_batch": 8,
         "schedule_policy": "lpm",
+        "moe_runner_backend": "triton",
     }
 
 

src/spindle/configuration.py

@@ -30,6 +30,8 @@ class BaseConfig:
 
     name = ""
     model = ""
+    # Hugging Face repo for base weights; defaults to ``model``.
+    model_weights = ""
     max_context_length = 16384
     parameterization = "lora"
     backend = "miles"

src/spindle/deployments.py

@@ -42,7 +42,7 @@ def create(cls, recipe: BaseConfig) -> DeploymentConfig:
         """Copy the recipe and attach backend settings."""
         pinned = BaseConfig(**_jsonable(vars(recipe)))
         trainer_settings, inference_settings = resolve_backend_settings(
-            pinned, f"/assets/{pinned.model}"
+            pinned, f"/assets/{pinned.model_weights or pinned.model}"
         )
         return cls(
             recipe=pinned,
@@ -78,9 +78,13 @@ def trainer_app_name(self) -> str:
     def inference_app_name(self) -> str:
         return f"spindle-inference-{self.recipe.name}"
 
+    @property
+    def weights_repo(self) -> str:
+        return self.recipe.model_weights or self.recipe.model
+
     @property
     def asset_path(self) -> str:
-        return f"/assets/{self.recipe.model}"
+        return f"/assets/{self.weights_repo}"
 
     @property
     def rollout_tensor_parallel_size(self) -> int:
@@ -94,6 +98,7 @@ def trainer_identity(self) -> tuple:
         return (
             recipe.name,
             recipe.model,
+            recipe.model_weights,
             recipe.parameterization,
             recipe.max_context_length,
             recipe.backend,
@@ -117,6 +122,7 @@ def inference_identity(self) -> tuple:
         return (
             recipe.name,
             recipe.model,
+            recipe.model_weights,
             recipe.parameterization,
             recipe.max_context_length,
             recipe.sglang_cfg,

src/spindle/providers/modal/app.py

@@ -181,7 +181,7 @@ def prepare_model_assets(definition_id: str) -> None:
     ):
         raise ValueError(f"invalid model asset path: {checkpoint}")
     snapshot_download(
-        repo_id=definition.model,
+        repo_id=definition.weights_repo,
         local_dir=checkpoint,
     )
     model_assets.commit()

tests/test_deployments.py

@@ -104,6 +104,16 @@ def test_asset_paths_follow_the_model():
     assert a.asset_path != b.asset_path
 
 
+def test_model_weights_override_assets_but_not_the_served_model():
+    row = resolved(recipe(model_weights="org/model-bf16"))
+    assert row.model == row.recipe.model
+    assert row.weights_repo == "org/model-bf16"
+    assert row.asset_path == "/assets/org/model-bf16"
+    assert row.trainer_settings["miles"]["hf_checkpoint"] == row.asset_path
+    assert row.trainer_identity() != resolved().trainer_identity()
+    assert row.inference_identity() != resolved().inference_identity()
+
+
 def test_routing_uses_deployment_order():
     small = resolved()
     large = resolved(recipe("qwen35-9b-lora-64k"))