From 828a9f839055bdd0fed0a8f120f1c6b3de2fb749 Mon Sep 17 00:00:00 2001 From: Deep Shah Date: Thu, 10 Sep 2026 21:33:10 +0000 Subject: [PATCH 1/3] docs(instance-preferences): resolve the example image from a candidate type The training notebook and the training guide resolved the PyTorch image with instance_type="ml.m5.xlarge", which selects the CPU image, and then paired it with GPU candidates (ml.p5.48xlarge, ml.p4d.24xlarge). A reader copying either example would run a GPU job on the CPU image. Both now resolve the image from one of the listed candidates (all three GPU candidates map to the same pytorch-training:2.0.0-gpu-py310 image), and state the rule the feature implies: the image is fixed at submission time while the instance type is not, so every candidate must be able to run it. The processing notebook and guide are updated the same way for consistency; the scikit-learn image is CPU-only and all candidates are m5 types, so their behaviour was already correct. --- docs/ml_ops/index.rst | 2 +- docs/training/index.rst | 14 +++++++++++++- .../v3-processing-instance-preferences.ipynb | 2 +- .../instance-preferences-example.ipynb | 8 ++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/ml_ops/index.rst b/docs/ml_ops/index.rst index 5fd4d6cbbe..ad603142c8 100644 --- a/docs/ml_ops/index.rst +++ b/docs/ml_ops/index.rst @@ -333,7 +333,7 @@ Run data preprocessing with ``ScriptProcessor`` (sklearn) or ``FrameworkProcesso processor.run(job_name="instance-prefs-processing") -Up to 5 candidates are allowed, each instance type at most once, and exactly one is selected; the list is mutually exclusive with ``instance_type``. Counts use exactly one of two modes — a top-level ``instance_count`` shared by whichever candidate wins, or an ``InstanceCount`` on every candidate — and mixed, partial, or omitted counts are rejected. Selection is based on capacity, not on workload fit, so list only types the job can genuinely run on. The winner is reported as ``SelectedInstanceType`` / ``SelectedInstanceCount`` on the job's ``ClusterConfig``, and billing is for that type and count. Supported on ``Processor``, ``ScriptProcessor``, ``PySparkProcessor``, and ``SparkJarProcessor``; training plans are training-only and do not apply to processing. +Up to 5 candidates are allowed, each instance type at most once, and exactly one is selected; the list is mutually exclusive with ``instance_type``. Counts use exactly one of two modes — a top-level ``instance_count`` shared by whichever candidate wins, or an ``InstanceCount`` on every candidate — and mixed, partial, or omitted counts are rejected. Selection is based on capacity, not on workload fit, so list only types that can run the job's ``image_uri`` (the image is fixed at submission time; the instance type is not). The winner is reported as ``SelectedInstanceType`` / ``SelectedInstanceCount`` on the job's ``ClusterConfig``, and billing is for that type and count. Supported on ``Processor``, ``ScriptProcessor``, ``PySparkProcessor``, and ``SparkJarProcessor``; training plans are training-only and do not apply to processing. :doc:`Instance Preferences example <../v3-examples/ml-ops-examples/v3-processing-instance-preferences>` diff --git a/docs/training/index.rst b/docs/training/index.rst index 858afbe321..56995d8200 100644 --- a/docs/training/index.rst +++ b/docs/training/index.rst @@ -398,10 +398,22 @@ Provide an ordered list of candidate instance types and the platform launches th .. code-block:: python + from sagemaker.core import image_uris from sagemaker.train.model_trainer import ModelTrainer from sagemaker.core.training.configs import Compute, SourceCode from sagemaker.core.shapes import InstancePreference + # The image is fixed at submission time while the instance type is not: + # resolve it from one of the candidates and list only types that can run it. + gpu_training_image = image_uris.retrieve( + framework="pytorch", + region=region, + version="2.0.0", + py_version="py310", + instance_type="ml.p5.48xlarge", + image_scope="training", + ) + compute = Compute( instance_preferences=[ InstancePreference(instance_type="ml.p5.48xlarge"), @@ -412,7 +424,7 @@ Provide an ordered list of candidate instance types and the platform launches th ) model_trainer = ModelTrainer( - training_image=training_image, + training_image=gpu_training_image, source_code=SourceCode(source_dir="./source", entry_script="train.py"), compute=compute, base_job_name="instance-preferences-training", diff --git a/v3-examples/ml-ops-examples/v3-processing-instance-preferences.ipynb b/v3-examples/ml-ops-examples/v3-processing-instance-preferences.ipynb index ec5ae9462d..15f31eb023 100644 --- a/v3-examples/ml-ops-examples/v3-processing-instance-preferences.ipynb +++ b/v3-examples/ml-ops-examples/v3-processing-instance-preferences.ipynb @@ -39,7 +39,7 @@ " framework=\"sklearn\",\n", " region=region,\n", " version=\"1.2-1\",\n", - " instance_type=\"ml.m5.xlarge\",\n", + " instance_type=\"ml.m5.4xlarge\", # a Step 2 candidate; every candidate must be able to run the image\n", " image_scope=\"training\",\n", ")\n" ] diff --git a/v3-examples/training-examples/instance-preferences-example.ipynb b/v3-examples/training-examples/instance-preferences-example.ipynb index fd5d67c5b7..c0b5063b2b 100644 --- a/v3-examples/training-examples/instance-preferences-example.ipynb +++ b/v3-examples/training-examples/instance-preferences-example.ipynb @@ -17,7 +17,11 @@ "source": [ "## Step 1: Setup Session\n", "\n", - "Initialize the SageMaker session, execution role, and a training image.\n" + "Initialize the SageMaker session, execution role, and a training image.\n", + "\n", + "The image is fixed at submission time while the instance type is not, so resolve it\n", + "from one of the candidate types you will list in Step 2 and make sure every candidate\n", + "can run it. Here both candidates are GPU types and share the same GPU image.\n" ] }, { @@ -42,7 +46,7 @@ " region=region,\n", " version=\"2.0.0\",\n", " py_version=\"py310\",\n", - " instance_type=\"ml.m5.xlarge\",\n", + " instance_type=\"ml.p5.48xlarge\", # a Step 2 candidate; ml.p4d.24xlarge resolves to the same image\n", " image_scope=\"training\",\n", ")\n" ] From 851ef17f1c89b2ba210ea1b853a9a5c562c016d4 Mon Sep 17 00:00:00 2001 From: Deep Shah Date: Thu, 10 Sep 2026 21:52:08 +0000 Subject: [PATCH 2/3] test(instance-preferences): decouple the PipeVar override test from the packaged service model TestInstancePreferencesPipeVarOverrides built a ShapesExtractor from the real service JSON, which lives under sample/ in the source tree and is not part of the installed package. Under tox the extractor is imported from site-packages, the file is absent, and all six cases errored at setup with FileNotFoundError. The test now injects a minimal in-memory model that mirrors the real member -> shape wiring for the six count members, so it exercises exactly the PIPE_VAR_OVERRIDES lookup and nothing else. A control shape with the same integer type and no override asserts the widening is targeted rather than blanket. Removing any one override still fails exactly that member's case. --- .../tests/unit/tools/test_shapes_extractor.py | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/sagemaker-core/tests/unit/tools/test_shapes_extractor.py b/sagemaker-core/tests/unit/tools/test_shapes_extractor.py index 54fdcbf2e6..5920b13ff6 100644 --- a/sagemaker-core/tests/unit/tools/test_shapes_extractor.py +++ b/sagemaker-core/tests/unit/tools/test_shapes_extractor.py @@ -350,7 +350,54 @@ class TestInstancePreferencesPipeVarOverrides: """The IntPipeVar annotations on instance-preferences count members come from PIPE_VAR_OVERRIDES, not the service model. If an override is dropped, codegen silently narrows the member back to int and pipeline variables stop being - accepted -- so assert the generated type directly.""" + accepted -- so assert the generated type directly. + + Uses a minimal in-memory model mirroring the real member -> shape wiring; + the packaged service JSON is not available in every test environment.""" + + _COUNT = {"type": "integer", "min": 1} + _STRING = {"type": "string"} + _MODEL = { + "TrainingInstanceCount": _COUNT, + "ProcessingInstanceCount": _COUNT, + "TrainingInstanceType": _STRING, + "ProcessingInstanceType": _STRING, + "ResourceConfig": { + "type": "structure", + "members": { + "InstanceType": {"shape": "TrainingInstanceType"}, + "InstanceCount": {"shape": "TrainingInstanceCount"}, + "SelectedInstanceCount": {"shape": "TrainingInstanceCount"}, + }, + }, + "InstancePreference": { + "type": "structure", + "members": { + "InstanceType": {"shape": "TrainingInstanceType"}, + "InstanceCount": {"shape": "TrainingInstanceCount"}, + }, + }, + "ProcessingClusterConfig": { + "type": "structure", + "members": { + "InstanceType": {"shape": "ProcessingInstanceType"}, + "InstanceCount": {"shape": "ProcessingInstanceCount"}, + "SelectedInstanceCount": {"shape": "ProcessingInstanceCount"}, + }, + }, + "ProcessingInstancePreference": { + "type": "structure", + "members": { + "InstanceType": {"shape": "ProcessingInstanceType"}, + "InstanceCount": {"shape": "ProcessingInstanceCount"}, + }, + }, + # Control: same integer shape, no override registered -> must stay int. + "UnrelatedConfig": { + "type": "structure", + "members": {"InstanceCount": {"shape": "TrainingInstanceCount"}}, + }, + } @pytest.fixture def extractor(self, tmp_path): @@ -363,7 +410,7 @@ def extractor(self, tmp_path): str(tmp_path / "shape_dag.py"), ), ): - return ShapesExtractor() + return ShapesExtractor(combined_shapes=self._MODEL) @pytest.mark.parametrize( "shape, member", @@ -383,3 +430,8 @@ def test_count_members_generate_as_int_pipe_var(self, extractor, shape, member): f"{shape}.{member} generated as {members[member]!r}; " "expected IntPipeVar via PIPE_VAR_OVERRIDES" ) + + def test_override_is_targeted_not_blanket(self, extractor): + members = extractor.generate_shape_members("UnrelatedConfig") + assert "IntPipeVar" not in members["instance_count"] + assert "int" in members["instance_count"] From fd2c93ae5f967964d48ebe22a31a37f876158281 Mon Sep 17 00:00:00 2001 From: Deep Shah Date: Thu, 10 Sep 2026 21:55:57 +0000 Subject: [PATCH 3/3] docs(instance-preferences): list the shallow test in the suite README; define processing_image in the guide The shallow suite README maps each shallow file to its deep counterpart; the instance-preferences pair was missing. The processing guide's instance-preferences snippet referenced processing_image without defining it on the page; it now resolves the image from one of the candidates, matching the training guide. --- docs/ml_ops/index.rst | 6 ++++++ sagemaker-train/tests/integ/train/shallow/README.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/docs/ml_ops/index.rst b/docs/ml_ops/index.rst index ad603142c8..b9f83e2284 100644 --- a/docs/ml_ops/index.rst +++ b/docs/ml_ops/index.rst @@ -321,8 +321,14 @@ Run data preprocessing with ``ScriptProcessor`` (sklearn) or ``FrameworkProcesso .. code-block:: python + from sagemaker.core import image_uris from sagemaker.core.processing import Processor + # Resolve the image from one of the candidates; every candidate must be able to run it. + processing_image = image_uris.retrieve( + framework="sklearn", region=region, version="1.2-1", instance_type="ml.m5.4xlarge" + ) + processor = Processor( role=role, image_uri=processing_image, volume_size_in_gb=100, instance_preferences=[ diff --git a/sagemaker-train/tests/integ/train/shallow/README.md b/sagemaker-train/tests/integ/train/shallow/README.md index 4a84bc0a79..2a246a1820 100644 --- a/sagemaker-train/tests/integ/train/shallow/README.md +++ b/sagemaker-train/tests/integ/train/shallow/README.md @@ -75,6 +75,7 @@ of any deep test is easy to find: | Shallow file | Deep counterpart | |---|---| | `test_model_trainer.py` | `test_model_trainer.py` | +| `test_instance_preferences.py` | `test_instance_preferences.py` | | `test_sft_trainer.py` | `test_sft_trainer_integration.py` | | `test_dpo_trainer.py` | `test_dpo_trainer_integration.py` | | `test_rlvr_trainer.py` | `test_rlvr_trainer_integration.py` | @@ -108,6 +109,7 @@ below accounts for all of them. | Deep test | Shallow equivalent | |---|---| | `test_model_trainer.py` — 8 tests (tar source, py/sh entry, MPI, torchrun, HP json/yaml, custom driver) | `test_model_trainer.py` — `TestSourceCodePackaging`, `TestPayloadShaping`, `TestComputeConfiguration` | +| `test_instance_preferences.py::test_instance_preferences_select_a_winner_and_complete` (runs to a selected winner and completion) | `test_instance_preferences.py` — `TestInstancePreferencesAccepted` (submit, Describe echo, stop) + `TestInstancePreferencesRejected` | | `test_sft_trainer_integration.py::test_sft_trainer_lora_complete_workflow` | `test_minimal_request_is_accepted` + `test_mlflow_resource_arn` | | `::test_sft_trainer_with_validation_dataset` | `test_with_validation_dataset` | | `::test_sft_trainer_lora_with_sequence_length` | `test_sft_trainer.py::test_sequence_length_is_accepted` |