Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,19 @@ after its first published distribution.
failures continue to exit `1` with a traceback.
- `ser doctor` and the training preflight now include training readiness checks.
- README Python API guidance now directs workflow users to `ser.api`.
- scikit-learn is now required at `>=1.9.0,<1.10.0`. scikit-learn 1.8 fixed how
`StratifiedGroupKFold(shuffle=True)` assigns groups to folds, so grouped
cross-validation folds, and the quality-gate metrics computed from them,
shift once relative to earlier runs with the same seed. Compare gate results
across this upgrade as a new baseline rather than as a regression.
- When grouped cross-validation is infeasible (more folds than speakers), the
quality gate now logs a WARNING with the requested fold count and the speaker
and sample counts before falling back to a single grouped holdout. The
fallback behavior and the reported `fold_strategy` are unchanged.
- Model artifacts now record the scikit-learn version they were saved with.
Loading an artifact pickled under a different scikit-learn version logs one
WARNING with re-train or re-save guidance instead of silently suppressing the
version warning; other unpickling warnings stay suppressed.

### Fixed

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ dependencies = [
"numpy>=2.0.0,<3.0.0; python_version >= '3.13'",
"openai-whisper>=20250625,<20260000; python_version < '3.13' or (sys_platform != 'darwin' or platform_machine != 'x86_64')",
"ffmpeg-python>=0.2.0,<0.3.0",
"scikit-learn>=1.3.2,<1.8.0",
"scikit-learn>=1.9.0,<1.10.0",
"soundfile>=0.14.0,<1.0.0",
"python-dotenv>=1.2.2,<2.0.0",
"PyYAML>=6.0,<7.0",
Expand Down
4 changes: 3 additions & 1 deletion ser/_internal/models/artifact_envelope.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from typing import NamedTuple, cast

import sklearn
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import Pipeline

Expand Down Expand Up @@ -71,7 +72,7 @@ def build_model_artifact_metadata(
seed: int | None = None,
evaluation_summary: dict[str, object] | None = None,
) -> dict[str, object]:
"""Builds normalized artifact metadata for persisted model envelopes."""
"""Builds normalized artifact metadata, stamping the scikit-learn version used to save."""
return build_v3_artifact_metadata(
artifact_version=MODEL_ARTIFACT_VERSION,
artifact_schema_version=ARTIFACT_SCHEMA_VERSION,
Expand All @@ -95,6 +96,7 @@ def build_model_artifact_metadata(
sampling_policy=sampling_policy,
seed=seed,
evaluation_summary=evaluation_summary,
sklearn_version=sklearn.__version__,
)


Expand Down
50 changes: 46 additions & 4 deletions ser/_internal/models/artifact_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,19 @@
import logging
import pickle
import warnings
from collections.abc import Callable
from collections.abc import Callable, Sequence
from importlib import import_module
from pathlib import Path
from typing import Literal, Protocol, TypeVar, cast

from sklearn.exceptions import InconsistentVersionWarning

type ArtifactFormat = Literal["pickle", "skops"]

_LOGGER = logging.getLogger(__name__)
# (artifact path, saved version, running version) already reported in this process.
_REPORTED_VERSION_MISMATCHES: set[tuple[str, str, str]] = set()


class ModelCandidateLike(Protocol):
"""Structural contract for model artifact candidates."""
Expand Down Expand Up @@ -160,16 +166,52 @@ def load_secure_model_artifact(
return loaded_model_factory(payload, feature_size)


def report_sklearn_version_mismatch(
candidate_path: Path,
caught: Sequence[warnings.WarningMessage],
*,
logger: logging.Logger,
) -> None:
"""Logs one WARNING when an artifact was pickled under another scikit-learn version."""
mismatches = [
cast(InconsistentVersionWarning, record.message)
for record in caught
if isinstance(record.message, InconsistentVersionWarning)
]
if not mismatches:
return
saved_version = mismatches[0].original_sklearn_version
running_version = mismatches[0].current_sklearn_version
key = (str(candidate_path), saved_version, running_version)
if key in _REPORTED_VERSION_MISMATCHES:
return
_REPORTED_VERSION_MISMATCHES.add(key)
estimators = sorted({mismatch.estimator_name for mismatch in mismatches})
logger.warning(
"Model artifact %s was saved with scikit-learn %s and is loaded with %s "
"(estimators: %s). Predictions are not guaranteed to match; re-train with "
"`ser --train` or re-save the artifact under scikit-learn %s to remove this warning.",
candidate_path,
saved_version,
running_version,
", ".join(estimators),
running_version,
)


def load_pickle_model_artifact(
*,
candidate_path: Path,
deserialize_payload: Callable[[object], PickleLoadedModelT],
logger: logging.Logger | None = None,
) -> PickleLoadedModelT:
"""Loads and deserializes one compatibility pickle artifact."""
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
"""Loads one compatibility pickle artifact, surfacing only version-skew warnings."""
# Recording keeps unpickling noise out of user output; version skew is reported explicitly.
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with candidate_path.open("rb") as model_fh:
payload = pickle.load(model_fh)
report_sklearn_version_mismatch(candidate_path, caught, logger=logger or _LOGGER)
return deserialize_payload(payload)


Expand Down
10 changes: 10 additions & 0 deletions ser/_internal/models/artifact_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ def normalize_v2_artifact_metadata(
if not isinstance(torch_dtype, str) or not torch_dtype.strip():
raise ValueError("Model artifact metadata contains invalid 'torch_dtype' value.")
normalized["torch_dtype"] = torch_dtype.strip().lower()
sklearn_version = metadata.get("sklearn_version")
if sklearn_version is not None:
if not isinstance(sklearn_version, str) or not sklearn_version.strip():
raise ValueError("Model artifact metadata contains invalid 'sklearn_version' value.")
normalized["sklearn_version"] = sklearn_version.strip()
return normalized


Expand All @@ -159,6 +164,7 @@ def build_v2_artifact_metadata(
torch_device: str | None,
torch_dtype: str | None,
provenance: dict[str, object] | None,
sklearn_version: str | None = None,
) -> dict[str, object]:
"""Builds normalized v2 artifact metadata for persisted model envelopes."""
resolved_feature_dim = feature_vector_size if feature_dim is None else feature_dim
Expand Down Expand Up @@ -193,6 +199,8 @@ def build_v2_artifact_metadata(
if not resolved_torch_dtype:
raise ValueError("Model artifact metadata contains invalid 'torch_dtype' value.")
payload["torch_dtype"] = resolved_torch_dtype
if sklearn_version is not None:
payload["sklearn_version"] = sklearn_version
return normalize_v2_artifact_metadata(
payload,
artifact_version=artifact_version,
Expand Down Expand Up @@ -280,6 +288,7 @@ def build_v3_artifact_metadata(
sampling_policy: dict[str, object] | None,
seed: int | None,
evaluation_summary: dict[str, object] | None,
sklearn_version: str | None = None,
) -> dict[str, object]:
"""Builds v3 metadata with reproducible dataset and training provenance."""
payload = build_v2_artifact_metadata(
Expand All @@ -298,6 +307,7 @@ def build_v3_artifact_metadata(
torch_device=torch_device,
torch_dtype=torch_dtype,
provenance=provenance,
sklearn_version=sklearn_version,
)
optional_fields: dict[str, object | None] = {
"recipe_digest": recipe_digest,
Expand Down
14 changes: 13 additions & 1 deletion ser/_internal/runtime/profile_quality_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,13 @@
)
from ser._internal.train.eval import grouped_train_test_split, speaker_independent_cv
from ser._internal.train.metrics import compute_ser_metrics
from ser._internal.utils.logger import get_logger
from ser._internal.utils.segment_canonicalization import canonicalize_segments
from ser.config import AppConfig, reload_settings
from ser.runtime.contracts import InferenceRequest

logger = get_logger(__name__)

type IndexArray = NDArray[np.int64]
type FoldIndices = tuple[IndexArray, IndexArray]
type LabeledAudioSample = tuple[str, str]
Expand Down Expand Up @@ -262,7 +265,16 @@ def _build_grouped_folds(
random_state=random_state,
)
return "stratified_group_kfold", folds
except ValueError:
except ValueError as error:
logger.warning(
"Grouped CV fell back to a single grouped holdout: requested n_splits=%d, "
"distinct speakers=%d, samples=%d, holdout test_size=%.2f, folds=1 (%s).",
n_splits,
len(set(speaker_ids)),
len(labels),
fallback_test_size,
error,
)
grouped_split = grouped_train_test_split(
features,
labels,
Expand Down
18 changes: 16 additions & 2 deletions tests/suites/integration/runtime/test_profile_quality_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import logging
from collections.abc import Sequence
from dataclasses import replace
from pathlib import Path
Expand Down Expand Up @@ -144,9 +145,12 @@ def test_profile_quality_gate_raises_when_speaker_metadata_is_missing() -> None:
)


def test_profile_quality_gate_falls_back_to_grouped_holdout() -> None:
"""Infeasible fold count should fall back to deterministic grouped holdout."""
def test_profile_quality_gate_falls_back_to_grouped_holdout(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Infeasible fold count should fall back to grouped holdout and say so loudly."""
samples = _build_samples()
caplog.set_level(logging.WARNING, logger=gate_module.__name__)

report = evaluate_profile_quality_gate(
samples=samples,
Expand All @@ -161,6 +165,16 @@ def test_profile_quality_gate_falls_back_to_grouped_holdout() -> None:
assert report.fold_strategy == "group_shuffle_holdout"
assert report.folds_evaluated == 1
assert report.grouped_evaluation.fold_speaker_overlap_counts == (0,)
fallback_warnings = [
record.getMessage()
for record in caplog.records
if record.levelno == logging.WARNING and "single grouped holdout" in record.getMessage()
]
assert len(fallback_warnings) == 1
speaker_count = len(set(gate_module._resolve_speaker_ids(samples)))
assert "requested n_splits=8" in fallback_warnings[0]
assert f"distinct speakers={speaker_count}" in fallback_warnings[0]
assert "folds=1" in fallback_warnings[0]


def test_enforce_quality_gate_noop_when_not_required() -> None:
Expand Down
81 changes: 81 additions & 0 deletions tests/suites/unit/models/test_artifact_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,18 @@

import logging
import pickle
import warnings
from dataclasses import dataclass
from pathlib import Path
from typing import Literal

import numpy as np
import pytest
import sklearn
import sklearn.base
from sklearn.preprocessing import StandardScaler

import ser._internal.models.artifact_loading as artifact_loading
from ser._internal.models.artifact_loading import (
load_model_with_resolution,
load_pickle_model_artifact,
Expand Down Expand Up @@ -238,3 +244,78 @@ def _load_secure(
assert resolved_model is expected
assert captured["candidate"] == resolved_candidate
assert captured["settings"] is settings


class _NoisyPayload:
"""Unpickles with an unrelated warning, standing in for third-party load noise."""

def __setstate__(self, state: dict[str, object]) -> None:
warnings.warn("unrelated unpickling noise", UserWarning, stacklevel=1)
self.__dict__.update(state)


def _scaler_from_payload(payload: object) -> StandardScaler:
if not isinstance(payload, dict) or not isinstance(payload.get("model"), StandardScaler):
raise AssertionError("Expected a pickled StandardScaler payload.")
return payload["model"]


def _write_skewed_scaler(path: Path, monkeypatch: pytest.MonkeyPatch, version: str) -> None:
scaler = StandardScaler().fit(np.arange(6, dtype=np.float64).reshape(3, 2))
with monkeypatch.context() as patch:
patch.setattr(sklearn.base, "__version__", version)
path.write_bytes(pickle.dumps({"model": scaler}, protocol=pickle.HIGHEST_PROTOCOL))


def test_pickle_loader_reports_sklearn_version_skew_once(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""Version skew is logged once per artifact with re-save guidance."""
monkeypatch.setattr(artifact_loading, "_REPORTED_VERSION_MISMATCHES", set())
candidate_path = tmp_path / "ser_model.pkl"
_write_skewed_scaler(candidate_path, monkeypatch, "0.0.1")
logger = logging.getLogger("tests.artifact_loading.version_skew")
caplog.set_level(logging.WARNING, logger=logger.name)

loads = [
load_pickle_model_artifact(
candidate_path=candidate_path,
deserialize_payload=_scaler_from_payload,
logger=logger,
)
for _ in range(2)
]

assert all(isinstance(loaded, StandardScaler) for loaded in loads)
messages = [record.getMessage() for record in caplog.records]
assert len(messages) == 1
assert f"saved with scikit-learn 0.0.1 and is loaded with {sklearn.__version__}" in (
messages[0]
)
assert "StandardScaler" in messages[0]
assert "re-save the artifact" in messages[0]


def test_pickle_loader_keeps_matching_versions_and_other_warnings_quiet(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""Same-version artifacts log nothing and unrelated unpickling warnings stay contained."""
monkeypatch.setattr(artifact_loading, "_REPORTED_VERSION_MISMATCHES", set())
matching_path = tmp_path / "matching.pkl"
_write_skewed_scaler(matching_path, monkeypatch, sklearn.__version__)
noisy_path = tmp_path / "noisy.pkl"
noisy_path.write_bytes(pickle.dumps(_NoisyPayload(), protocol=pickle.HIGHEST_PROTOCOL))
logger = logging.getLogger("tests.artifact_loading.quiet")
caplog.set_level(logging.DEBUG, logger=logger.name)

with warnings.catch_warnings(record=True) as escaped:
warnings.simplefilter("always")
for path in (matching_path, noisy_path):
load_pickle_model_artifact(
candidate_path=path,
deserialize_payload=lambda payload: payload,
logger=logger,
)

assert caplog.records == []
assert escaped == []
31 changes: 31 additions & 0 deletions tests/suites/unit/models/test_artifact_upgrade_path.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import pytest
import sklearn
from sklearn.neural_network import MLPClassifier

import ser._internal.models.artifact_envelope as artifact_envelope
Expand Down Expand Up @@ -229,3 +230,33 @@ def test_deserialize_rejects_legacy_raw_model_object() -> None:
match="versioned dictionary envelope",
):
artifact_envelope.deserialize_model_artifact(_build_classifier())


def test_new_artifacts_record_the_saving_sklearn_version() -> None:
"""Save time stamps the scikit-learn version so later loads can explain skew."""
artifact = artifact_envelope.build_model_artifact(
model=_build_classifier(),
feature_vector_size=193,
training_samples=32,
labels=["happy", "sad"],
)

loaded = artifact_envelope.deserialize_model_artifact(artifact)

assert loaded.artifact_metadata is not None
assert loaded.artifact_metadata["sklearn_version"] == sklearn.__version__


def test_deserialize_rejects_blank_sklearn_version() -> None:
artifact = artifact_envelope.build_model_artifact(
model=_build_classifier(),
feature_vector_size=193,
training_samples=32,
labels=["happy", "sad"],
)
metadata = artifact["metadata"]
assert isinstance(metadata, dict)
metadata["sklearn_version"] = " "

with pytest.raises(ValueError, match="sklearn_version"):
artifact_envelope.deserialize_model_artifact(artifact)
Loading
Loading