From cbdad3a612862f5eeb373c375c3eaeb2670f9c2f Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Tue, 10 Nov 2026 03:04:16 -0300 Subject: [PATCH] ci: build validation datasets with three speakers The training readiness contract requires a speaker-disjoint train/dev/test split with minimum per-class support in every partition. With the default split policy (test 0.25, dev 0.10, minimum 1 per class) GroupShuffleSplit needs three speakers: one held out for test, leaving two so the dev cut can still group by speaker. The builder default of two actors left the dev split empty, so every hosted lane failed readiness before training. Workflows now pass --actors explicitly; the builder default is left alone. A regression test builds each lane dataset and runs the real readiness contract. --- .../workflows/darwin-x86_64-validation.yml | 2 +- .../full-dataset-quality-gate-regression.yml | 4 +- .../linux-python-3_13-cli-validation.yml | 2 +- .../linux-selfhosted-gpu-validation.yml | 4 +- .github/workflows/macos15-mps-validation.yml | 2 +- ..._synthetic_validation_dataset_readiness.py | 97 +++++++++++++++++++ 6 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/suites/integration/models/test_synthetic_validation_dataset_readiness.py diff --git a/.github/workflows/darwin-x86_64-validation.yml b/.github/workflows/darwin-x86_64-validation.yml index b64895e..58c040c 100644 --- a/.github/workflows/darwin-x86_64-validation.yml +++ b/.github/workflows/darwin-x86_64-validation.yml @@ -42,7 +42,7 @@ jobs: run: ./scripts/workflows/setup_validation_environment.sh --python 3.12 --no-dev - name: Build synthetic RAVDESS smoke dataset - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh diff --git a/.github/workflows/full-dataset-quality-gate-regression.yml b/.github/workflows/full-dataset-quality-gate-regression.yml index 5eff0f6..c235e46 100644 --- a/.github/workflows/full-dataset-quality-gate-regression.yml +++ b/.github/workflows/full-dataset-quality-gate-regression.yml @@ -87,7 +87,7 @@ jobs: uv-${{ runner.os }}-full-gate- - name: Bootstrap synthetic RAVDESS dataset for hosted validation - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh @@ -166,7 +166,7 @@ jobs: uv-${{ runner.os }}-full-gate- - name: Bootstrap synthetic RAVDESS dataset for hosted validation - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh diff --git a/.github/workflows/linux-python-3_13-cli-validation.yml b/.github/workflows/linux-python-3_13-cli-validation.yml index cdc765e..6b26d32 100644 --- a/.github/workflows/linux-python-3_13-cli-validation.yml +++ b/.github/workflows/linux-python-3_13-cli-validation.yml @@ -74,7 +74,7 @@ jobs: ${{ inputs.run_accurate_research && '--accurate-research' || '' }} - name: Build synthetic RAVDESS smoke dataset - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh --python 3.13 diff --git a/.github/workflows/linux-selfhosted-gpu-validation.yml b/.github/workflows/linux-selfhosted-gpu-validation.yml index fff074c..4f445c3 100644 --- a/.github/workflows/linux-selfhosted-gpu-validation.yml +++ b/.github/workflows/linux-selfhosted-gpu-validation.yml @@ -138,7 +138,7 @@ jobs: --runtime cuda - name: Build synthetic RAVDESS smoke dataset - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh --python ${{ inputs.python_version }} @@ -250,7 +250,7 @@ jobs: --runtime xpu - name: Build synthetic RAVDESS smoke dataset - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Run XPU-focused policy/runtime tests run: > diff --git a/.github/workflows/macos15-mps-validation.yml b/.github/workflows/macos15-mps-validation.yml index ca80855..2cf6e94 100644 --- a/.github/workflows/macos15-mps-validation.yml +++ b/.github/workflows/macos15-mps-validation.yml @@ -89,7 +89,7 @@ jobs: --runtime mps - name: Build synthetic RAVDESS smoke dataset - run: python ./scripts/build_synthetic_ravdess_dataset.py + run: python ./scripts/build_synthetic_ravdess_dataset.py --actors 1 2 3 - name: Persist validation dataset consents run: ./scripts/configure_validation_dataset_consents.sh --python ${{ inputs.python_version }} diff --git a/tests/suites/integration/models/test_synthetic_validation_dataset_readiness.py b/tests/suites/integration/models/test_synthetic_validation_dataset_readiness.py new file mode 100644 index 0000000..f74dbed --- /dev/null +++ b/tests/suites/integration/models/test_synthetic_validation_dataset_readiness.py @@ -0,0 +1,97 @@ +"""Hosted validation lanes must build a synthetic dataset that passes training readiness.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from dataclasses import replace +from functools import partial +from pathlib import Path + +import pytest + +from ser._internal.config.bootstrap import reload_settings # noqa: TID251 +from ser._internal.data.adapters.ravdess import build_ravdess_utterances # noqa: TID251 +from ser._internal.data.label_ontology import resolve_label_ontology # noqa: TID251 +from ser._internal.data.manifest import Utterance # noqa: TID251 +from ser._internal.models.training_readiness import run_training_readiness # noqa: TID251 +from ser.config import AppConfig + +_BUILDER = "scripts/build_synthetic_ravdess_dataset.py" +_INVOCATION = re.compile(rf"python \./{re.escape(_BUILDER)}(?P[^\n]*)") +# Lanes train with the default split policy; neutralize developer overrides of it. +_SPLIT_POLICY_ENV = ( + "SER_TEST_SIZE", + "SER_DEV_SIZE", + "SER_MIN_REMAINING_PER_CLASS_SPLIT", + "SER_DATASET_MANIFESTS", + "SER_DATASET_RECIPE", +) + + +def _builder_invocations(repo_root: Path) -> list[tuple[str, tuple[str, ...]]]: + invocations: list[tuple[str, tuple[str, ...]]] = [] + for workflow in sorted((repo_root / ".github" / "workflows").glob("*.yml")): + for match in _INVOCATION.finditer(workflow.read_text(encoding="utf-8")): + invocations.append((workflow.name, tuple(match.group("args").split()))) + return invocations + + +def _lane_settings(work_root: Path) -> AppConfig: + settings = reload_settings() + return replace( + settings, + tmp_folder=work_root / "tmp", + models=replace( + settings.models, + folder=work_root / "models", + model_cache_dir=work_root / "cache", + ), + ) + + +def _load_lane_utterances(settings: AppConfig) -> list[Utterance] | None: + return build_ravdess_utterances( + dataset_root=settings.dataset.folder, + dataset_glob_pattern=settings.dataset.glob_pattern, + emotion_code_map=dict(settings.emotions), + default_language=settings.default_language, + ontology=resolve_label_ontology(settings), + max_failed_file_ratio=settings.data_loader.max_failed_file_ratio, + ) + + +def test_every_lane_passes_an_explicit_actor_list(repo_root: Path) -> None: + """The builder's default actor list is not what the lanes are validated against.""" + invocations = _builder_invocations(repo_root) + + assert invocations, "No workflow builds the synthetic dataset." + assert [name for name, args in invocations if "--actors" not in args] == [] + + +def test_lane_synthetic_datasets_pass_training_readiness( + repo_root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Each distinct lane dataset must satisfy the speaker-disjoint three-way split.""" + for variable in _SPLIT_POLICY_ENV: + monkeypatch.delenv(variable, raising=False) + + for index, args in enumerate(sorted({args for _, args in _builder_invocations(repo_root)})): + dataset_root = tmp_path / f"dataset-{index}" + subprocess.run( + [sys.executable, str(repo_root / _BUILDER), "--output-root", str(dataset_root), *args], + check=True, + capture_output=True, + ) + monkeypatch.setenv("DATASET_FOLDER", str(dataset_root)) + settings = _lane_settings(tmp_path / f"work-{index}") + + report, _ = run_training_readiness( + settings=settings, + load_utterances=partial(_load_lane_utterances, settings), + persist_quarantine_ledger=False, + report_path=tmp_path / f"readiness-{index}.json", + ) + + assert report.ready, (args, [finding.message for finding in report.findings])