From e0d370d0d29b34dffeb8a09ce1f63afa5a139a2e Mon Sep 17 00:00:00 2001 From: Niroj33 <86530493+professor3333@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:05:38 +0545 Subject: [PATCH] Keep the neural training path free of baseline dependencies The transformer trainer imported its provenance helpers from the CRF module, which imports joblib, so the GPU runbook (neural extra only) failed at its first experiment. The helpers move to models/provenance.py. A test imports the neural training and prediction modules with baseline and serving packages blocked. Re-running the runbook reuses its venv. --- scripts/gpu/run_neural.sh | 2 +- src/triggermap/models/crf/model.py | 13 +--------- src/triggermap/models/provenance.py | 21 +++++++++++++++ src/triggermap/models/transformer/train.py | 2 +- tests/test_transformer.py | 30 ++++++++++++++++++++++ 5 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 src/triggermap/models/provenance.py diff --git a/scripts/gpu/run_neural.sh b/scripts/gpu/run_neural.sh index 990b27e..57f5384 100755 --- a/scripts/gpu/run_neural.sh +++ b/scripts/gpu/run_neural.sh @@ -35,7 +35,7 @@ mkdir -p "$OUT_DIR" # 1. Environment: Python 3.12 with CUDA PyTorch from PyPI. --no-sources skips the CPU-only torch # index that pyproject.toml pins for Linux development machines. command -v uv >/dev/null || python3 -m pip install --quiet "uv==$UV_VERSION" -uv venv --quiet --python 3.12 .venv-gpu +uv venv --quiet --allow-existing --python 3.12 .venv-gpu uv pip install --quiet --python .venv-gpu --no-sources -e ".[neural]" if [ -n "${TORCH_INDEX:-}" ]; then uv pip install --quiet --python .venv-gpu --reinstall-package torch \ diff --git a/src/triggermap/models/crf/model.py b/src/triggermap/models/crf/model.py index fe1ea6a..6e179fd 100644 --- a/src/triggermap/models/crf/model.py +++ b/src/triggermap/models/crf/model.py @@ -12,7 +12,6 @@ from __future__ import annotations -import hashlib import json from collections import Counter from collections.abc import Sequence @@ -42,6 +41,7 @@ split_at_lines, tokenize, ) +from triggermap.models.provenance import training_hash, training_origins from triggermap.schema.models import ( AdvisoryAnnotation, Category, @@ -136,17 +136,6 @@ def _pairs( return out -def training_hash(records: Sequence[AdvisoryAnnotation]) -> str: - lines = sorted(r.model_dump_json() for r in records) - return hashlib.sha256("\n".join(lines).encode()).hexdigest() - - -def training_origins(records: Sequence[AdvisoryAnnotation]) -> dict[str, int]: - """How many training records each label origin contributed (``unrecorded`` before 1.1).""" - counts = Counter(r.origin.value if r.origin else "unrecorded" for r in records) - return dict(sorted(counts.items())) - - class CrfExtractor: def __init__( self, diff --git a/src/triggermap/models/provenance.py b/src/triggermap/models/provenance.py new file mode 100644 index 0000000..56804ec --- /dev/null +++ b/src/triggermap/models/provenance.py @@ -0,0 +1,21 @@ +"""Training-data provenance shared by every learner (no learner dependencies).""" + +from __future__ import annotations + +import hashlib +from collections import Counter +from collections.abc import Sequence + +from triggermap.schema.models import AdvisoryAnnotation + + +def training_hash(records: Sequence[AdvisoryAnnotation]) -> str: + """SHA-256 of the training records, independent of their order.""" + lines = sorted(r.model_dump_json() for r in records) + return hashlib.sha256("\n".join(lines).encode()).hexdigest() + + +def training_origins(records: Sequence[AdvisoryAnnotation]) -> dict[str, int]: + """How many training records each label origin contributed (``unrecorded`` before 1.1).""" + counts = Counter(r.origin.value if r.origin else "unrecorded" for r in records) + return dict(sorted(counts.items())) diff --git a/src/triggermap/models/transformer/train.py b/src/triggermap/models/transformer/train.py index 6d90339..0de259a 100644 --- a/src/triggermap/models/transformer/train.py +++ b/src/triggermap/models/transformer/train.py @@ -25,7 +25,7 @@ from transformers import AutoModel, AutoTokenizer, get_linear_schedule_with_warmup from triggermap.eval.metrics import collect, summary -from triggermap.models.crf.model import training_hash, training_origins +from triggermap.models.provenance import training_hash, training_origins from triggermap.models.transformer.data import ( BIO, CATEGORIES, diff --git a/tests/test_transformer.py b/tests/test_transformer.py index 5732a44..1c87cf9 100644 --- a/tests/test_transformer.py +++ b/tests/test_transformer.py @@ -340,3 +340,33 @@ def test_scope_index_matches_linear_scope_of() -> None: for start in range(len(doc.text)): span = (start, min(start + 3, len(doc.text))) assert index.scope(span) == scope_of(doc, span) + + +def test_neural_path_needs_only_the_neural_extra() -> None: + """The GPU runbook installs only `.[neural]`: training and prediction must not import + baseline (CRF, spaCy, scikit-learn) or serving packages.""" + import subprocess + import sys + + blocked = ["joblib", "sklearn", "pycrfsuite", "spacy", "fastapi", "uvicorn", "onnxruntime"] + modules = [ + "triggermap.cli", + "triggermap.models.transformer.train", + "triggermap.models.transformer.extractor", + "triggermap.models.experiment", + "triggermap.models.registry", + "triggermap.corpus.splits", + "triggermap.eval.runner", + "triggermap.eval.report", + "triggermap.serve.policy", + "triggermap.schema.validate", + "triggermap.ingest.build", + "triggermap.text.preprocessing", + ] + code = ( + "import importlib, sys\n" + f"for name in {blocked!r}: sys.modules[name] = None\n" + f"for name in {modules!r}: importlib.import_module(name)\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr[-2000:]