diff --git a/scripts/gpu/run_neural.sh b/scripts/gpu/run_neural.sh index 990b27e..57f5384 100755 --- a/scripts/gpu/run_neural.sh +++ b/scripts/gpu/run_neural.sh @@ -35,7 +35,7 @@ mkdir -p "$OUT_DIR" # 1. Environment: Python 3.12 with CUDA PyTorch from PyPI. --no-sources skips the CPU-only torch # index that pyproject.toml pins for Linux development machines. command -v uv >/dev/null || python3 -m pip install --quiet "uv==$UV_VERSION" -uv venv --quiet --python 3.12 .venv-gpu +uv venv --quiet --allow-existing --python 3.12 .venv-gpu uv pip install --quiet --python .venv-gpu --no-sources -e ".[neural]" if [ -n "${TORCH_INDEX:-}" ]; then uv pip install --quiet --python .venv-gpu --reinstall-package torch \ diff --git a/src/triggermap/models/crf/model.py b/src/triggermap/models/crf/model.py index fe1ea6a..6e179fd 100644 --- a/src/triggermap/models/crf/model.py +++ b/src/triggermap/models/crf/model.py @@ -12,7 +12,6 @@ from __future__ import annotations -import hashlib import json from collections import Counter from collections.abc import Sequence @@ -42,6 +41,7 @@ split_at_lines, tokenize, ) +from triggermap.models.provenance import training_hash, training_origins from triggermap.schema.models import ( AdvisoryAnnotation, Category, @@ -136,17 +136,6 @@ def _pairs( return out -def training_hash(records: Sequence[AdvisoryAnnotation]) -> str: - lines = sorted(r.model_dump_json() for r in records) - return hashlib.sha256("\n".join(lines).encode()).hexdigest() - - -def training_origins(records: Sequence[AdvisoryAnnotation]) -> dict[str, int]: - """How many training records each label origin contributed (``unrecorded`` before 1.1).""" - counts = Counter(r.origin.value if r.origin else "unrecorded" for r in records) - return dict(sorted(counts.items())) - - class CrfExtractor: def __init__( self, diff --git a/src/triggermap/models/provenance.py b/src/triggermap/models/provenance.py new file mode 100644 index 0000000..56804ec --- /dev/null +++ b/src/triggermap/models/provenance.py @@ -0,0 +1,21 @@ +"""Training-data provenance shared by every learner (no learner dependencies).""" + +from __future__ import annotations + +import hashlib +from collections import Counter +from collections.abc import Sequence + +from triggermap.schema.models import AdvisoryAnnotation + + +def training_hash(records: Sequence[AdvisoryAnnotation]) -> str: + """SHA-256 of the training records, independent of their order.""" + lines = sorted(r.model_dump_json() for r in records) + return hashlib.sha256("\n".join(lines).encode()).hexdigest() + + +def training_origins(records: Sequence[AdvisoryAnnotation]) -> dict[str, int]: + """How many training records each label origin contributed (``unrecorded`` before 1.1).""" + counts = Counter(r.origin.value if r.origin else "unrecorded" for r in records) + return dict(sorted(counts.items())) diff --git a/src/triggermap/models/transformer/train.py b/src/triggermap/models/transformer/train.py index 6d90339..0de259a 100644 --- a/src/triggermap/models/transformer/train.py +++ b/src/triggermap/models/transformer/train.py @@ -25,7 +25,7 @@ from transformers import AutoModel, AutoTokenizer, get_linear_schedule_with_warmup from triggermap.eval.metrics import collect, summary -from triggermap.models.crf.model import training_hash, training_origins +from triggermap.models.provenance import training_hash, training_origins from triggermap.models.transformer.data import ( BIO, CATEGORIES, diff --git a/tests/test_transformer.py b/tests/test_transformer.py index 5732a44..1c87cf9 100644 --- a/tests/test_transformer.py +++ b/tests/test_transformer.py @@ -340,3 +340,33 @@ def test_scope_index_matches_linear_scope_of() -> None: for start in range(len(doc.text)): span = (start, min(start + 3, len(doc.text))) assert index.scope(span) == scope_of(doc, span) + + +def test_neural_path_needs_only_the_neural_extra() -> None: + """The GPU runbook installs only `.[neural]`: training and prediction must not import + baseline (CRF, spaCy, scikit-learn) or serving packages.""" + import subprocess + import sys + + blocked = ["joblib", "sklearn", "pycrfsuite", "spacy", "fastapi", "uvicorn", "onnxruntime"] + modules = [ + "triggermap.cli", + "triggermap.models.transformer.train", + "triggermap.models.transformer.extractor", + "triggermap.models.experiment", + "triggermap.models.registry", + "triggermap.corpus.splits", + "triggermap.eval.runner", + "triggermap.eval.report", + "triggermap.serve.policy", + "triggermap.schema.validate", + "triggermap.ingest.build", + "triggermap.text.preprocessing", + ] + code = ( + "import importlib, sys\n" + f"for name in {blocked!r}: sys.modules[name] = None\n" + f"for name in {modules!r}: importlib.import_module(name)\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr[-2000:]