From 42687448db99102dde6a5fee21f6a678704af3e3 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Wed, 2 Sep 2026 09:40:19 +0100 Subject: [PATCH] Deploy LFS assets; stop bengio_infer from wedging the worker After the cold-start fix, staging surfaced two model failures that were previously masked by the timeout: - shannon_words_ngram -> "MDB_INVALID: File is not an LMDB file" - bengio_infer -> torch.load on a 133-byte file, unhandled exception, which killed the stdio RPC worker and 503'd every request for ~30s until shimmy respawned it. Both are Git LFS: *.mdb / *.pt are LFS-tracked but the deploy workflows checked out with lfs:false, shipping pointer stubs. basic_nn.pt predates the LFS filter (stored inline) so it was unaffected. - staging-deploy.yml / production-deploy.yml: lfs: true - bengio_infer.run(): wrap load+infer in try/except -> return a failing Result instead of raising, so a bad asset degrades one request rather than taking down the worker - bengio_infer.predict_next(): clamp the re-encoded context to exactly N tokens (it could exceed N and hit "mat1 and mat2 shapes cannot be multiplied"), so the model actually produces output once assets load Verified against an image with real LFS assets: all five models (basic_nn, shannon_letters_single, shannon_letters_ngram, shannon_words_ngram, bengio_infer) return is_correct results. pytest passes. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UsgxJUsq1vG176rbznHRqz --- .github/workflows/production-deploy.yml | 4 ++++ .github/workflows/staging-deploy.yml | 5 ++++- evaluation_function/models/bengio_infer.py | 24 +++++++++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index 901c8d9..e8aa7be 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -38,6 +38,10 @@ jobs: with: template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' environment: "production" + # LFS-tracked assets (*.mdb n-gram shards, bengio_model.pt) must be + # checked out for real, not as pointer files, or shannon_words_ngram + # and bengio_infer fail at runtime. + lfs: true version-bump: ${{ inputs.version-bump }} branch: ${{ inputs.branch }} run-database-tests: false diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml index c693871..6443ef7 100644 --- a/.github/workflows/staging-deploy.yml +++ b/.github/workflows/staging-deploy.yml @@ -57,7 +57,10 @@ jobs: template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" build-platforms: "aws" environment: "staging" - lfs: false + # LFS-tracked assets (*.mdb n-gram shards, bengio_model.pt) must be + # checked out for real, not as pointer files, or shannon_words_ngram + # and bengio_infer fail at runtime. + lfs: true secrets: aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} diff --git a/evaluation_function/models/bengio_infer.py b/evaluation_function/models/bengio_infer.py index dbdcbb3..896fc70 100644 --- a/evaluation_function/models/bengio_infer.py +++ b/evaluation_function/models/bengio_infer.py @@ -1,5 +1,5 @@ # Inference code for Bengio-style Neural N-gram Language Model -import json, os +import json, os, traceback from evaluation_function.lazy_load import LazyModule from evaluation_function.models.utils import NeuralLM @@ -15,8 +15,12 @@ def predict_next(context_words, topk=5, model=None,config=None, sp=None, device= UNK = sp.unk_id() with torch.no_grad(): ctx_ids = encode(context_words[-N:]) + # Re-encoding N word pieces can yield more or fewer than N subword + # tokens; the model's first layer needs exactly N. if len(ctx_ids) < N: ctx_ids = [UNK] * (N - len(ctx_ids)) + ctx_ids + else: + ctx_ids = ctx_ids[-N:] x = torch.tensor([ctx_ids], dtype=torch.long, device=device) logits = model(x) probs = torch.softmax(logits, dim=-1).squeeze() @@ -36,6 +40,24 @@ def complete(prompt, steps=10,model=None,config=None,sp=None,device=None): return sp.decode(words) def run(response, answer, params: Params) -> Result: + # Guard the whole load+infer path: a missing/corrupt model asset (e.g. an + # un-fetched Git LFS pointer) must degrade this one request, not raise out + # of the worker and take down the RPC loop for every following request. + try: + return _run_inference(response, params) + except Exception as e: + print("### bengio_infer failed ###") + traceback.print_exc() + return Result( + is_correct=False, + feedback_items=[ + ("general", "Could not run the Bengio n-gram language model."), + ("error", str(e)), + ], + ) + + +def _run_inference(response, params: Params) -> Result: print("### Loading Bengio-style Neural N-gram Language Model for inference... ###") device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")