diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index 901c8d9..e8aa7be 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -38,6 +38,10 @@ jobs: with: template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' environment: "production" + # LFS-tracked assets (*.mdb n-gram shards, bengio_model.pt) must be + # checked out for real, not as pointer files, or shannon_words_ngram + # and bengio_infer fail at runtime. + lfs: true version-bump: ${{ inputs.version-bump }} branch: ${{ inputs.branch }} run-database-tests: false diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml index c693871..6443ef7 100644 --- a/.github/workflows/staging-deploy.yml +++ b/.github/workflows/staging-deploy.yml @@ -57,7 +57,10 @@ jobs: template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" build-platforms: "aws" environment: "staging" - lfs: false + # LFS-tracked assets (*.mdb n-gram shards, bengio_model.pt) must be + # checked out for real, not as pointer files, or shannon_words_ngram + # and bengio_infer fail at runtime. + lfs: true secrets: aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}} diff --git a/evaluation_function/models/bengio_infer.py b/evaluation_function/models/bengio_infer.py index dbdcbb3..896fc70 100644 --- a/evaluation_function/models/bengio_infer.py +++ b/evaluation_function/models/bengio_infer.py @@ -1,5 +1,5 @@ # Inference code for Bengio-style Neural N-gram Language Model -import json, os +import json, os, traceback from evaluation_function.lazy_load import LazyModule from evaluation_function.models.utils import NeuralLM @@ -15,8 +15,12 @@ def predict_next(context_words, topk=5, model=None,config=None, sp=None, device= UNK = sp.unk_id() with torch.no_grad(): ctx_ids = encode(context_words[-N:]) + # Re-encoding N word pieces can yield more or fewer than N subword + # tokens; the model's first layer needs exactly N. if len(ctx_ids) < N: ctx_ids = [UNK] * (N - len(ctx_ids)) + ctx_ids + else: + ctx_ids = ctx_ids[-N:] x = torch.tensor([ctx_ids], dtype=torch.long, device=device) logits = model(x) probs = torch.softmax(logits, dim=-1).squeeze() @@ -36,6 +40,24 @@ def complete(prompt, steps=10,model=None,config=None,sp=None,device=None): return sp.decode(words) def run(response, answer, params: Params) -> Result: + # Guard the whole load+infer path: a missing/corrupt model asset (e.g. an + # un-fetched Git LFS pointer) must degrade this one request, not raise out + # of the worker and take down the RPC loop for every following request. + try: + return _run_inference(response, params) + except Exception as e: + print("### bengio_infer failed ###") + traceback.print_exc() + return Result( + is_correct=False, + feedback_items=[ + ("general", "Could not run the Bengio n-gram language model."), + ("error", str(e)), + ], + ) + + +def _run_inference(response, params: Params) -> Result: print("### Loading Bengio-style Neural N-gram Language Model for inference... ###") device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")