diff --git a/internal/handler/corpus.go b/internal/handler/corpus.go index 1b68765..c983eca 100644 --- a/internal/handler/corpus.go +++ b/internal/handler/corpus.go @@ -54,6 +54,13 @@ type corpusRequest struct { MaxPagesPerDocument int `json:"max_pages_per_document"` MaxEvidence int `json:"max_evidence"` MaxHops int `json:"max_hops"` + // CardChars shortens each routing card (200-900 characters): routing + // over a large collection is most of a question's cost. + CardChars int `json:"card_chars"` + // Abstain returns no evidence when the collection does not appear to + // answer the question (coverage under the measured threshold). Default + // true; false returns the best evidence regardless. + Abstain *bool `json:"abstain"` // Answer-only fields. Model string `json:"model"` MaxTokens int `json:"max_tokens"` @@ -66,9 +73,8 @@ const ( corpusMaxPagesPerDocCap = 16 corpusMaxEvidenceCap = 64 corpusListPage = 200 - // corpusDeclineBelow is the coverage under which /v1/answer/corpus - // tells the generator the collection may not answer the question. - corpusDeclineBelow = 0.25 + corpusMinCardChars = 200 + corpusMaxCardChars = 900 ) func (h *CorpusHandler) decode(w http.ResponseWriter, r *http.Request) (string, corpusRequest, bool) { @@ -113,6 +119,9 @@ func (h *CorpusHandler) run(ctx context.Context, orgID, store string, body corpu if body.MaxHops > 0 { nav.MaxHops = min(body.MaxHops, corpusMaxHopsCap) } + if body.CardChars > 0 { + nav.CardChars = max(corpusMinCardChars, min(body.CardChars, corpusMaxCardChars)) + } res, err := nav.Navigate(ctx, body.Query, docs) if err != nil { return nil, err @@ -193,15 +202,22 @@ func (h *CorpusHandler) refs(ctx context.Context, orgID, store string, ids []tre return refs, nil } -func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time) map[string]any { +func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time, abstain bool) map[string]any { docs := res.Documents + evidence := res.Evidence + abstained := abstain && !res.Answerable + if abstained { + evidence = []retrieval.CorpusEvidence{} + } if len(docs) > 25 { docs = docs[:25] // the full ranking of a large collection is noise past the top } - return map[string]any{ + m := map[string]any{ "query": query, "strategy": "corpus", - "evidence": res.Evidence, + "evidence": evidence, + "answerable": res.Answerable, + "abstained": abstained, "documents": docs, "documents_considered": res.Considered, "hops": res.Hops, @@ -211,6 +227,10 @@ func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time "usage": usageMap(res.Usage), "elapsed_ms": time.Since(started).Milliseconds(), } + if abstained { + m["abstain_reason"] = "the collection does not appear to answer the question" + } + return m } // HandleQueryCorpus is POST /v1/query/corpus. @@ -226,11 +246,13 @@ func (h *CorpusHandler) HandleQueryCorpus(w http.ResponseWriter, r *http.Request writeErr(w, http.StatusBadGateway, "corpus retrieval failed: "+err.Error()) return } - resp := corpusResponse(body.Query, res, started) + resp := corpusResponse(body.Query, res, started, abstains(body)) withTiming(r, resp) writeJSON(w, http.StatusOK, resp) } +func abstains(body corpusRequest) bool { return body.Abstain == nil || *body.Abstain } + // HandleAnswerCorpus is POST /v1/answer/corpus: the evidence of // /v1/query/corpus, then one generated answer citing document and page, // each citation placed on its page. @@ -251,8 +273,8 @@ func (h *CorpusHandler) HandleAnswerCorpus(w http.ResponseWriter, r *http.Reques writeErr(w, http.StatusBadGateway, "corpus retrieval failed: "+err.Error()) return } - resp := corpusResponse(body.Query, res, started) - if len(res.Evidence) == 0 { + resp := corpusResponse(body.Query, res, started, abstains(body)) + if len(res.Evidence) == 0 || resp["abstained"] == true { resp["answer"] = "No document in this collection appears to hold what the question needs." resp["citations"] = []any{} resp["elapsed_ms"] = time.Since(started).Milliseconds() @@ -268,11 +290,11 @@ func (h *CorpusHandler) HandleAnswerCorpus(w http.ResponseWriter, r *http.Reques }) } query := body.Query - if res.Coverage < corpusDeclineBelow { - // The Judge found the evidence incomplete: tell the generator, so a - // question the collection does not answer is declined rather than - // completed from the model's own knowledge. - query += "\n\n(Retrieval judged these sections unlikely to contain everything the question needs. " + + if !res.Answerable { + // The caller asked for evidence regardless (abstain: false): tell the + // generator, so the answer says what is missing instead of filling it + // from the model's own knowledge. + query += "\n\n(Retrieval judged these sections unlikely to answer the question. " + "If they do not answer it, say so and state what is missing.)" } model := body.Model diff --git a/internal/handler/corpus_test.go b/internal/handler/corpus_test.go new file mode 100644 index 0000000..ba7e059 --- /dev/null +++ b/internal/handler/corpus_test.go @@ -0,0 +1,37 @@ +package handler + +import ( + "testing" + "time" + + "github.com/hallelx2/vectorless-engine/pkg/retrieval" +) + +func TestCorpusResponseAbstainsWhenTheCollectionDoesNotAnswer(t *testing.T) { + res := &retrieval.CorpusResult{ + Evidence: []retrieval.CorpusEvidence{{DocumentID: "d", Page: 1, Text: "unrelated", Confidence: 0.2}}, + Documents: []retrieval.CorpusDocScore{}, + Hops: []retrieval.CorpusHop{}, + Coverage: 0.02, + Answerable: false, + } + resp := corpusResponse("q", res, time.Now(), true) + if resp["abstained"] != true || len(resp["evidence"].([]retrieval.CorpusEvidence)) != 0 || resp["abstain_reason"] == nil { + t.Fatalf("an unanswerable collection should abstain with no evidence: %+v", resp) + } + kept := corpusResponse("q", res, time.Now(), false) + if kept["abstained"] != false || len(kept["evidence"].([]retrieval.CorpusEvidence)) != 1 { + t.Fatalf("abstain: false keeps the evidence: %+v", kept) + } + res.Answerable, res.Coverage = true, 0.4 + if a := corpusResponse("q", res, time.Now(), true); a["abstained"] != false { + t.Fatalf("an answerable collection never abstains: %+v", a) + } +} + +func TestAbstainDefaultsToTrue(t *testing.T) { + no := false + if !abstains(corpusRequest{}) || abstains(corpusRequest{Abstain: &no}) { + t.Fatal("abstain defaults to true and can be turned off") + } +} diff --git a/pkg/retrieval/corpus.go b/pkg/retrieval/corpus.go index 696f69f..a096730 100644 --- a/pkg/retrieval/corpus.go +++ b/pkg/retrieval/corpus.go @@ -93,8 +93,12 @@ type CorpusResult struct { Evidence []CorpusEvidence `json:"evidence"` Documents []CorpusDocScore `json:"documents"` Hops []CorpusHop `json:"hops"` - // Coverage is the final completeness probability. + // Coverage is the final probability that the evidence answers the + // question. Coverage float64 `json:"coverage"` + // Answerable is Coverage at or above AnswerableThreshold: whether the + // collection appears to answer the question at all. + Answerable bool `json:"answerable"` // Considered is how many documents the question was asked over. Considered int `json:"documents_considered"` // Errors are documents that could not be read, by ID; the question is @@ -144,9 +148,13 @@ type CorpusNavigator struct { EvidenceThreshold float64 // MaxHops bounds the rounds. Zero selects 3; 1 disables the bridge. MaxHops int - // CoverageThreshold is the completeness probability at or above which - // no further hop is taken. Zero selects 0.5. + // CoverageThreshold is the coverage at or above which no further hop + // is taken. Zero selects 0.5. CoverageThreshold float64 + // AnswerableThreshold is the coverage below which the collection is + // judged not to answer the question (Result.Answerable false). Zero + // selects 0.06, the measured split (see coverage). + AnswerableThreshold float64 // CardChars bounds one card. Zero selects 900. CardChars int // FoundChars bounds the found-so-far state of the coverage and bridge @@ -164,6 +172,7 @@ const ( defaultCorpusMaxEvidence = 16 defaultCorpusMaxHops = 3 defaultCorpusCoverage = 0.5 + defaultCorpusAnswerable = 0.06 defaultCorpusCardChars = 900 defaultCorpusFoundChars = 6000 defaultCorpusReadPar = 8 @@ -285,6 +294,7 @@ func (c *CorpusNavigator) Navigate(ctx context.Context, query string, docs []Cor if len(res.Errors) == 0 { res.Errors = nil } + res.Answerable = len(res.Evidence) > 0 && res.Coverage >= corpusOr(c.AnswerableThreshold, defaultCorpusAnswerable) return res, nil } @@ -576,16 +586,21 @@ func (c *CorpusNavigator) coverage(ctx context.Context, query string, ev []Corpu return 0, Usage{}, 0, nil } state := map[string]any{"question": query, "found": c.foundState(ev)} + // The "answerable" wording (HAL-3097, measured 2026-10-16 on 25 + // answerable and 25 null MultiHop questions over the same evidence): + // AUC 0.968, means 0.412 vs 0.044, and at 0.06 it kept 24/25 and + // declined 23/25. The earlier "states every fact" wording separated the + // groups too (AUC 0.965) but sat at 0.03-0.5 on answerable questions, so + // a 0.5 stop meant every question bridged and none was ever declined. q := map[string]llmgate.Question{ - "complete": llmgate.Noul{ - Instructions: "`question` is to be answered from a collection of documents. `found` lists " + - "the passages retrieved for it, each with its document and page. Do these passages, " + - "taken together, state every fact the question needs — each entity, event, figure, " + - "date or source it asks about? Judge only what the passages state, not what could be " + - "guessed. A question about something the passages never mention is not covered.", + "answerable": llmgate.Noul{ + Instructions: "`question` is to be answered from `found`, passages retrieved from a collection " + + "of documents, each with its document and page. Could a careful reader give a definite " + + "answer to the question using only these passages, even if some wording must be matched " + + "up across them?", Criteria: &llmgate.NoulCriteria{ - True: "Every fact the question needs is stated in the passages", - False: "At least one fact the question needs is missing from the passages", + True: "A definite answer can be given from these passages", + False: "These passages do not support a definite answer", }, }, } @@ -593,7 +608,7 @@ func (c *CorpusNavigator) coverage(ctx context.Context, query string, ev []Corpu if err != nil { return 0, Usage{}, 0, err } - p, err := jr.Noul("complete") + p, err := jr.Noul("answerable") if err != nil { return 0, judgeUsage(jr), 1, nil } diff --git a/pkg/retrieval/corpus_test.go b/pkg/retrieval/corpus_test.go index 212f628..3547f08 100644 --- a/pkg/retrieval/corpus_test.go +++ b/pkg/retrieval/corpus_test.go @@ -32,7 +32,7 @@ func corpusJudge(t *testing.T) (*llmgate.MockJudge, *corpusLog) { log.add(id, fmt.Sprint(q.(llmgate.Noul).Instructions)) p := 0.1 switch { - case id == "complete": + case id == "answerable": if strings.Contains(found, "Beta-fact") { p = 0.9 } else { @@ -452,3 +452,25 @@ func TestCorpusLibraryReportsAStorageFailureInsteadOfFallingBack(t *testing.T) { t.Fatalf("no persisted pages means an unpaged document, read from its tree: docs=%d errs=%v", len(docs), errs) } } + +func TestCorpusAnswerableFollowsTheMeasuredThreshold(t *testing.T) { + j, _ := corpusJudge(t) + nav := CorpusNavigator{Judge: j, MinDocuments: 2} + res, err := nav.Navigate(context.Background(), "Which company did Acme partner with, and when was it founded?", newsCorpus()) + if err != nil { + t.Fatal(err) + } + if !res.Answerable || res.Coverage < defaultCorpusAnswerable { + t.Fatalf("a covered question is answerable: %+v", res) + } + // the mock scores coverage 0.2 when the Beta fact is missing: above 0.06, + // so a higher threshold is what declines it + strict := CorpusNavigator{Judge: j, AnswerableThreshold: 0.5} + res2, err := strict.Navigate(context.Background(), "What revenue did Acme report?", newsCorpus()[3:]) + if err != nil { + t.Fatal(err) + } + if res2.Answerable { + t.Fatalf("below the threshold the collection does not answer: %+v", res2) + } +}