feat(corpus): the measured coverage question, abstention, card size per request

internal/handler/corpus.go

@@ -54,6 +54,13 @@ type corpusRequest struct {
 	MaxPagesPerDocument int               `json:"max_pages_per_document"`
 	MaxEvidence         int               `json:"max_evidence"`
 	MaxHops             int               `json:"max_hops"`
+	// CardChars shortens each routing card (200-900 characters): routing
+	// over a large collection is most of a question's cost.
+	CardChars int `json:"card_chars"`
+	// Abstain returns no evidence when the collection does not appear to
+	// answer the question (coverage under the measured threshold). Default
+	// true; false returns the best evidence regardless.
+	Abstain *bool `json:"abstain"`
 	// Answer-only fields.
 	Model     string `json:"model"`
 	MaxTokens int    `json:"max_tokens"`
@@ -66,9 +73,8 @@ const (
 	corpusMaxPagesPerDocCap = 16
 	corpusMaxEvidenceCap    = 64
 	corpusListPage          = 200
-	// corpusDeclineBelow is the coverage under which /v1/answer/corpus
-	// tells the generator the collection may not answer the question.
-	corpusDeclineBelow = 0.25
+	corpusMinCardChars      = 200
+	corpusMaxCardChars      = 900
 )
 
 func (h *CorpusHandler) decode(w http.ResponseWriter, r *http.Request) (string, corpusRequest, bool) {
@@ -113,6 +119,9 @@ func (h *CorpusHandler) run(ctx context.Context, orgID, store string, body corpu
 	if body.MaxHops > 0 {
 		nav.MaxHops = min(body.MaxHops, corpusMaxHopsCap)
 	}
+	if body.CardChars > 0 {
+		nav.CardChars = max(corpusMinCardChars, min(body.CardChars, corpusMaxCardChars))
+	}
 	res, err := nav.Navigate(ctx, body.Query, docs)
 	if err != nil {
 		return nil, err
@@ -193,15 +202,22 @@ func (h *CorpusHandler) refs(ctx context.Context, orgID, store string, ids []tre
 	return refs, nil
 }
 
-func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time) map[string]any {
+func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time, abstain bool) map[string]any {
 	docs := res.Documents
+	evidence := res.Evidence
+	abstained := abstain && !res.Answerable
+	if abstained {
+		evidence = []retrieval.CorpusEvidence{}
+	}
 	if len(docs) > 25 {
 		docs = docs[:25] // the full ranking of a large collection is noise past the top
 	}
-	return map[string]any{
+	m := map[string]any{
 		"query":                query,
 		"strategy":             "corpus",
-		"evidence":             res.Evidence,
+		"evidence":             evidence,
+		"answerable":           res.Answerable,
+		"abstained":            abstained,
 		"documents":            docs,
 		"documents_considered": res.Considered,
 		"hops":                 res.Hops,
@@ -211,6 +227,10 @@ func corpusResponse(query string, res *retrieval.CorpusResult, started time.Time
 		"usage":                usageMap(res.Usage),
 		"elapsed_ms":           time.Since(started).Milliseconds(),
 	}
+	if abstained {
+		m["abstain_reason"] = "the collection does not appear to answer the question"
+	}
+	return m
 }
 
 // HandleQueryCorpus is POST /v1/query/corpus.
@@ -226,11 +246,13 @@ func (h *CorpusHandler) HandleQueryCorpus(w http.ResponseWriter, r *http.Request
 		writeErr(w, http.StatusBadGateway, "corpus retrieval failed: "+err.Error())
 		return
 	}
-	resp := corpusResponse(body.Query, res, started)
+	resp := corpusResponse(body.Query, res, started, abstains(body))
 	withTiming(r, resp)
 	writeJSON(w, http.StatusOK, resp)
 }
 
+func abstains(body corpusRequest) bool { return body.Abstain == nil || *body.Abstain }
+
 // HandleAnswerCorpus is POST /v1/answer/corpus: the evidence of
 // /v1/query/corpus, then one generated answer citing document and page,
 // each citation placed on its page.
@@ -251,8 +273,8 @@ func (h *CorpusHandler) HandleAnswerCorpus(w http.ResponseWriter, r *http.Reques
 		writeErr(w, http.StatusBadGateway, "corpus retrieval failed: "+err.Error())
 		return
 	}
-	resp := corpusResponse(body.Query, res, started)
-	if len(res.Evidence) == 0 {
+	resp := corpusResponse(body.Query, res, started, abstains(body))
+	if len(res.Evidence) == 0 || resp["abstained"] == true {
 		resp["answer"] = "No document in this collection appears to hold what the question needs."
 		resp["citations"] = []any{}
 		resp["elapsed_ms"] = time.Since(started).Milliseconds()
@@ -268,11 +290,11 @@ func (h *CorpusHandler) HandleAnswerCorpus(w http.ResponseWriter, r *http.Reques
 		})
 	}
 	query := body.Query
-	if res.Coverage < corpusDeclineBelow {
-		// The Judge found the evidence incomplete: tell the generator, so a
-		// question the collection does not answer is declined rather than
-		// completed from the model's own knowledge.
-		query += "\n\n(Retrieval judged these sections unlikely to contain everything the question needs. " +
+	if !res.Answerable {
+		// The caller asked for evidence regardless (abstain: false): tell the
+		// generator, so the answer says what is missing instead of filling it
+		// from the model's own knowledge.
+		query += "\n\n(Retrieval judged these sections unlikely to answer the question. " +
 			"If they do not answer it, say so and state what is missing.)"
 	}
 	model := body.Model

internal/handler/corpus_test.go

@@ -0,0 +1,37 @@
+package handler
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hallelx2/vectorless-engine/pkg/retrieval"
+)
+
+func TestCorpusResponseAbstainsWhenTheCollectionDoesNotAnswer(t *testing.T) {
+	res := &retrieval.CorpusResult{
+		Evidence:   []retrieval.CorpusEvidence{{DocumentID: "d", Page: 1, Text: "unrelated", Confidence: 0.2}},
+		Documents:  []retrieval.CorpusDocScore{},
+		Hops:       []retrieval.CorpusHop{},
+		Coverage:   0.02,
+		Answerable: false,
+	}
+	resp := corpusResponse("q", res, time.Now(), true)
+	if resp["abstained"] != true || len(resp["evidence"].([]retrieval.CorpusEvidence)) != 0 || resp["abstain_reason"] == nil {
+		t.Fatalf("an unanswerable collection should abstain with no evidence: %+v", resp)
+	}
+	kept := corpusResponse("q", res, time.Now(), false)
+	if kept["abstained"] != false || len(kept["evidence"].([]retrieval.CorpusEvidence)) != 1 {
+		t.Fatalf("abstain: false keeps the evidence: %+v", kept)
+	}
+	res.Answerable, res.Coverage = true, 0.4
+	if a := corpusResponse("q", res, time.Now(), true); a["abstained"] != false {
+		t.Fatalf("an answerable collection never abstains: %+v", a)
+	}
+}
+
+func TestAbstainDefaultsToTrue(t *testing.T) {
+	no := false
+	if !abstains(corpusRequest{}) || abstains(corpusRequest{Abstain: &no}) {
+		t.Fatal("abstain defaults to true and can be turned off")
+	}
+}

pkg/retrieval/corpus.go

@@ -93,8 +93,12 @@ type CorpusResult struct {
 	Evidence  []CorpusEvidence `json:"evidence"`
 	Documents []CorpusDocScore `json:"documents"`
 	Hops      []CorpusHop      `json:"hops"`
-	// Coverage is the final completeness probability.
+	// Coverage is the final probability that the evidence answers the
+	// question.
 	Coverage float64 `json:"coverage"`
+	// Answerable is Coverage at or above AnswerableThreshold: whether the
+	// collection appears to answer the question at all.
+	Answerable bool `json:"answerable"`
 	// Considered is how many documents the question was asked over.
 	Considered int `json:"documents_considered"`
 	// Errors are documents that could not be read, by ID; the question is
@@ -144,9 +148,13 @@ type CorpusNavigator struct {
 	EvidenceThreshold float64
 	// MaxHops bounds the rounds. Zero selects 3; 1 disables the bridge.
 	MaxHops int
-	// CoverageThreshold is the completeness probability at or above which
-	// no further hop is taken. Zero selects 0.5.
+	// CoverageThreshold is the coverage at or above which no further hop
+	// is taken. Zero selects 0.5.
 	CoverageThreshold float64
+	// AnswerableThreshold is the coverage below which the collection is
+	// judged not to answer the question (Result.Answerable false). Zero
+	// selects 0.06, the measured split (see coverage).
+	AnswerableThreshold float64
 	// CardChars bounds one card. Zero selects 900.
 	CardChars int
 	// FoundChars bounds the found-so-far state of the coverage and bridge
@@ -164,6 +172,7 @@ const (
 	defaultCorpusMaxEvidence    = 16
 	defaultCorpusMaxHops        = 3
 	defaultCorpusCoverage       = 0.5
+	defaultCorpusAnswerable     = 0.06
 	defaultCorpusCardChars      = 900
 	defaultCorpusFoundChars     = 6000
 	defaultCorpusReadPar        = 8
@@ -285,6 +294,7 @@ func (c *CorpusNavigator) Navigate(ctx context.Context, query string, docs []Cor
 	if len(res.Errors) == 0 {
 		res.Errors = nil
 	}
+	res.Answerable = len(res.Evidence) > 0 && res.Coverage >= corpusOr(c.AnswerableThreshold, defaultCorpusAnswerable)
 	return res, nil
 }
 
@@ -576,24 +586,29 @@ func (c *CorpusNavigator) coverage(ctx context.Context, query string, ev []Corpu
 		return 0, Usage{}, 0, nil
 	}
 	state := map[string]any{"question": query, "found": c.foundState(ev)}
+	// The "answerable" wording (HAL-3097, measured 2026-10-16 on 25
+	// answerable and 25 null MultiHop questions over the same evidence):
+	// AUC 0.968, means 0.412 vs 0.044, and at 0.06 it kept 24/25 and
+	// declined 23/25. The earlier "states every fact" wording separated the
+	// groups too (AUC 0.965) but sat at 0.03-0.5 on answerable questions, so
+	// a 0.5 stop meant every question bridged and none was ever declined.
 	q := map[string]llmgate.Question{
-		"complete": llmgate.Noul{
-			Instructions: "`question` is to be answered from a collection of documents. `found` lists " +
-				"the passages retrieved for it, each with its document and page. Do these passages, " +
-				"taken together, state every fact the question needs — each entity, event, figure, " +
-				"date or source it asks about? Judge only what the passages state, not what could be " +
-				"guessed. A question about something the passages never mention is not covered.",
+		"answerable": llmgate.Noul{
+			Instructions: "`question` is to be answered from `found`, passages retrieved from a collection " +
+				"of documents, each with its document and page. Could a careful reader give a definite " +
+				"answer to the question using only these passages, even if some wording must be matched " +
+				"up across them?",
 			Criteria: &llmgate.NoulCriteria{
-				True:  "Every fact the question needs is stated in the passages",
-				False: "At least one fact the question needs is missing from the passages",
+				True:  "A definite answer can be given from these passages",
+				False: "These passages do not support a definite answer",
 			},
 		},
 	}
 	jr, err := c.Judge.Judge(ctx, llmgate.JudgeRequest{State: state, Questions: q})
 	if err != nil {
 		return 0, Usage{}, 0, err
 	}
-	p, err := jr.Noul("complete")
+	p, err := jr.Noul("answerable")
 	if err != nil {
 		return 0, judgeUsage(jr), 1, nil
 	}

pkg/retrieval/corpus_test.go

@@ -32,7 +32,7 @@ func corpusJudge(t *testing.T) (*llmgate.MockJudge, *corpusLog) {
 			log.add(id, fmt.Sprint(q.(llmgate.Noul).Instructions))
 			p := 0.1
 			switch {
-			case id == "complete":
+			case id == "answerable":
 				if strings.Contains(found, "Beta-fact") {
 					p = 0.9
 				} else {
@@ -452,3 +452,25 @@ func TestCorpusLibraryReportsAStorageFailureInsteadOfFallingBack(t *testing.T) {
 		t.Fatalf("no persisted pages means an unpaged document, read from its tree: docs=%d errs=%v", len(docs), errs)
 	}
 }
+
+func TestCorpusAnswerableFollowsTheMeasuredThreshold(t *testing.T) {
+	j, _ := corpusJudge(t)
+	nav := CorpusNavigator{Judge: j, MinDocuments: 2}
+	res, err := nav.Navigate(context.Background(), "Which company did Acme partner with, and when was it founded?", newsCorpus())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !res.Answerable || res.Coverage < defaultCorpusAnswerable {
+		t.Fatalf("a covered question is answerable: %+v", res)
+	}
+	// the mock scores coverage 0.2 when the Beta fact is missing: above 0.06,
+	// so a higher threshold is what declines it
+	strict := CorpusNavigator{Judge: j, AnswerableThreshold: 0.5}
+	res2, err := strict.Navigate(context.Background(), "What revenue did Acme report?", newsCorpus()[3:])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res2.Answerable {
+		t.Fatalf("below the threshold the collection does not answer: %+v", res2)
+	}
+}