From 824c768172e49561eb5ee1894a7a5392f321ab85 Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 17 Oct 2026 06:53:34 +0100 Subject: [PATCH 1/4] feat(answer): do an answer's arithmetic in code and trace each operand to its page An answer model that sets a calculation up correctly still gets the arithmetic wrong. On FinanceBench, gemini-2.5-flash-lite wrote 365 * 1,380.5 / 7,890 = 67.75 (it is 63.86), and the old prompt declined derived figures outright. pkg/calc evaluates <> set-ups: numbers read the way a filing writes them, + - * /, parentheses and signs, nothing else. Each one is replaced in the answer by "value (= expression)". Trace finds every operand on the evidence block that states it, as written or in the other common form ("2763" / "2,763"), as a whole number only. A figure on no page is marked found: false. Both /v1/answer paths and the standalone server tell the model how to set a calculation up (calc.Instructions) and return "calculations" with operands, pages and values. --- internal/api/server.go | 27 +- internal/handler/answer.go | 29 +- internal/handler/answer_calc_test.go | 72 +++++ internal/handler/answer_pages.go | 22 +- pkg/calc/calc.go | 401 +++++++++++++++++++++++++++ pkg/calc/calc_test.go | 112 ++++++++ 6 files changed, 645 insertions(+), 18 deletions(-) create mode 100644 internal/handler/answer_calc_test.go create mode 100644 pkg/calc/calc.go create mode 100644 pkg/calc/calc_test.go diff --git a/internal/api/server.go b/internal/api/server.go index b083cbc..e1f5efc 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -21,6 +21,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/hallelx2/llmgate" + "github.com/hallelx2/vectorless-engine/pkg/calc" "github.com/hallelx2/vectorless-engine/pkg/config" "github.com/hallelx2/vectorless-engine/pkg/db" "github.com/hallelx2/vectorless-engine/pkg/ingest" @@ -1014,6 +1015,16 @@ func (d Deps) handleAnswer(w http.ResponseWriter, r *http.Request) { return } totalUsage.Add(synthUsage) + // The model set its calculations up; code works them out (HAL-3161). + answerText, calcs := calc.Resolve(answerText) + sources := make([]calc.Source, len(enriched)) + for i, e := range enriched { + sources[i] = calc.Source{Page: e.sec.PageStart, Text: e.content} + } + calc.Trace(calcs, sources) + if calcs == nil { + calcs = []calc.Calculation{} + } citations := make([]map[string]any, 0, len(enriched)) finalIDs := make([]tree.SectionID, 0, len(enriched)) @@ -1049,12 +1060,13 @@ func (d Deps) handleAnswer(w http.ResponseWriter, r *http.Request) { traceToken := retrieval.ComputeTraceToken(body.DocumentID, "1", synthModel, finalIDs) resp := map[string]any{ - "document_id": body.DocumentID, - "query": body.Query, - "answer": answerText, - "citations": citations, - "strategy": retrieval.PublicName(d.Strategy.Name()), - "model": synthModel, + "document_id": body.DocumentID, + "query": body.Query, + "answer": answerText, + "calculations": calcs, + "citations": citations, + "strategy": retrieval.PublicName(d.Strategy.Name()), + "model": synthModel, "usage": map[string]any{ "input_tokens": totalUsage.InputTokens, "output_tokens": totalUsage.OutputTokens, @@ -1120,7 +1132,8 @@ func synthesiseAnswer(ctx context.Context, client llmgate.Client, model, query s } } b.WriteString("\nWrite a concise answer to the user's query. ") - b.WriteString("If the evidence does not contain an answer, say so. ") + b.WriteString(calc.Instructions) + b.WriteString(" If the evidence does not contain an answer, say so. ") b.WriteString("Inline citations should reference the section_id values shown above. ") b.WriteString("Output plain prose; no JSON.") diff --git a/internal/handler/answer.go b/internal/handler/answer.go index 4b3968b..0d0c869 100644 --- a/internal/handler/answer.go +++ b/internal/handler/answer.go @@ -15,6 +15,7 @@ import ( "github.com/hallelx2/llmgate" + "github.com/hallelx2/vectorless-engine/pkg/calc" enginecfg "github.com/hallelx2/vectorless-engine/pkg/config" "github.com/hallelx2/vectorless-engine/pkg/db" "github.com/hallelx2/vectorless-engine/pkg/pincite" @@ -359,6 +360,12 @@ func (h *AnswerHandler) answerFromSections(ctx context.Context, t *tree.Tree, bo return nil, nil, "", "", fmt.Errorf("synthesis failed: %w", err) } totalUsage.Add(synthUsage) + answerText, calcs := calc.Resolve(answerText) + sources := make([]calc.Source, len(enriched)) + for i, e := range enriched { + sources[i] = calc.Source{Page: e.sec.PageStart, Text: e.content} + } + calc.Trace(calcs, sources) citations := make([]map[string]any, 0, len(enriched)) finalIDs := make([]tree.SectionID, 0, len(enriched)) @@ -390,15 +397,16 @@ func (h *AnswerHandler) answerFromSections(ctx context.Context, t *tree.Tree, bo traceToken := retrieval.ComputeTraceToken(body.DocumentID, "1", synthModel, finalIDs) resp := map[string]any{ - "document_id": body.DocumentID, - "query": body.Query, - "answer": answerText, - "citations": citations, - "strategy": retrieval.PublicName(h.strategy.Name()), - "model": synthModel, - "usage": usageMap(totalUsage), - "elapsed_ms": time.Since(started).Milliseconds(), - "trace_token": traceToken, + "document_id": body.DocumentID, + "query": body.Query, + "answer": answerText, + "citations": citations, + "calculations": nonNilCalcs(calcs), + "strategy": retrieval.PublicName(h.strategy.Name()), + "model": synthModel, + "usage": usageMap(totalUsage), + "elapsed_ms": time.Since(started).Milliseconds(), + "trace_token": traceToken, } return resp, finalIDs, synthModel, traceToken, nil } @@ -537,7 +545,8 @@ func synthesiseAnswer(ctx context.Context, client llmgate.Client, model, query s } } b.WriteString("\nWrite a concise answer to the user's query. ") - b.WriteString("If the evidence does not contain an answer, say so. ") + b.WriteString(calc.Instructions) + b.WriteString(" If the evidence does not contain an answer, say so. ") b.WriteString("Inline citations should reference the section_id values shown above. ") b.WriteString("Output plain prose; no JSON.") diff --git a/internal/handler/answer_calc_test.go b/internal/handler/answer_calc_test.go new file mode 100644 index 0000000..8b815aa --- /dev/null +++ b/internal/handler/answer_calc_test.go @@ -0,0 +1,72 @@ +package handler + +import ( + "context" + "log/slog" + "strings" + "testing" + "time" + + "github.com/hallelx2/llmgate" + + "github.com/hallelx2/vectorless-engine/pkg/calc" + enginecfg "github.com/hallelx2/vectorless-engine/pkg/config" + "github.com/hallelx2/vectorless-engine/pkg/pincite" + "github.com/hallelx2/vectorless-engine/pkg/retrieval" +) + +// The answer model sets a calculation up; the engine works it out and +// finds each input on the page that states it (HAL-3161). FinanceBench's +// PepsiCo FY2022 EBITDA margin, from the income statement and the cash +// flow statement. +func TestPageAnswerComputesItsCalculations(t *testing.T) { + var prompt string + llm := &llmgate.Mock{Respond: func(_ context.Context, req llmgate.Request) (*llmgate.Response, error) { + prompt = req.Messages[len(req.Messages)-1].Content + return &llmgate.Response{Content: "Unadjusted EBITDA margin was <<(11,512 + 2,763) / 86,392 * 100>>% [1][2]."}, nil + }} + h := NewAnswerHandler(slog.Default(), nil, nil, evidenceStrategy{}, llm, "m", + enginecfg.AnswerSpanBlock{}, enginecfg.AnswerBlock{}, nil) + res := &retrieval.Result{EvidencePages: []retrieval.EvidencePage{ + {Page: 62, Title: "Income statement", Text: "Net Revenue $ 86,392 $ 79,474. Operating Profit 11,512 11,162.", Confidence: 0.9}, + {Page: 64, Title: "Cash flows", Text: "Depreciation and amortization 2,763 2,710.", Confidence: 0.8}, + }} + resp, _, _, _, err := h.answerFromPages(context.Background(), queryStrategyTestTree(), pincite.Source{}, + answerRequest{DocumentID: "doc_x", Query: "PepsiCo FY2022 unadjusted EBITDA % margin?"}, res, time.Now(), func(string, any) {}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(prompt, "<>") { + t.Error("the answer prompt does not tell the model how to set a calculation up") + } + answer := resp["answer"].(string) + if !strings.Contains(answer, "16.5235 (= (11,512 + 2,763) / 86,392 * 100)%") { + t.Errorf("answer = %q, want the computed value beside its set-up", answer) + } + calcs := resp["calculations"].([]calc.Calculation) + if len(calcs) != 1 || calcs[0].Value == nil { + t.Fatalf("calculations = %+v", calcs) + } + ops := calcs[0].Operands + if !ops[0].Found || ops[0].Page != 62 || !ops[1].Found || ops[1].Page != 64 || !ops[2].Found || ops[2].Page != 62 { + t.Errorf("operands not traced to their pages: %+v", ops) + } + if ops[3].Found { + t.Errorf("100 is the formula's own constant, on no page: %+v", ops[3]) + } +} + +func TestPageAnswerWithoutCalculationsSaysSo(t *testing.T) { + llm := &llmgate.Mock{Reply: "3M spent $1,577 million [1]."} + h := NewAnswerHandler(slog.Default(), nil, nil, evidenceStrategy{}, llm, "m", + enginecfg.AnswerSpanBlock{}, enginecfg.AnswerBlock{}, nil) + res := &retrieval.Result{EvidencePages: []retrieval.EvidencePage{{Page: 39, Text: "Total Company 1,577 1,373"}}} + resp, _, _, _, err := h.answerFromPages(context.Background(), queryStrategyTestTree(), pincite.Source{}, + answerRequest{DocumentID: "doc_x", Query: "capex?"}, res, time.Now(), func(string, any) {}) + if err != nil { + t.Fatal(err) + } + if c := resp["calculations"].([]calc.Calculation); c == nil || len(c) != 0 { + t.Errorf("calculations = %#v, want an empty array", c) + } +} diff --git a/internal/handler/answer_pages.go b/internal/handler/answer_pages.go index 350f26b..d6e8520 100644 --- a/internal/handler/answer_pages.go +++ b/internal/handler/answer_pages.go @@ -11,6 +11,7 @@ import ( "github.com/hallelx2/llmgate" + "github.com/hallelx2/vectorless-engine/pkg/calc" "github.com/hallelx2/vectorless-engine/pkg/pincite" "github.com/hallelx2/vectorless-engine/pkg/retrieval" "github.com/hallelx2/vectorless-engine/pkg/tree" @@ -82,6 +83,14 @@ func (h *AnswerHandler) answerFromPages( return nil, nil, "", "", fmt.Errorf("synthesis failed: %w", err) } total.Add(synthUsage) + // The model set its calculations up; code works them out, and each + // operand is found on the evidence page that states it (HAL-3161). + answer, calcs := calc.Resolve(answer) + sources := make([]calc.Source, len(evidence)) + for i, ev := range evidence { + sources[i] = calc.Source{Page: ev.Page, Text: ev.Text} + } + calc.Trace(calcs, sources) answerMS := time.Since(started).Milliseconds() - retrievalMS var layout *pincite.Layout @@ -215,6 +224,7 @@ func (h *AnswerHandler) answerFromPages( "citations": citations, "pincites": pincites, "evidence": evList, + "calculations": nonNilCalcs(calcs), "markers_inline": inline, "markers_stripped": stripped, "strategy": retrieval.PublicName(h.strategy.Name()), @@ -233,6 +243,15 @@ func (h *AnswerHandler) answerFromPages( return resp, finalIDs, model, traceToken, nil } +// nonNilCalcs keeps "calculations" an array in the response when the answer +// set none up. +func nonNilCalcs(c []calc.Calculation) []calc.Calculation { + if c == nil { + return []calc.Calculation{} + } + return c +} + // publicSelector names how a pincite's sentence was chosen, without // exposing which model makes the choice: "model" or "overlap" (the // lexical fallback). @@ -283,7 +302,8 @@ func synthesiseFromPages(ctx context.Context, client llmgate.Client, model, quer b.WriteString("End every sentence that states a fact from the evidence with the marker of the evidence block it came from, like [2]; ") b.WriteString("use only the numbers shown above, and use two markers like [1][3] only when a sentence draws on two blocks. ") b.WriteString("Give figures exactly as the page states them, with their units and periods. ") - b.WriteString("If the evidence does not answer the question, say so plainly and do not guess.") + b.WriteString(calc.Instructions) + b.WriteString(" If the evidence does not answer the question, say so plainly and do not guess.") resp, err := client.Complete(ctx, llmgate.Request{ Model: model, diff --git a/pkg/calc/calc.go b/pkg/calc/calc.go new file mode 100644 index 0000000..12718c8 --- /dev/null +++ b/pkg/calc/calc.go @@ -0,0 +1,401 @@ +// Package calc does an answer's arithmetic in code. +// +// The answer model sets a calculation up from figures on the evidence pages +// and writes it as <>; Resolve evaluates every one and puts the +// value in its place, and Trace finds each operand on the evidence so the +// figure can be checked back to its page. A model that sets a calculation up +// correctly still gets the arithmetic wrong: on FinanceBench, +// gemini-2.5-flash-lite wrote 365 * 1,380.5 / 7,890 = 67.75, which is 63.86 +// (HAL-3161). +// +// An expression holds numbers, + - * /, parentheses and signs, nothing +// else. Numbers are read the way a filing writes them: "$1,174" is 1174, +// "12.5%" is 0.125, "(1,234)" stays a parenthesised 1234. +package calc + +import ( + "errors" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "unicode" +) + +// Instructions tell an answer model how to write a calculation so Resolve +// can do it. Every answer prompt that may derive a figure includes them. +const Instructions = "When the question asks for a figure the evidence lets you derive (a ratio, margin, growth rate, " + + "change, average or sum), take each input from the evidence and set the calculation up, but do not work it out: " + + "write it as <> using only numbers from the evidence, + - * / and parentheses, and it is computed " + + "for you. Example: operating margin was <<11,512 / 86,392 * 100>>% [1]. Apply the question's own definitions " + + "term by term. When the question asks for a judgement (is a business cyclical, what drove a change), answer it " + + "from the facts the evidence states and cite them." + +// ErrInvalid is any expression that is not plain arithmetic. +var ErrInvalid = errors.New("not an arithmetic expression") + +// Operand is one number an expression used, as it was written. +type Operand struct { + Text string `json:"text"` // as written, e.g. "$11,512" + Value float64 `json:"value"` // as read + // Evidence is the 1-based evidence block the number was found in; 0 + // when it is on none of them. + Evidence int `json:"evidence,omitempty"` + Page int `json:"page,omitempty"` + // Found is false when the number appears on no evidence block: the + // figure is the model's, not the document's. + Found bool `json:"found"` +} + +// Calculation is one evaluated <>. +type Calculation struct { + Expression string `json:"expression"` + Value *float64 `json:"value,omitempty"` + Display string `json:"display,omitempty"` + Error string `json:"error,omitempty"` + Operands []Operand `json:"operands,omitempty"` +} + +var exprRE = regexp.MustCompile(`(?s)<<(.+?)>>`) + +// Resolve replaces every <> in text with its value followed by +// the expression, "63.8634 (= 365 * ...)", so a reader sees both. One that +// does not evaluate stays visible with a note, never a made-up number. +func Resolve(text string) (string, []Calculation) { + var calcs []Calculation + out := exprRE.ReplaceAllStringFunc(text, func(m string) string { + expr := strings.TrimSpace(m[2 : len(m)-2]) + c := Calculation{Expression: expr} + ops, err := operands(expr) + if err == nil { + c.Operands = ops + } + v, err := Evaluate(expr) + if err != nil { + c.Error = err.Error() + calcs = append(calcs, c) + return expr + " (could not be computed)" + } + c.Value = &v + c.Display = Format(v) + calcs = append(calcs, c) + return c.Display + " (= " + expr + ")" + }) + return out, calcs +} + +// Source is one evidence block an operand may come from. +type Source struct { + Page int + Text string +} + +// Trace finds each operand of each calculation on the evidence, in order, +// and records the first block that states it. +func Trace(calcs []Calculation, sources []Source) { + norm := make([]string, len(sources)) + for i, s := range sources { + norm[i] = s.Text + } + for ci := range calcs { + for oi := range calcs[ci].Operands { + op := &calcs[ci].Operands[oi] + for i := range sources { + if containsNumber(norm[i], op) { + op.Evidence, op.Page, op.Found = i+1, sources[i].Page, true + break + } + } + } + } +} + +// Format writes a value the way a reader would: no float noise, at most +// four decimals, thousands separated. +func Format(v float64) string { + if v == math.Trunc(v) && math.Abs(v) < 1e15 { + return group(strconv.FormatInt(int64(v), 10)) + } + s := strconv.FormatFloat(v, 'f', 4, 64) + s = strings.TrimRight(strings.TrimRight(s, "0"), ".") + intPart, frac, _ := strings.Cut(s, ".") + out := group(intPart) + if frac != "" { + out += "." + frac + } + return out +} + +func group(digits string) string { + neg := strings.HasPrefix(digits, "-") + digits = strings.TrimPrefix(digits, "-") + var b strings.Builder + for i, r := range digits { + if i > 0 && (len(digits)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(r) + } + if neg { + return "-" + b.String() + } + return b.String() +} + +// --- the evaluator: a recursive-descent parser over tokens --- + +type tokKind int + +const ( + tNum tokKind = iota + tOp + tLParen + tRParen +) + +type token struct { + kind tokKind + op byte + num float64 + text string +} + +var numberRE = regexp.MustCompile(`^[$€£]?\s?(?:\d[\d,]*(?:\.\d+)?|\.\d+)%?`) + +func tokenize(expr string) ([]token, error) { + expr = strings.NewReplacer("×", "*", "÷", "/", "−", "-", "–", "-").Replace(expr) + var toks []token + for i := 0; i < len(expr); { + c := expr[i] + switch { + case c == ' ' || c == '\t' || c == '\n': + i++ + case c == '+' || c == '-' || c == '*' || c == '/': + toks = append(toks, token{kind: tOp, op: c}) + i++ + case c == '(': + toks = append(toks, token{kind: tLParen}) + i++ + case c == ')': + toks = append(toks, token{kind: tRParen}) + i++ + default: + m := numberRE.FindString(expr[i:]) + if m == "" { + return nil, fmt.Errorf("%w: unexpected %q", ErrInvalid, expr[i:min(len(expr), i+12)]) + } + v, err := readNumber(m) + if err != nil { + return nil, err + } + toks = append(toks, token{kind: tNum, num: v, text: strings.TrimSpace(m)}) + i += len(m) + } + } + if len(toks) == 0 { + return nil, fmt.Errorf("%w: empty", ErrInvalid) + } + return toks, nil +} + +func readNumber(s string) (float64, error) { + pct := strings.HasSuffix(s, "%") + clean := strings.Map(func(r rune) rune { + if unicode.IsDigit(r) || r == '.' { + return r + } + return -1 + }, s) + v, err := strconv.ParseFloat(clean, 64) + if err != nil { + return 0, fmt.Errorf("%w: number %q", ErrInvalid, s) + } + if pct { + v /= 100 + } + return v, nil +} + +type parser struct { + toks []token + pos int +} + +func (p *parser) peek() *token { + if p.pos < len(p.toks) { + return &p.toks[p.pos] + } + return nil +} + +// expr := term (("+"|"-") term)* +func (p *parser) expr() (float64, error) { + v, err := p.term() + if err != nil { + return 0, err + } + for t := p.peek(); t != nil && t.kind == tOp && (t.op == '+' || t.op == '-'); t = p.peek() { + p.pos++ + r, err := p.term() + if err != nil { + return 0, err + } + if t.op == '+' { + v += r + } else { + v -= r + } + } + return v, nil +} + +// term := unary (("*"|"/") unary)* +func (p *parser) term() (float64, error) { + v, err := p.unary() + if err != nil { + return 0, err + } + for t := p.peek(); t != nil && t.kind == tOp && (t.op == '*' || t.op == '/'); t = p.peek() { + p.pos++ + r, err := p.unary() + if err != nil { + return 0, err + } + if t.op == '*' { + v *= r + } else { + if r == 0 { + return 0, fmt.Errorf("%w: division by zero", ErrInvalid) + } + v /= r + } + } + return v, nil +} + +// unary := ("+"|"-") unary | atom +func (p *parser) unary() (float64, error) { + if t := p.peek(); t != nil && t.kind == tOp && (t.op == '+' || t.op == '-') { + p.pos++ + v, err := p.unary() + if t.op == '-' { + v = -v + } + return v, err + } + return p.atom() +} + +// atom := number | "(" expr ")" +func (p *parser) atom() (float64, error) { + t := p.peek() + if t == nil { + return 0, fmt.Errorf("%w: ends early", ErrInvalid) + } + switch t.kind { + case tNum: + p.pos++ + return t.num, nil + case tLParen: + p.pos++ + v, err := p.expr() + if err != nil { + return 0, err + } + if c := p.peek(); c == nil || c.kind != tRParen { + return 0, fmt.Errorf("%w: unclosed parenthesis", ErrInvalid) + } + p.pos++ + return v, nil + } + return 0, fmt.Errorf("%w: unexpected operator", ErrInvalid) +} + +// Evaluate is the value of one arithmetic expression. +func Evaluate(expr string) (float64, error) { + toks, err := tokenize(expr) + if err != nil { + return 0, err + } + p := &parser{toks: toks} + v, err := p.expr() + if err != nil { + return 0, err + } + if p.pos != len(p.toks) { + return 0, fmt.Errorf("%w: trailing input", ErrInvalid) + } + if math.IsNaN(v) || math.IsInf(v, 0) { + return 0, fmt.Errorf("%w: not a finite number", ErrInvalid) + } + return v, nil +} + +// operands lists every number an expression uses, in order. Constants a +// formula brings (2 for an average, 100 for a percentage, 365 for days) are +// listed too; Trace marks which numbers the evidence states. +func operands(expr string) ([]Operand, error) { + toks, err := tokenize(expr) + if err != nil { + return nil, err + } + var out []Operand + for _, t := range toks { + if t.kind == tNum { + out = append(out, Operand{Text: t.text, Value: t.num}) + } + } + return out, nil +} + +// containsNumber reports whether text states op's number: as written, or +// in the other common form (with or without thousands separators). +func containsNumber(text string, op *Operand) bool { + digits := strings.Map(func(r rune) rune { + if unicode.IsDigit(r) || r == '.' || r == ',' { + return r + } + return -1 + }, op.Text) + if digits == "" { + return false + } + bare := strings.ReplaceAll(digits, ",", "") + forms := []string{digits, bare} + if intPart, frac, ok := strings.Cut(bare, "."); ok { + forms = append(forms, group(intPart)+"."+frac) + } else { + forms = append(forms, group(bare)) + } + for _, f := range forms { + if f != "" && containsToken(text, f) { + return true + } + } + return false +} + +// containsToken finds f in text as a whole number, not inside a longer one +// ("12" is not in "1,234" or "312"). +func containsToken(text, f string) bool { + for start := 0; ; { + i := strings.Index(text[start:], f) + if i < 0 { + return false + } + i += start + before := i == 0 || !isNumRune(text[i-1]) + end := i + len(f) + after := end >= len(text) || !isNumRune(text[end]) || + // a sentence's full stop or a list comma after the number + ((text[end] == '.' || text[end] == ',') && (end+1 >= len(text) || !isDigit(text[end+1]))) + if before && after { + return true + } + start = i + 1 + } +} + +func isDigit(b byte) bool { return b >= '0' && b <= '9' } +func isNumRune(b byte) bool { return isDigit(b) || b == '.' || b == ',' } diff --git a/pkg/calc/calc_test.go b/pkg/calc/calc_test.go new file mode 100644 index 0000000..71a8684 --- /dev/null +++ b/pkg/calc/calc_test.go @@ -0,0 +1,112 @@ +package calc + +import ( + "errors" + "math" + "strings" + "testing" +) + +func TestEvaluate(t *testing.T) { + cases := []struct { + expr string + want float64 + }{ + {"2112 / 14082 * 100", 14.99787}, + // FinanceBench Amazon FY2017 DPO, set up as the question defines it + {"365 * ((34,616 + 25,309) / 2) / (111,934 + (16,047 - 11,461))", 93.8578}, + // Corning FY2020 DPO: the model wrote 67.75 for this + {"365 * ((1,174 + 1,587) / 2) / (7,772 + (2,438 - 2,320))", 63.8634}, + {"$1,174 + $1,587", 2761}, + {"12.5% * 200", 25}, + {"-3 + 5", 2}, + {"-(2 - 5) * 2", 6}, + {"6 × 7 ÷ 2 − 1", 20}, + {".5 + 1", 1.5}, + {"10 - 2 - 3", 5}, + {"64 / 4 / 2", 8}, + } + for _, c := range cases { + got, err := Evaluate(c.expr) + if err != nil { + t.Errorf("%q: %v", c.expr, err) + continue + } + if math.Abs(got-c.want) > 1e-3*math.Max(1, math.Abs(c.want)) { + t.Errorf("%q = %v, want %v", c.expr, got, c.want) + } + } +} + +func TestOnlyArithmeticEvaluates(t *testing.T) { + for _, expr := range []string{"", "1 +", "2 ** 3", "abs(-1)", "x + 1", "1 / 0", "(1 + 2", "1 + 2)", "os.exit(1)", "1 2"} { + if _, err := Evaluate(expr); !errors.Is(err, ErrInvalid) { + t.Errorf("%q: err = %v, want ErrInvalid", expr, err) + } + } +} + +func TestResolveShowsValueAndSetUp(t *testing.T) { + out, calcs := Resolve("Margin was <<11,512 / 86,392 * 100>>% [1], growth <<(86,392 - 79,474) / 79,474 * 100>>%, and <>.") + want := "Margin was 13.3253 (= 11,512 / 86,392 * 100)% [1], growth 8.7047 (= (86,392 - 79,474) / 79,474 * 100)%, and abs(1) (could not be computed)." + if out != want { + t.Fatalf("out =\n%s\nwant\n%s", out, want) + } + if len(calcs) != 3 || calcs[0].Value == nil || calcs[2].Value != nil || calcs[2].Error == "" { + t.Fatalf("calcs = %+v", calcs) + } + if n := len(calcs[0].Operands); n != 3 || calcs[0].Operands[0].Text != "11,512" || calcs[0].Operands[2].Value != 100 { + t.Errorf("operands = %+v", calcs[0].Operands) + } +} + +func TestTraceFindsEachOperandOnItsPage(t *testing.T) { + _, calcs := Resolve("<<(11,512 + 2763) / 86392 * 100>>") + Trace(calcs, []Source{ + {Page: 62, Text: "Net revenue 86,392 79,474. Operating profit 11,512 11,162."}, + {Page: 64, Text: "Depreciation and other amortization 2,763 2,710"}, + }) + ops := calcs[0].Operands + if !ops[0].Found || ops[0].Page != 62 || ops[0].Evidence != 1 { + t.Errorf("11,512: %+v", ops[0]) + } + if !ops[1].Found || ops[1].Page != 64 { + t.Errorf("2763 written without a separator, stated as 2,763: %+v", ops[1]) + } + if !ops[2].Found || ops[2].Page != 62 { + t.Errorf("86392 stated as 86,392: %+v", ops[2]) + } + if ops[3].Found { + t.Errorf("100 is the formula's, on no page: %+v", ops[3]) + } +} + +func TestTraceMatchesWholeNumbersOnly(t *testing.T) { + _, calcs := Resolve("<<12 + 3.5>>") + Trace(calcs, []Source{{Page: 1, Text: "Revenue 1,234 and 312; rate 13.55. Units 12. Share 3.5."}}) + if !calcs[0].Operands[0].Found { + t.Error("12 is stated (\"Units 12.\")") + } + if !calcs[0].Operands[1].Found { + t.Error("3.5 is stated (\"Share 3.5.\")") + } + _, miss := Resolve("<<12>>") + Trace(miss, []Source{{Page: 1, Text: "Revenue 1,234 and 312; 120 units."}}) + if miss[0].Operands[0].Found { + t.Error("12 matched inside a longer number") + } +} + +func TestFormat(t *testing.T) { + for v, want := range map[float64]string{1234: "1,234", 0.1 + 0.2: "0.3", -3.70004: "-3.7", 1234567.891: "1,234,567.891", -1500: "-1,500", 63.86343: "63.8634"} { + if got := Format(v); got != want { + t.Errorf("Format(%v) = %q, want %q", v, got, want) + } + } +} + +func TestInstructionsShowTheNotation(t *testing.T) { + if !strings.Contains(Instructions, "<>") || !strings.Contains(Instructions, "<<11,512 / 86,392 * 100>>") { + t.Error("the instructions must show the model the notation Resolve reads") + } +} From 9717705fcbcccff5ac69f9e5c8f3de8b0538963c Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 17 Oct 2026 06:55:54 +0100 Subject: [PATCH 2/4] fix(calc): read thousands separators only in real groups --- pkg/calc/calc.go | 6 +++++- pkg/calc/calc_test.go | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/calc/calc.go b/pkg/calc/calc.go index 12718c8..bff43ea 100644 --- a/pkg/calc/calc.go +++ b/pkg/calc/calc.go @@ -161,7 +161,11 @@ type token struct { text string } -var numberRE = regexp.MustCompile(`^[$€£]?\s?(?:\d[\d,]*(?:\.\d+)?|\.\d+)%?`) +// numberRE reads one number: an optional currency sign (a space may follow +// it), digits with thousands separators only in real groups ("1,174"; +// "1,2" is not a number and fails rather than reading as 12), decimals and +// a percent sign. +var numberRE = regexp.MustCompile(`^(?:[$€£]\s?)?(?:(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?|\.\d+)%?`) func tokenize(expr string) ([]token, error) { expr = strings.NewReplacer("×", "*", "÷", "/", "−", "-", "–", "-").Replace(expr) diff --git a/pkg/calc/calc_test.go b/pkg/calc/calc_test.go index 71a8684..57d0f7c 100644 --- a/pkg/calc/calc_test.go +++ b/pkg/calc/calc_test.go @@ -25,6 +25,8 @@ func TestEvaluate(t *testing.T) { {".5 + 1", 1.5}, {"10 - 2 - 3", 5}, {"64 / 4 / 2", 8}, + {"$ 1,587 - 12,345.5", -10758.5}, + {"1174", 1174}, } for _, c := range cases { got, err := Evaluate(c.expr) @@ -39,7 +41,7 @@ func TestEvaluate(t *testing.T) { } func TestOnlyArithmeticEvaluates(t *testing.T) { - for _, expr := range []string{"", "1 +", "2 ** 3", "abs(-1)", "x + 1", "1 / 0", "(1 + 2", "1 + 2)", "os.exit(1)", "1 2"} { + for _, expr := range []string{"", "1 +", "2 ** 3", "abs(-1)", "x + 1", "1 / 0", "(1 + 2", "1 + 2)", "os.exit(1)", "1 2", "5 3", "1, 2", "1,2", "1,174,5"} { if _, err := Evaluate(expr); !errors.Is(err, ErrInvalid) { t.Errorf("%q: err = %v, want ErrInvalid", expr, err) } From a85e430f6f2179689082338a33e11236e87295bd Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 17 Oct 2026 07:05:21 +0100 Subject: [PATCH 3/4] fix(calc): keep a unary sign on its operand --- pkg/calc/calc.go | 27 ++++++++++++++++++++++++--- pkg/calc/calc_test.go | 17 +++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pkg/calc/calc.go b/pkg/calc/calc.go index bff43ea..2a0281c 100644 --- a/pkg/calc/calc.go +++ b/pkg/calc/calc.go @@ -345,14 +345,35 @@ func operands(expr string) ([]Operand, error) { return nil, err } var out []Operand - for _, t := range toks { - if t.kind == tNum { - out = append(out, Operand{Text: t.text, Value: t.num}) + for i, t := range toks { + if t.kind != tNum { + continue } + // A unary sign belongs to the operand: "-12" is minus twelve, not + // twelve. Trace still finds it by magnitude, because a filing + // prints a negative as "(12)" as often as "-12". + text, value := t.text, t.num + for j := i - 1; unarySign(toks, j); j-- { + text = string(toks[j].op) + text + if toks[j].op == '-' { + value = -value + } + } + out = append(out, Operand{Text: text, Value: value}) } return out, nil } +// unarySign reports whether toks[i] is a sign on what follows it rather +// than a binary operator: it opens the expression or follows another +// operator or an opening parenthesis. +func unarySign(toks []token, i int) bool { + if i < 0 || toks[i].kind != tOp || (toks[i].op != '+' && toks[i].op != '-') { + return false + } + return i == 0 || toks[i-1].kind == tOp || toks[i-1].kind == tLParen +} + // containsNumber reports whether text states op's number: as written, or // in the other common form (with or without thousands separators). func containsNumber(text string, op *Operand) bool { diff --git a/pkg/calc/calc_test.go b/pkg/calc/calc_test.go index 57d0f7c..0bdc471 100644 --- a/pkg/calc/calc_test.go +++ b/pkg/calc/calc_test.go @@ -2,6 +2,7 @@ package calc import ( "errors" + "fmt" "math" "strings" "testing" @@ -112,3 +113,19 @@ func TestInstructionsShowTheNotation(t *testing.T) { t.Error("the instructions must show the model the notation Resolve reads") } } + +func TestOperandsKeepTheirSign(t *testing.T) { + _, calcs := Resolve("<<-1,234 + (-(5)) - 3 * -2>>") + got := []string{} + for _, o := range calcs[0].Operands { + got = append(got, fmt.Sprintf("%s=%v", o.Text, o.Value)) + } + if strings.Join(got, " ") != "-1,234=-1234 5=5 3=3 -2=-2" { + t.Errorf("operands = %v", got) + } + // Found by magnitude: the page prints the loss in parentheses. + Trace(calcs, []Source{{Page: 9, Text: "Net loss (1,234) for the year."}}) + if !calcs[0].Operands[0].Found || calcs[0].Operands[0].Page != 9 { + t.Errorf("-1,234 not found as (1,234): %+v", calcs[0].Operands[0]) + } +} From 0a86ed108250141c5df34dd50e2918a3a039f1ae Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 17 Oct 2026 07:11:08 +0100 Subject: [PATCH 4/4] style(calc): tagged switch in the tokenizer --- pkg/calc/calc.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/calc/calc.go b/pkg/calc/calc.go index 2a0281c..205bf14 100644 --- a/pkg/calc/calc.go +++ b/pkg/calc/calc.go @@ -172,16 +172,16 @@ func tokenize(expr string) ([]token, error) { var toks []token for i := 0; i < len(expr); { c := expr[i] - switch { - case c == ' ' || c == '\t' || c == '\n': + switch c { + case ' ', '\t', '\n': i++ - case c == '+' || c == '-' || c == '*' || c == '/': + case '+', '-', '*', '/': toks = append(toks, token{kind: tOp, op: c}) i++ - case c == '(': + case '(': toks = append(toks, token{kind: tLParen}) i++ - case c == ')': + case ')': toks = append(toks, token{kind: tRParen}) i++ default: