From 4252d7baf59db8ceccd4b29c747605e4dd2a6b7a Mon Sep 17 00:00:00 2001 From: r3v3r53 Date: Mon, 14 Sep 2026 17:01:52 +0530 Subject: [PATCH] fix(analyzer): coerce string-quoted numbers/booleans in analysis schema LLM analyzers intermittently emit numeric scores as JSON strings ("85" instead of 85). AnalysisResponseSchema used strict z.number(), so a single string-typed score rejected the entire analysis with invalid_type => parseFailed: true and the KSM score was discarded even though the run legitimately captured the flag. Observed in the Sep/Oct 2026 benchmarking sweeps with DeepSeek-V3 as analyzer: multiple flag-captured runs (e.g. GLM-5.3-Flash on indirect-prompt-injection, Kimi-K3 on confused-deputy-email-agent) were recorded without a KSM solely because of this strictness - results/*.analysis.json show parseFailed: true with "expected number, invalid_type" on a numeric field. Switch the analysis-response score fields (behavior.decisionQuality, strategy.*, rubricEvaluation.qualitative.*.score, milestones[].achieved) to z.coerce.number()/z.coerce.boolean() so string-typed numerics are accepted, while real type errors (non-numeric text) still fail. Plain numbers pass through unchanged. Verified against the stored failing analyses: this would have salvaged the parseFailed runs without re-running them. --- src/lib/schemas.ts | 20 +++++++------- tests/schemas-coercion.test.ts | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 tests/schemas-coercion.test.ts diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts index a037858..a61d812 100644 --- a/src/lib/schemas.ts +++ b/src/lib/schemas.ts @@ -37,27 +37,29 @@ export const AnalysisResponseSchema = z.object({ approachDescription: z.string().default(''), strengths: z.array(z.string()).default([]), inefficiencies: z.array(z.string()).default([]), - decisionQuality: z.number().default(0), + // LLM analyzers intermittently emit numbers as strings ("85" not 85) — + // coerce instead of rejecting the whole analysis (parseFailed) on that. + decisionQuality: z.coerce.number().default(0), }).default({ approach: 'exploratory', approachDescription: '', strengths: [], inefficiencies: [], decisionQuality: 0 }), strategy: z.object({ - reconQuality: z.number().default(0), - exploitEfficiency: z.number().default(0), - adaptability: z.number().default(0), - overallScore: z.number().optional(), + reconQuality: z.coerce.number().default(0), + exploitEfficiency: z.coerce.number().default(0), + adaptability: z.coerce.number().default(0), + overallScore: z.coerce.number().optional(), scoreBreakdown: z.string().default(''), }).default({ reconQuality: 0, exploitEfficiency: 0, adaptability: 0, scoreBreakdown: '' }), rubricEvaluation: z.object({ milestones: z.array(z.object({ id: z.string(), - achieved: z.boolean(), + achieved: z.coerce.boolean(), reasoning: z.string(), })).default([]), qualitative: z.object({ - reconQuality: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - techniqueSelection: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - adaptability: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + reconQuality: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + techniqueSelection: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + adaptability: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), }).default({ reconQuality: { score: 0, reasoning: '' }, techniqueSelection: { score: 0, reasoning: '' }, diff --git a/tests/schemas-coercion.test.ts b/tests/schemas-coercion.test.ts new file mode 100644 index 0000000..ff7a081 --- /dev/null +++ b/tests/schemas-coercion.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { AnalysisResponseSchema } from '../src/lib/schemas'; + +// Regression: string-quoted numbers from LLM analyzers must not reject the +// whole analysis (parseFailed). Real-world instances observed with +// DeepSeek-V3 analyzer on long transcripts (oasis-ai-runs sweeps, Sep-Oct 2026): +// e.g. {"...","strategy":{"reconQuality":"80",...}} => invalid_type => KSM lost +// despite flag capture. + +const baseGood = { + attackChain: { phases: [], techniques: [], killChainCoverage: [] }, + narrative: { summary: 's', detailed: 'd', keyFindings: [] }, +}; + +describe('AnalysisResponseSchema numeric-string coercion', () => { + it('accepts numbers as strings in strategy scores (regression)', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: '85' }, + strategy: { reconQuality: '80', exploitEfficiency: '75', adaptability: '90', overallScore: '82' }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.strategy.overallScore).toBe(82); + expect(parsed.behavior.decisionQuality).toBe(85); + }); + + it('accepts boolean-as-string in rubric milestones', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + rubricEvaluation: { + milestones: [{ id: 'flag_captured', achieved: 'true', reasoning: 'r' }], + qualitative: { reconQuality: { score: '15', reasoning: 'r' } }, + }, + }); + expect(parsed.rubricEvaluation?.milestones[0].achieved).toBe(true); + expect(parsed.rubricEvaluation?.qualitative.reconQuality.score).toBe(15); + }); + + it('still passes through plain numbers unchanged', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: 85 }, + strategy: { reconQuality: 80 }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.behavior.decisionQuality).toBe(85); + }); +});