diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts index a037858..dcc8821 100644 --- a/src/lib/schemas.ts +++ b/src/lib/schemas.ts @@ -11,9 +11,27 @@ export const AnalysisResponseSchema = z.object({ attackChain: z.object({ phases: z.array(z.object({ phase: z.string(), - stepRange: z.tuple([z.number(), z.number()]), - description: z.string(), - techniques: z.array(z.string()), + // Analyzers frequently emit a single-number range [3] or "3" instead of + // [3, 7] — tolerate both (mirror to [n, n]) instead of rejecting the + // whole analysis (observed in 17/21 parseFailed analyses, Sep/Oct 2026). + stepRange: z.preprocess( + (v) => { + if (Array.isArray(v)) { + const nums = v.map(x => (typeof x === 'number' ? x : Number(x))).filter(n => !Number.isNaN(n)); + if (nums.length === 0) return [0, 0]; + if (nums.length === 1) return [nums[0], nums[0]]; + return [nums[0], Math.max(...nums)]; + } + if (typeof v === 'number' || typeof v === 'string') { + const n = Number(v); + return Number.isNaN(n) ? [0, 0] : [n, n]; + } + return [0, 0]; + }, + z.tuple([z.number(), z.number()]) + ), + description: z.string().default(''), + techniques: z.array(z.string()).default([]), })).default([]), techniques: z.array(z.object({ id: z.string(), @@ -37,27 +55,29 @@ export const AnalysisResponseSchema = z.object({ approachDescription: z.string().default(''), strengths: z.array(z.string()).default([]), inefficiencies: z.array(z.string()).default([]), - decisionQuality: z.number().default(0), + // LLM analyzers intermittently emit numbers as strings ("85" not 85) — + // coerce instead of rejecting the whole analysis (parseFailed) on that. + decisionQuality: z.coerce.number().default(0), }).default({ approach: 'exploratory', approachDescription: '', strengths: [], inefficiencies: [], decisionQuality: 0 }), strategy: z.object({ - reconQuality: z.number().default(0), - exploitEfficiency: z.number().default(0), - adaptability: z.number().default(0), - overallScore: z.number().optional(), + reconQuality: z.coerce.number().default(0), + exploitEfficiency: z.coerce.number().default(0), + adaptability: z.coerce.number().default(0), + overallScore: z.coerce.number().optional(), scoreBreakdown: z.string().default(''), }).default({ reconQuality: 0, exploitEfficiency: 0, adaptability: 0, scoreBreakdown: '' }), rubricEvaluation: z.object({ milestones: z.array(z.object({ id: z.string(), - achieved: z.boolean(), + achieved: z.coerce.boolean(), reasoning: z.string(), })).default([]), qualitative: z.object({ - reconQuality: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - techniqueSelection: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - adaptability: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + reconQuality: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + techniqueSelection: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + adaptability: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), }).default({ reconQuality: { score: 0, reasoning: '' }, techniqueSelection: { score: 0, reasoning: '' }, diff --git a/tests/schemas-coercion.test.ts b/tests/schemas-coercion.test.ts new file mode 100644 index 0000000..ff7a081 --- /dev/null +++ b/tests/schemas-coercion.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { AnalysisResponseSchema } from '../src/lib/schemas'; + +// Regression: string-quoted numbers from LLM analyzers must not reject the +// whole analysis (parseFailed). Real-world instances observed with +// DeepSeek-V3 analyzer on long transcripts (oasis-ai-runs sweeps, Sep-Oct 2026): +// e.g. {"...","strategy":{"reconQuality":"80",...}} => invalid_type => KSM lost +// despite flag capture. + +const baseGood = { + attackChain: { phases: [], techniques: [], killChainCoverage: [] }, + narrative: { summary: 's', detailed: 'd', keyFindings: [] }, +}; + +describe('AnalysisResponseSchema numeric-string coercion', () => { + it('accepts numbers as strings in strategy scores (regression)', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: '85' }, + strategy: { reconQuality: '80', exploitEfficiency: '75', adaptability: '90', overallScore: '82' }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.strategy.overallScore).toBe(82); + expect(parsed.behavior.decisionQuality).toBe(85); + }); + + it('accepts boolean-as-string in rubric milestones', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + rubricEvaluation: { + milestones: [{ id: 'flag_captured', achieved: 'true', reasoning: 'r' }], + qualitative: { reconQuality: { score: '15', reasoning: 'r' } }, + }, + }); + expect(parsed.rubricEvaluation?.milestones[0].achieved).toBe(true); + expect(parsed.rubricEvaluation?.qualitative.reconQuality.score).toBe(15); + }); + + it('still passes through plain numbers unchanged', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: 85 }, + strategy: { reconQuality: 80 }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.behavior.decisionQuality).toBe(85); + }); +});