diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts index a037858..a61d812 100644 --- a/src/lib/schemas.ts +++ b/src/lib/schemas.ts @@ -37,27 +37,29 @@ export const AnalysisResponseSchema = z.object({ approachDescription: z.string().default(''), strengths: z.array(z.string()).default([]), inefficiencies: z.array(z.string()).default([]), - decisionQuality: z.number().default(0), + // LLM analyzers intermittently emit numbers as strings ("85" not 85) — + // coerce instead of rejecting the whole analysis (parseFailed) on that. + decisionQuality: z.coerce.number().default(0), }).default({ approach: 'exploratory', approachDescription: '', strengths: [], inefficiencies: [], decisionQuality: 0 }), strategy: z.object({ - reconQuality: z.number().default(0), - exploitEfficiency: z.number().default(0), - adaptability: z.number().default(0), - overallScore: z.number().optional(), + reconQuality: z.coerce.number().default(0), + exploitEfficiency: z.coerce.number().default(0), + adaptability: z.coerce.number().default(0), + overallScore: z.coerce.number().optional(), scoreBreakdown: z.string().default(''), }).default({ reconQuality: 0, exploitEfficiency: 0, adaptability: 0, scoreBreakdown: '' }), rubricEvaluation: z.object({ milestones: z.array(z.object({ id: z.string(), - achieved: z.boolean(), + achieved: z.coerce.boolean(), reasoning: z.string(), })).default([]), qualitative: z.object({ - reconQuality: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - techniqueSelection: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), - adaptability: z.object({ score: z.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + reconQuality: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + techniqueSelection: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), + adaptability: z.object({ score: z.coerce.number(), reasoning: z.string() }).default({ score: 0, reasoning: '' }), }).default({ reconQuality: { score: 0, reasoning: '' }, techniqueSelection: { score: 0, reasoning: '' }, diff --git a/tests/schemas-coercion.test.ts b/tests/schemas-coercion.test.ts new file mode 100644 index 0000000..ff7a081 --- /dev/null +++ b/tests/schemas-coercion.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { AnalysisResponseSchema } from '../src/lib/schemas'; + +// Regression: string-quoted numbers from LLM analyzers must not reject the +// whole analysis (parseFailed). Real-world instances observed with +// DeepSeek-V3 analyzer on long transcripts (oasis-ai-runs sweeps, Sep-Oct 2026): +// e.g. {"...","strategy":{"reconQuality":"80",...}} => invalid_type => KSM lost +// despite flag capture. + +const baseGood = { + attackChain: { phases: [], techniques: [], killChainCoverage: [] }, + narrative: { summary: 's', detailed: 'd', keyFindings: [] }, +}; + +describe('AnalysisResponseSchema numeric-string coercion', () => { + it('accepts numbers as strings in strategy scores (regression)', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: '85' }, + strategy: { reconQuality: '80', exploitEfficiency: '75', adaptability: '90', overallScore: '82' }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.strategy.overallScore).toBe(82); + expect(parsed.behavior.decisionQuality).toBe(85); + }); + + it('accepts boolean-as-string in rubric milestones', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + rubricEvaluation: { + milestones: [{ id: 'flag_captured', achieved: 'true', reasoning: 'r' }], + qualitative: { reconQuality: { score: '15', reasoning: 'r' } }, + }, + }); + expect(parsed.rubricEvaluation?.milestones[0].achieved).toBe(true); + expect(parsed.rubricEvaluation?.qualitative.reconQuality.score).toBe(15); + }); + + it('still passes through plain numbers unchanged', () => { + const parsed = AnalysisResponseSchema.parse({ + ...baseGood, + behavior: { decisionQuality: 85 }, + strategy: { reconQuality: 80 }, + }); + expect(parsed.strategy.reconQuality).toBe(80); + expect(parsed.behavior.decisionQuality).toBe(85); + }); +});