Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/lib/analyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
fallbackOverallScore,
} from './scoring.js';
import { AnalysisResponseSchema } from './schemas.js';
import { MAX_COMPLETION_TOKENS, ANALYZER_OUTPUT_LIMIT } from './constants.js';
import { MAX_COMPLETION_TOKENS, ANALYZER_OUTPUT_LIMIT, ANALYZER_TIMEOUT_MS } from './constants.js';
import { withRateLimitRetry } from './retry.js';
import { isAnthropicProvider, resolveProvider } from './providers.js';
import { normalizeProvider } from './config.js';
Expand Down Expand Up @@ -501,6 +501,8 @@ async function callAnthropicAnalyzer(
messages: [{ role: 'user', content: prompt }],
}),
'Analysis',
false,
ANALYZER_TIMEOUT_MS,
);
const textContent = response.content.find(c => c.type === 'text');
if (!textContent || textContent.type !== 'text') {
Expand All @@ -524,6 +526,8 @@ async function callOpenAIAnalyzer(
],
}),
'Analysis',
false,
ANALYZER_TIMEOUT_MS,
);
const content = response.choices[0]?.message?.content;
if (!content) {
Expand Down
6 changes: 6 additions & 0 deletions src/lib/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ export const DOCKER_WAIT_TIMEOUT = 30_000;
export const DOCKER_POLL_INTERVAL = 2_000;
export const DOCKER_STARTUP_POLL = 2_500;

// Analyzer LLM call timeout. The analyzer prompt embeds a full attack chain
// (up to 45 steps), which can be far larger than a benchmark step, so it needs
// more headroom than the generic 120s API default. Long-running benchmarks were
// losing KSM scores to "Analysis: timed out after 120s" on complex transcripts.
export const ANALYZER_TIMEOUT_MS = 300_000; // 5 minutes

// Display
export const VERBOSE_OUTPUT_PREVIEW = 2_000;

Expand Down
20 changes: 20 additions & 0 deletions tests/unit/analyzer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -339,3 +339,23 @@ describe('parseAnalysisResponse', () => {
expect(result.strategy.overallScore).toBe(42);
});
});

// =============================================================================
// Analyzer timeout budget
// =============================================================================

describe('ANALYZER_TIMEOUT_MS', () => {
it('is larger than the generic API timeout (analyses embed full attack chains)', async () => {
const { ANALYZER_TIMEOUT_MS } = await import('../../src/lib/constants.js');
const { DEFAULT_API_TIMEOUT_MS } = await import('../../src/lib/retry.js');

expect(ANALYZER_TIMEOUT_MS).toBeGreaterThan(DEFAULT_API_TIMEOUT_MS);
});

it('is a positive finite number of milliseconds', async () => {
const { ANALYZER_TIMEOUT_MS } = await import('../../src/lib/constants.js');

expect(Number.isFinite(ANALYZER_TIMEOUT_MS)).toBe(true);
expect(ANALYZER_TIMEOUT_MS).toBeGreaterThan(0);
});
});
Loading