diff --git a/src/__tests__/analyze-dashboard-events.test.tsx b/src/__tests__/analyze-dashboard-events.test.tsx index 2a9c4e20..7866d14f 100644 --- a/src/__tests__/analyze-dashboard-events.test.tsx +++ b/src/__tests__/analyze-dashboard-events.test.tsx @@ -185,7 +185,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { ({ role, name, status, message, expectedIcon, expectedRoleIcon }) => { const { unmount } = render( , - { stdout: mockStdout as any } + { stdout: mockStdout as any, interactive: true } ); const output = mockStdout.frames.join('\n'); @@ -202,6 +202,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { it('returns null when no active tool is executing', () => { const { unmount } = render(, { stdout: mockStdout as any, + interactive: false, }); const output = mockStdout.frames.join('\n'); @@ -212,6 +213,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { it('displays idle message when showIdle is enabled and tool is null', () => { const { unmount } = render(, { stdout: mockStdout as any, + interactive: true, }); const output = mockStdout.frames.join('\n'); @@ -230,7 +232,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { status: 'running', }} />, - { stdout: mockStdout as any } + { stdout: mockStdout as any, interactive: true } ); const output = mockStdout.frames.join('\n'); @@ -252,7 +254,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { status: 'completed', }} />, - { stdout: mockStdout as any } + { stdout: mockStdout as any, interactive: true } ); const output = mockStdout.frames.join('\n'); @@ -268,7 +270,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { it('renders structured consensus with root cause, confidence badge, findings, and fix command', () => { const { unmount } = render( , - { stdout: mockStdout as any } + { stdout: mockStdout as any, interactive: true } ); const output = mockStdout.frames.join('\n'); @@ -290,7 +292,7 @@ describe('Phase 13: Live NDJSON Streaming UI Components', () => { const detailsText = 'Root Cause (HIGH confidence):\n Node pressure\nRecommended Remediation:\n Scale up node pool'; const { unmount } = render( , - { stdout: mockStdout as any } + { stdout: mockStdout as any, interactive: true } ); const output = mockStdout.frames.join('\n'); diff --git a/src/__tests__/cli-flags.test.ts b/src/__tests__/cli-flags.test.ts index 3ef0776f..13361556 100644 --- a/src/__tests__/cli-flags.test.ts +++ b/src/__tests__/cli-flags.test.ts @@ -4,6 +4,7 @@ import { runAnalysis } from '../analysis/analysis'; import { registerAnalyzeCommand } from '../commands/analyze'; import { registerConfigCommand } from '../commands/config'; import * as pythonBridge from '../agent/python-bridge'; +import * as probesModule from '../analysis/probes'; import { clearConfig, getConfig, setMultiAgentConfig } from '../config/store'; import { logger } from '../utils/logger'; import type { ConsensusDiagnosis } from '../agent/types'; @@ -77,6 +78,11 @@ describe('Phase 23 - Feature Flags, Rollout Modes & Legacy Fallback (Issue #294) delete process.env.KDM_MULTI_AGENT; vi.mocked(pythonBridge.runAgentCouncilProcess).mockResolvedValue(MOCK_CONSENSUS); + vi.spyOn(probesModule, 'checkCouncilDependencies').mockResolvedValue({ + pythonAvailable: true, + ollamaAvailable: true, + canRunCouncil: true, + }); }); afterEach(() => { diff --git a/src/__tests__/e2e-system.test.ts b/src/__tests__/e2e-system.test.ts index 4d4a637c..570c9b5c 100644 --- a/src/__tests__/e2e-system.test.ts +++ b/src/__tests__/e2e-system.test.ts @@ -291,6 +291,11 @@ describe('Phase 24: Final System Architecture Integration & E2E Validation', () } as any); vi.spyOn(pythonBridge, 'runAgentCouncilProcess').mockResolvedValue(CANONICAL_CONSENSUS); + vi.spyOn(probesModule, 'checkCouncilDependencies').mockResolvedValue({ + pythonAvailable: true, + ollamaAvailable: true, + canRunCouncil: true, + }); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); diff --git a/src/__tests__/safety-prompt.test.ts b/src/__tests__/safety-prompt.test.ts index ce93edf6..71ace87d 100644 --- a/src/__tests__/safety-prompt.test.ts +++ b/src/__tests__/safety-prompt.test.ts @@ -91,6 +91,7 @@ async function setupPromptHarness( { stdout: stdout as any, stdin: stdin as any, + interactive: true, } ); @@ -119,6 +120,7 @@ describe('SafetyPrompt Component & Interactive Safety Gate', () => { { stdout: mockStdout as any, stdin: mockStdin as any, + interactive: true, } ); @@ -152,6 +154,7 @@ describe('SafetyPrompt Component & Interactive Safety Gate', () => { { stdout: mockStdout as any, stdin: mockStdin as any, + interactive: true, } ); @@ -241,6 +244,7 @@ describe('SafetyPrompt Component & Interactive Safety Gate', () => { const promise = promptSafetyConfirmation(SAMPLE_PROPOSAL, { stdin: mockStdin as any, stdout: mockStdout as any, + interactive: true, }); await waitForFrame(mockStdout, 'Apply this remediation to the cluster?'); @@ -254,6 +258,7 @@ describe('SafetyPrompt Component & Interactive Safety Gate', () => { const promise = promptSafetyConfirmation(SAMPLE_PROPOSAL, { stdin: mockStdin as any, stdout: mockStdout as any, + interactive: true, }); await waitForFrame(mockStdout, 'Apply this remediation to the cluster?'); diff --git a/src/agent/python-bridge.ts b/src/agent/python-bridge.ts index 8a8cc81a..b2ed34e9 100644 --- a/src/agent/python-bridge.ts +++ b/src/agent/python-bridge.ts @@ -17,7 +17,7 @@ import { fileURLToPath } from 'node:url'; import { parseNDJSONLine } from './event-schema'; import type { ConsensusDiagnosis, AgentProgressEvent, NDJSONEvent } from './types'; -const DEFAULT_TIMEOUT_MS = 45_000; +export const DEFAULT_TIMEOUT_MS = 120_000; const SIGKILL_GRACE_PERIOD_MS = 1_500; const DEFAULT_MODEL = 'llama3.1'; const DEFAULT_BASE_URL = 'http://localhost:11434'; @@ -38,7 +38,7 @@ export interface CouncilBridgeOptions { model?: string; /** Base URL for Ollama API (default: 'http://localhost:11434'). */ baseUrl?: string; - /** Maximum execution duration in milliseconds before watchdog triggers SIGTERM/SIGKILL (default: 45000). */ + /** Maximum execution duration in milliseconds before watchdog triggers SIGTERM/SIGKILL. */ timeoutMs?: number; /** Callback invoked on parsed NDJSON streaming events. */ onEvent?: (event: NDJSONEvent) => void; @@ -50,6 +50,16 @@ export interface CouncilBridgeOptions { spawnFn?: typeof spawn; } +/** Resolves a valid per-call, environment, or default council timeout. */ +export function resolveCouncilTimeoutMs(timeoutMs?: number): number { + if (Number.isSafeInteger(timeoutMs) && timeoutMs! > 0) return timeoutMs!; + + const fromEnvironment = Number(process.env.KDM_COUNCIL_TIMEOUT_MS); + if (Number.isSafeInteger(fromEnvironment) && fromEnvironment > 0) return fromEnvironment; + + return DEFAULT_TIMEOUT_MS; +} + /** * Execution parameters for the Python agent council (backward-compatibility alias). */ @@ -368,7 +378,7 @@ export async function runAgentCouncilProcess( ): Promise { const scriptPath = getCouncilScriptPath(); const spawnImpl = options.spawnFn || spawn; - const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const timeoutMs = resolveCouncilTimeoutMs(options.timeoutMs); const payload = JSON.stringify({ failureText: options.failureText, diff --git a/src/analysis/__tests__/analysis-baseline.test.ts b/src/analysis/__tests__/analysis-baseline.test.ts index b987f918..ed574b0c 100644 --- a/src/analysis/__tests__/analysis-baseline.test.ts +++ b/src/analysis/__tests__/analysis-baseline.test.ts @@ -12,7 +12,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { runAnalysis, explainSingleResult, resolveBackend } from '../analysis'; import { clearConfig, setAIConfig } from '../../config/store'; -import { isPythonAgentAvailable, runPythonAgentCouncil } from '../../agent/python-bridge'; +import { isPythonAgentAvailable, runAgentCouncilProcess, runPythonAgentCouncil } from '../../agent/python-bridge'; +import * as probesModule from '../probes'; import { registry, PodAnalyzer, @@ -83,6 +84,7 @@ vi.mock('../../ai/factory', () => ({ })); vi.mock('../../agent/python-bridge', () => ({ + runAgentCouncilProcess: vi.fn(), runPythonAgentCouncil: vi.fn(), isPythonAgentAvailable: vi.fn(async () => false), })); @@ -108,6 +110,11 @@ let originalMultiAgentEnv: string | undefined; describe('Baseline Pipeline Regression Tests', () => { beforeEach(() => { + vi.spyOn(probesModule, 'checkCouncilDependencies').mockResolvedValue({ + pythonAvailable: true, + ollamaAvailable: true, + canRunCouncil: true, + }); clearConfig(); registry.clear(); registry.register(PodAnalyzer); @@ -299,7 +306,12 @@ describe('Baseline Pipeline Regression Tests', () => { // Test 2: With env var flag set, pipeline still works (falls through to legacy) process.env.KDM_MULTI_AGENT = 'true'; - mockGetCompletion.mockResolvedValue(OOM_KILLED_LLM_RESPONSE); + vi.mocked(runAgentCouncilProcess).mockResolvedValue({ + confidence: 'high', + rootCause: 'council diagnosis', + bestSolution: { actionTitle: 'council fix', steps: [] }, + findings: [], + } as any); const multiAgentResult = await runAnalysis({ filters: ['Pod'], @@ -307,13 +319,18 @@ describe('Baseline Pipeline Regression Tests', () => { backend: 'openai', noCache: true, }); - // Should still produce valid results since multi-agent falls through to legacy + // Multi-agent mode uses council output instead of the legacy single-prompt response. expect(multiAgentResult.status).toBe('ProblemDetected'); - expect(multiAgentResult.results[0].details).toBe(OOM_KILLED_LLM_RESPONSE); + expect(multiAgentResult.results[0].details).toContain('council diagnosis'); // Test 3: With options.multiAgent flag set delete process.env.KDM_MULTI_AGENT; - mockGetCompletion.mockResolvedValue(OOM_KILLED_LLM_RESPONSE); + vi.mocked(runAgentCouncilProcess).mockResolvedValue({ + confidence: 'high', + rootCause: 'council diagnosis', + bestSolution: { actionTitle: 'council fix', steps: [] }, + findings: [], + } as any); const optionsFlagResult = await runAnalysis({ filters: ['Pod'], @@ -323,6 +340,6 @@ describe('Baseline Pipeline Regression Tests', () => { multiAgent: true, }); expect(optionsFlagResult.status).toBe('ProblemDetected'); - expect(optionsFlagResult.results[0].details).toBe(OOM_KILLED_LLM_RESPONSE); + expect(optionsFlagResult.results[0].details).toContain('council diagnosis'); }); }); diff --git a/src/analysis/analysis.ts b/src/analysis/analysis.ts index 76bc36f8..193aeb69 100644 --- a/src/analysis/analysis.ts +++ b/src/analysis/analysis.ts @@ -177,6 +177,7 @@ export interface ExplainSingleParams { noCache: boolean; customHeaders?: Record; baseUrl?: string; + timeoutMs?: number; onAgentProgress?: (event: AgentProgressEvent) => void; onEvent?: (event: NDJSONEvent) => void; } @@ -238,6 +239,7 @@ async function tryRunAgentCouncil( }, model: params.model || model, baseUrl: params.baseUrl || baseUrl, + timeoutMs: params.timeoutMs, onProgress: params.onAgentProgress, onEvent: params.onEvent, }); @@ -451,6 +453,7 @@ async function explainResults(results: AnalyzerResult[], options: AnalysisOption shouldAnonymize, noCache, customHeaders: options.customHeaders, + timeoutMs: options.councilTimeoutMs, }); } } @@ -573,7 +576,7 @@ async function executeCouncilForResults( results: AnalyzerResult[], options: AnalysisOptions, ): Promise { - const primaryResult = results[0]; + const primaryResult = results.find((result) => result.errors.length > 0) || results[0]; const failureText = results .flatMap((r) => r.errors.map((e) => e.text)) .join('\n') || (options.targetResource ? `Workload ${options.targetResource} failure` : options.labelSelector ? `Workload matching selector ${options.labelSelector} failure` : 'Cluster workload failure'); @@ -588,37 +591,94 @@ async function executeCouncilForResults( }; try { + const baseUrl = resolveOllamaBaseUrl(); + const dependencies = await checkCouncilDependencies(baseUrl); + if (!dependencies.canRunCouncil) { + throw new Error(dependencies.degradationReason || 'Council dependencies are unavailable'); + } + const runner = runAgentCouncilProcess || runPythonAgentCouncil; const consensus = await runner({ failureText, context: targetContext, model: options.model, - baseUrl: getAIConfig().providers.find((provider) => provider.name.toLowerCase() === 'ollama')?.baseUrl || resolveOllamaBaseUrl(), + baseUrl, + timeoutMs: options.councilTimeoutMs, }); - if (consensus) { - if (primaryResult) { - (primaryResult as any).consensus = consensus; - primaryResult.details = formatConsensusExplanation(consensus); - primaryResult.fallbackStep = 'council'; - } else { - results.push({ - kind: targetContext.kind, - name: targetContext.name, - namespace: targetContext.namespace || 'default', - errors: [{ text: failureText }], - details: formatConsensusExplanation(consensus), - consensus, - fallbackStep: 'council', - } as any); - } - return; + if (primaryResult) { + (primaryResult as any).consensus = consensus; + primaryResult.details = formatConsensusExplanation(consensus); + primaryResult.fallbackStep = 'council'; + } else { + results.push({ + kind: targetContext.kind, + name: targetContext.name, + namespace: targetContext.namespace || 'default', + errors: [{ text: failureText }], + details: formatConsensusExplanation(consensus), + consensus, + fallbackStep: 'council', + } as any); } } catch (err) { - logger.warn(`Multi-agent council bridge failed: ${(err as Error).message}. Falling back to legacy pipeline.`); + const degradationReason = (err as Error).message; + logger.warn(`Multi-agent council bridge failed: ${degradationReason}. Falling back to diagnostics.`); + await applyCouncilFallbacks(results, options, degradationReason); } +} - await explainResults(results, options); +/** Fills result details through the rule, cloud, or raw-summary degradation ladder. */ +async function applyCouncilFallbacks( + results: AnalyzerResult[], + options: AnalysisOptions, + degradationReason: string, +): Promise { + const backend = resolveBackend(options); + for (const result of results) { + if (result.errors.length > 0) { + await applyCouncilFallbackForResult(result, options, backend, degradationReason); + } + } +} + +function createCouncilFallbackContext( + result: AnalyzerResult, + options: AnalysisOptions, + backend: string, + degradationReason: string, +): { context: FallbackContext; mapping: { original: string; placeholder: string }[] } { + const failureText = result.errors.map((error) => error.text).join('\n'); + const { promptText, mapping } = preparePromptMapping(failureText, options.anonymize); + const prompt = buildPrompt({ + failureText: promptText, + language: options.language ?? DEFAULT_LANGUAGE, + analyzerName: result.kind, + }); + return { + context: { + result, + backend, + failureText: promptText, + prompt, + degradationReason, + customHeaders: options.customHeaders, + }, + mapping, + }; +} + +async function applyCouncilFallbackForResult( + result: AnalyzerResult, + options: AnalysisOptions, + backend: string, + degradationReason: string, +): Promise { + const { context, mapping } = createCouncilFallbackContext(result, options, backend, degradationReason); + if (!applyRuleEngineFallback(context) && !(await applyLegacyAiFallback(context))) { + applyRawSummaryFallback(context); + } + result.details = applyAnonymization(result.details || '', mapping, options.anonymize); } /** diff --git a/src/analysis/types.ts b/src/analysis/types.ts index e538f81f..99dff280 100644 --- a/src/analysis/types.ts +++ b/src/analysis/types.ts @@ -22,18 +22,14 @@ export interface AnalysisOptions { requestedFixes?: string[]; /** When true, route analysis through the multi-agent engine (Phases 1–24). */ multiAgent?: boolean; + /** Maximum time for the local council to run, in milliseconds. */ + councilTimeoutMs?: number; /** Optional file path to export sanitized incident audit trail. */ exportAudit?: string; /** Optional specific target workload string (e.g. deployment/checkout-api). */ targetResource?: string; /** Optional target resource name parsed from targetResource. */ resourceName?: string; - /** Optional file path to export sanitized incident audit trail. */ - exportAudit?: string; - /** Optional specific target workload string (e.g. 'deployment/checkout-api'). */ - targetResource?: string; - /** Optional target resource name parsed from targetResource. */ - resourceName?: string; } export interface AnalysisStats { diff --git a/src/commands/analyze.ts b/src/commands/analyze.ts index 82cfc730..235ef12e 100644 --- a/src/commands/analyze.ts +++ b/src/commands/analyze.ts @@ -50,6 +50,14 @@ const parseCustomHeaders = ( return result; }; +const parseCouncilTimeout = (value: string): number => { + const timeoutMs = Number(value); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error('Council timeout must be a positive whole number of milliseconds'); + } + return timeoutMs; +}; + /** * Builds the analysis configuration options from the CLI raw options. * @param options CLI parsed options. @@ -74,15 +82,12 @@ const buildAnalysisOptions = (options: any, signal: AbortSignal): AnalysisOption anonymize: Boolean(options.anonymize), customHeaders: options.customHeaders, noCache: Boolean(options.noCache), + councilTimeoutMs: options.councilTimeout, exportAudit: options.exportAudit, multiAgent: options.multiAgent ? Boolean(options.multiAgent) : undefined, targetResource: options.targetResource, resourceName: options.resourceName, interactive: options.interactive !== false && options.output !== 'json' && Boolean(process.stdout.isTTY && process.stdin.isTTY), - exportAudit: options.exportAudit, - multiAgent: options.multiAgent ? Boolean(options.multiAgent) : undefined, - targetResource: options.targetResource, - resourceName: options.resourceName, }); /** @@ -284,5 +289,6 @@ export const registerAnalyzeCommand = (program: Command) => { .option('-c, --no-cache', 'Skip cache for AI explanations') .option('--export-audit ', 'Export sanitized incident audit trail to a JSON file') .option('--multi-agent', 'Enable experimental multi-agent SRE council') + .option('--council-timeout ', 'Maximum council runtime in milliseconds', parseCouncilTimeout) .action(handleAnalyze); }; diff --git a/src/remediation/safety-prompt.ts b/src/remediation/safety-prompt.ts index dfe7829b..6f3be85f 100644 --- a/src/remediation/safety-prompt.ts +++ b/src/remediation/safety-prompt.ts @@ -34,6 +34,7 @@ export interface SafetyPromptProps { export interface PromptSafetyOptions { stdin?: NodeJS.ReadableStream; stdout?: NodeJS.WritableStream; + interactive?: boolean; } interface NormalizedPromptState { @@ -322,10 +323,10 @@ function createPromptProps( /** * Resolves standard or mocked stdin/stdout streams. */ -function resolveIOOptions(options?: PromptSafetyOptions): { stdin: any; stdout: any } { +function resolveIOOptions(options?: PromptSafetyOptions): { stdin: any; stdout: any; interactive?: boolean } { const stdin = options && options.stdin ? options.stdin : process.stdin; const stdout = options && options.stdout ? options.stdout : process.stdout; - return { stdin, stdout }; + return { stdin, stdout, interactive: options?.interactive }; } /** diff --git a/src/ui/AnalyzeDashboard.tsx b/src/ui/AnalyzeDashboard.tsx index fc9bd0b7..8106fe23 100644 --- a/src/ui/AnalyzeDashboard.tsx +++ b/src/ui/AnalyzeDashboard.tsx @@ -160,7 +160,7 @@ const DashboardHeader: React.FC<{ status?: string; isLoading: boolean; }> = ({ namespace, backend, problemsCount, status, isLoading }) => { - const isOk = status === 'OK' || problemsCount === 0; + const isOk = problemsCount === 0 && status !== 'ProblemDetected'; const statusColor = isOk ? 'green' : 'yellow'; const statusLabel = isOk ? 'OK' : 'PROBLEMS DETECTED';