diff --git a/CodeSmellAuditor.Cli/BatchRunSummary.cs b/CodeSmellAuditor.Cli/BatchRunSummary.cs index a99f8e7..3d5ccc5 100644 --- a/CodeSmellAuditor.Cli/BatchRunSummary.cs +++ b/CodeSmellAuditor.Cli/BatchRunSummary.cs @@ -15,8 +15,9 @@ public static string Format(AuditRunResult result) return "Batch summary: 0 files audited."; } - int passed = result.FileResults.Count(file => file.HasPassed); - int failed = result.FileResults.Count - passed; + int passed = result.FileResults.Count(file => file.Outcome == AuditOutcome.Passed); + int reviewRequired = result.FileResults.Count(file => file.Outcome == AuditOutcome.ReviewRequired); + int failed = result.FileResults.Count(file => file.Outcome == AuditOutcome.Failed); var builder = new StringBuilder(); builder.Append("Batch summary: "); @@ -24,6 +25,8 @@ public static string Format(AuditRunResult result) builder.Append(" files - "); builder.Append(passed); builder.Append(" passed, "); + builder.Append(reviewRequired); + builder.Append(" review required, "); builder.Append(failed); builder.Append(" failed"); @@ -31,11 +34,19 @@ public static string Format(AuditRunResult result) { builder.AppendLine(); builder.Append(" "); - builder.Append(file.HasPassed ? "PASS" : "FAIL"); + builder.Append(FormatOutcome(file.Outcome)); builder.Append(" "); builder.Append(file.FileName); } return builder.ToString(); } + + private static string FormatOutcome(AuditOutcome outcome) => outcome switch + { + AuditOutcome.Passed => "PASS", + AuditOutcome.ReviewRequired => "FAIL", + AuditOutcome.Failed => "ERROR", + _ => "UNKNOWN" + }; } diff --git a/CodeSmellAuditor.Cli/CliAuditHost.cs b/CodeSmellAuditor.Cli/CliAuditHost.cs index 9d069f4..a12c722 100644 --- a/CodeSmellAuditor.Cli/CliAuditHost.cs +++ b/CodeSmellAuditor.Cli/CliAuditHost.cs @@ -41,7 +41,7 @@ public async Task RunAsync(string rulesPath, string targetsPath, rules, reportsDirectoryPath); - AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n"); + PrintReportPath(result); fileResults.Add(result); } @@ -58,27 +58,10 @@ public async Task RunSniffAsync( IReadOnlyList filePaths, string? stackName = null) { - if (filePaths.Count == 0) - { - throw new ArgumentException("sniff requires at least one path to a .cs file."); - } - - var absolutePaths = new List(filePaths.Count); - foreach (string filePath in filePaths) - { - string absolutePath = Path.GetFullPath(filePath); - if (!File.Exists(absolutePath)) - { - throw new FileNotFoundException($"sniff target not found: {absolutePath}", absolutePath); - } - - if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) - { - throw new ArgumentException($"sniff path must be a .cs file: {absolutePath}"); - } - - absolutePaths.Add(absolutePath); - } + IReadOnlyList absolutePaths = AuditEngine.ResolveExistingCsPaths( + filePaths, + minimumCount: 1, + commandName: "sniff"); IReadOnlyList rules = await _engine.LoadRulesAsync(rulesPath, stackName); PrintRulesLoaded(rules); @@ -95,7 +78,7 @@ public async Task RunSniffAsync( rules, reportsDirectoryPath); - AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n"); + PrintReportPath(result); fileResults.Add(result); } @@ -113,27 +96,10 @@ public async Task RunSniffSystemAsync( string? manifestPath, string? stackName = null) { - if (filePaths.Count < 2) - { - throw new ArgumentException("sniff-system requires at least two paths to .cs files."); - } - - var absolutePaths = new List(filePaths.Count); - foreach (string filePath in filePaths) - { - string absolutePath = Path.GetFullPath(filePath); - if (!File.Exists(absolutePath)) - { - throw new FileNotFoundException($"sniff-system target not found: {absolutePath}", absolutePath); - } - - if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) - { - throw new ArgumentException($"sniff-system path must be a .cs file: {absolutePath}"); - } - - absolutePaths.Add(absolutePath); - } + IReadOnlyList absolutePaths = AuditEngine.ResolveExistingCsPaths( + filePaths, + minimumCount: 2, + commandName: "sniff-system"); string? manifestText = null; if (!string.IsNullOrWhiteSpace(manifestPath)) @@ -148,29 +114,7 @@ public async Task RunSniffSystemAsync( AnsiConsole.MarkupLine($"[grey]Manifest:[/] [cyan]{Markup.Escape(absoluteManifest)}[/]"); } - string architectureRulesPath = AuditEngine.ResolveArchitectureRulesPath(rulesPath); - if (!Directory.Exists(architectureRulesPath)) - { - throw new DirectoryNotFoundException( - $"Architecture rules folder not found: {architectureRulesPath}"); - } - - // Architecture packs are the primary rules for system audits; optional stack packs - // layer on from the root Rules/Stacks folder (not under Architecture/). - IReadOnlyList architectureRules = await _engine.LoadRulesAsync(architectureRulesPath); - if (architectureRules.Count == 0) - { - throw new InvalidOperationException( - $"No architecture rule packs found under {architectureRulesPath}"); - } - - IReadOnlyList rules = string.IsNullOrWhiteSpace(stackName) - ? architectureRules - : await _engine.LoadRulesAsync( - architectureRulesPath, - stackName, - stackRootRulesPath: rulesPath); - + IReadOnlyList rules = await _engine.LoadSystemRulesAsync(rulesPath, stackName); PrintRulesLoaded(rules); string reportsDirectoryPath = AuditEngine.ResolveReportsDirectory(rulesPath); @@ -211,7 +155,7 @@ await AnsiConsole.Status() FileAuditResult systemResult = result ?? throw new InvalidOperationException("System audit did not produce a result."); - AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{systemResult.ReportPath}[/]\n"); + PrintReportPath(systemResult); AuditRunResult runResult = new(new[] { systemResult }); PrintBatchSummary(runResult); @@ -270,6 +214,17 @@ private static void PrintRulesLoaded(IReadOnlyList rules) AnsiConsole.WriteLine(); } + private static void PrintReportPath(FileAuditResult result) + { + if (string.IsNullOrEmpty(result.ReportPath)) + { + AnsiConsole.MarkupLine("[grey]└──[/] [yellow]No report saved (tool failure).[/]\n"); + return; + } + + AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n"); + } + private static void PrintBatchSummary(AuditRunResult result) { AnsiConsole.WriteLine(); diff --git a/CodeSmellAuditor.Cli/Program.cs b/CodeSmellAuditor.Cli/Program.cs index 636f56a..92fccea 100644 --- a/CodeSmellAuditor.Cli/Program.cs +++ b/CodeSmellAuditor.Cli/Program.cs @@ -30,7 +30,7 @@ AnsiConsole.MarkupLine($"[grey]Looked in:[/] {baseStoragePath}"); AnsiConsole.MarkupLine("[grey]Set CODESMELL_STORAGE or --storage, or run from the repo with WorkstationStorage present.[/]"); WaitForEnterIfInteractive(cliArgs); - Environment.ExitCode = 1; + Environment.ExitCode = 2; return; } @@ -40,7 +40,7 @@ AnsiConsole.MarkupLine($"[grey]Looked in:[/] {baseStoragePath}"); AnsiConsole.MarkupLine("[grey]Set CODESMELL_STORAGE or --storage, or run from the repo with WorkstationStorage present.[/]"); WaitForEnterIfInteractive(cliArgs); - Environment.ExitCode = 1; + Environment.ExitCode = 2; return; } @@ -108,8 +108,14 @@ static void WaitForEnterIfInteractive(CliArgs parsed) NumCtx: int.TryParse(Environment.GetEnvironmentVariable("CODESMELL_NUM_CTX"), out int ctx) ? ctx : 8192, NumPredict: int.TryParse(Environment.GetEnvironmentVariable("CODESMELL_NUM_PREDICT"), out int predict) ? predict : 1200); +using var httpClient = new HttpClient +{ + BaseAddress = new Uri(auditConfig.OllamaBaseAddress), + Timeout = TimeSpan.FromMinutes(10) +}; + IRuleRepository repository = new MarkdownRuleRepository(); -IAiOrchestrator aiService = new OllamaAiOrchestrator(auditConfig); +IAiOrchestrator aiService = new OllamaAiOrchestrator(httpClient, auditConfig); var engine = new AuditEngine(repository, aiService); var host = new CliAuditHost(engine); @@ -118,20 +124,20 @@ static void WaitForEnterIfInteractive(CliArgs parsed) AnsiConsole.MarkupLine($"[grey]Stack pack:[/] [cyan]{Markup.Escape(cliArgs.Stack)}[/]"); } -if (cliArgs.Mode == CliMode.Batch) -{ - AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath, cliArgs.Stack); - Environment.ExitCode = batchResult.ExitCode; - AnsiConsole.MarkupLine( - batchResult.AllPassed - ? "[bold green]Batch processing complete.[/]" - : "[bold yellow]Batch processing complete with failures.[/]"); - WaitForEnterIfInteractive(cliArgs); - return; -} - try { + if (cliArgs.Mode == CliMode.Batch) + { + AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath, cliArgs.Stack); + Environment.ExitCode = batchResult.ExitCode; + AnsiConsole.MarkupLine( + batchResult.AllPassed + ? "[bold green]Batch processing complete.[/]" + : "[bold yellow]Batch processing complete with failures.[/]"); + WaitForEnterIfInteractive(cliArgs); + return; + } + if (cliArgs.Mode == CliMode.SniffSystem) { AuditRunResult systemResult = await host.RunSniffSystemAsync( @@ -149,5 +155,5 @@ static void WaitForEnterIfInteractive(CliArgs parsed) catch (Exception ex) when (ex is FileNotFoundException or ArgumentException or DirectoryNotFoundException or InvalidOperationException) { AnsiConsole.MarkupLine($"[bold red]ERROR:[/] {Markup.Escape(ex.Message)}"); - Environment.ExitCode = 1; + Environment.ExitCode = 2; } diff --git a/CodeSmellAuditor.Core.Tests/AuditEngineTests.cs b/CodeSmellAuditor.Core.Tests/AuditEngineTests.cs index 65f05e6..b72af09 100644 --- a/CodeSmellAuditor.Core.Tests/AuditEngineTests.cs +++ b/CodeSmellAuditor.Core.Tests/AuditEngineTests.cs @@ -298,6 +298,98 @@ await Assert.ThrowsAsync(() => } } + [Fact] + public async Task LoadSystemRulesAsync_LoadsArchitectureAndOptionalStack() + { + string workspace = CreateTempWorkspace(); + + try + { + string rulesPath = Path.Combine(workspace, "Rules"); + string architecturePath = Path.Combine(rulesPath, "Architecture"); + string stackPath = Path.Combine(rulesPath, "Stacks", "Unity"); + Directory.CreateDirectory(architecturePath); + Directory.CreateDirectory(stackPath); + + await File.WriteAllTextAsync(Path.Combine(architecturePath, "arch.mdc"), "Architecture rule."); + await File.WriteAllTextAsync(Path.Combine(stackPath, "unity.mdc"), "Unity rule."); + + var engine = new AuditEngine( + new MarkdownRuleRepository(), + new FakeAiOrchestrator("Status: COMPLIANT")); + + IReadOnlyList architectureOnly = await engine.LoadSystemRulesAsync(rulesPath); + IReadOnlyList withStack = await engine.LoadSystemRulesAsync(rulesPath, "Unity"); + + Assert.Single(architectureOnly); + Assert.Equal(2, withStack.Count); + Assert.Contains(withStack, r => r.Name == "arch.mdc"); + Assert.Contains(withStack, r => r.Name == "unity.mdc"); + } + finally + { + Directory.Delete(workspace, recursive: true); + } + } + + [Fact] + public async Task LoadSystemRulesAsync_ThrowsWhenArchitectureFolderMissing() + { + string workspace = CreateTempWorkspace(); + + try + { + string rulesPath = Path.Combine(workspace, "Rules"); + Directory.CreateDirectory(rulesPath); + + var engine = new AuditEngine( + new MarkdownRuleRepository(), + new FakeAiOrchestrator("Status: COMPLIANT")); + + await Assert.ThrowsAsync(() => + engine.LoadSystemRulesAsync(rulesPath)); + } + finally + { + Directory.Delete(workspace, recursive: true); + } + } + + [Fact] + public async Task AuditFileAsync_ToolFailure_SkipsReportPersist() + { + string workspace = CreateTempWorkspace(); + + try + { + string rulesPath = Path.Combine(workspace, "Rules"); + string targetsPath = Path.Combine(workspace, "Targets"); + Directory.CreateDirectory(rulesPath); + Directory.CreateDirectory(targetsPath); + + string filePath = Path.Combine(targetsPath, "Sample.cs"); + await File.WriteAllTextAsync(filePath, "public class Sample { }"); + + var engine = new AuditEngine( + new FakeRuleRepository(new AuditRule("rule.mdc", "Use interfaces.")), + new FailingAiOrchestrator()); + + string reportsPath = AuditEngine.ResolveReportsDirectory(rulesPath); + FileAuditResult result = await engine.AuditFileAsync( + filePath, + await engine.LoadRulesAsync(rulesPath), + reportsPath); + + Assert.Equal(AuditOutcome.Failed, result.Outcome); + Assert.Equal(string.Empty, result.ReportPath); + Assert.Empty(Directory.GetFiles(reportsPath)); + } + finally + { + Directory.Delete(workspace, recursive: true); + } + } + private static string CreateTempWorkspace() { return Path.Combine(Path.GetTempPath(), $"codesmell-engine-{Guid.NewGuid():N}"); @@ -337,8 +429,7 @@ public Task AnalyzeCodeAsync( Action onTokenReceived) { onTokenReceived(_critique); - bool hasPassed = AuditReportParser.ParsePassStatus(_critique); - return Task.FromResult(new AuditReport(file.FilePath, _critique, hasPassed)); + return Task.FromResult(AuditReport.FromParsedStatus(file.FilePath, _critique)); } public Task AnalyzeSystemAsync( @@ -350,8 +441,30 @@ public Task AnalyzeSystemAsync( LastSystemFileCount = files.Count; LastManifestText = manifestText; onTokenReceived(_critique); - bool hasPassed = AuditReportParser.ParsePassStatus(_critique); - return Task.FromResult(new AuditReport(SystemAuditIdentity.ReportBaseName, _critique, hasPassed)); + return Task.FromResult( + AuditReport.FromParsedStatus(SystemAuditIdentity.ReportBaseName, _critique)); + } + } + + private sealed class FailingAiOrchestrator : IAiOrchestrator + { + public Task AnalyzeCodeAsync( + SourceFile file, + IEnumerable rules, + Action onTokenReceived) + { + return Task.FromResult( + AuditReport.ToolFailure(file.FilePath, "# Local Engine Failure\n\nUnavailable")); + } + + public Task AnalyzeSystemAsync( + IReadOnlyList files, + string? manifestText, + IEnumerable rules, + Action onTokenReceived) + { + return Task.FromResult( + AuditReport.ToolFailure(SystemAuditIdentity.ReportBaseName, "# Local Engine Failure\n\nUnavailable")); } } } diff --git a/CodeSmellAuditor.Core.Tests/AuditRunResultTests.cs b/CodeSmellAuditor.Core.Tests/AuditRunResultTests.cs index b641667..74dfbdd 100644 --- a/CodeSmellAuditor.Core.Tests/AuditRunResultTests.cs +++ b/CodeSmellAuditor.Core.Tests/AuditRunResultTests.cs @@ -11,8 +11,8 @@ public void ExitCode_AllPassed_IsZero() { var result = new AuditRunResult(new[] { - new FileAuditResult("A.cs", "a.md", HasPassed: true), - new FileAuditResult("B.cs", "b.md", HasPassed: true) + new FileAuditResult("A.cs", "a.md", AuditOutcome.Passed), + new FileAuditResult("B.cs", "b.md", AuditOutcome.Passed) }); Assert.True(result.AllPassed); @@ -20,18 +20,43 @@ public void ExitCode_AllPassed_IsZero() } [Fact] - public void ExitCode_AnyFailure_IsOne() + public void ExitCode_ReviewRequired_IsOne() { var result = new AuditRunResult(new[] { - new FileAuditResult("A.cs", "a.md", HasPassed: true), - new FileAuditResult("B.cs", "b.md", HasPassed: false) + new FileAuditResult("A.cs", "a.md", AuditOutcome.Passed), + new FileAuditResult("B.cs", "b.md", AuditOutcome.ReviewRequired) }); Assert.False(result.AllPassed); Assert.Equal(1, result.ExitCode); } + [Fact] + public void ExitCode_ToolFailure_IsTwo() + { + var result = new AuditRunResult(new[] + { + new FileAuditResult("A.cs", "a.md", AuditOutcome.Passed), + new FileAuditResult("B.cs", "b.md", AuditOutcome.Failed) + }); + + Assert.False(result.AllPassed); + Assert.Equal(2, result.ExitCode); + } + + [Fact] + public void ExitCode_FailedDominatesReviewRequired() + { + var result = new AuditRunResult(new[] + { + new FileAuditResult("A.cs", "a.md", AuditOutcome.ReviewRequired), + new FileAuditResult("B.cs", "b.md", AuditOutcome.Failed) + }); + + Assert.Equal(2, result.ExitCode); + } + [Fact] public void ExitCode_EmptyRun_IsOne() { @@ -53,18 +78,20 @@ public void Format_EmptyRun_ReportsZeroFiles() } [Fact] - public void Format_MixedResults_ListsPassFailPerFile() + public void Format_MixedResults_ListsPassFailAndError() { var result = new AuditRunResult(new[] { - new FileAuditResult("Good.cs", "g.md", HasPassed: true), - new FileAuditResult("Bad.cs", "b.md", HasPassed: false) + new FileAuditResult("Good.cs", "g.md", AuditOutcome.Passed), + new FileAuditResult("Bad.cs", "b.md", AuditOutcome.ReviewRequired), + new FileAuditResult("Down.cs", string.Empty, AuditOutcome.Failed) }); string text = BatchRunSummary.Format(result); - Assert.Contains("Batch summary: 2 files - 1 passed, 1 failed", text); + Assert.Contains("Batch summary: 3 files - 1 passed, 1 review required, 1 failed", text); Assert.Contains("PASS Good.cs", text); Assert.Contains("FAIL Bad.cs", text); + Assert.Contains("ERROR Down.cs", text); } } diff --git a/CodeSmellAuditor.Core.Tests/MarkdownRuleRepositoryTests.cs b/CodeSmellAuditor.Core.Tests/MarkdownRuleRepositoryTests.cs index 21c983e..b9d94b3 100644 --- a/CodeSmellAuditor.Core.Tests/MarkdownRuleRepositoryTests.cs +++ b/CodeSmellAuditor.Core.Tests/MarkdownRuleRepositoryTests.cs @@ -39,6 +39,29 @@ await File.WriteAllTextAsync( } } + [Fact] + public async Task GetActiveRulesAsync_ReturnsRulesInOrdinalIgnoreCaseOrder() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codesmell-rules-order-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + + try + { + await File.WriteAllTextAsync(Path.Combine(tempDir, "20-second.mdc"), "Second rule."); + await File.WriteAllTextAsync(Path.Combine(tempDir, "00-first.mdc"), "First rule."); + await File.WriteAllTextAsync(Path.Combine(tempDir, "10-middle.mdc"), "Middle rule."); + + var repository = new MarkdownRuleRepository(); + var rules = (await repository.GetActiveRulesAsync(tempDir)).ToList(); + + Assert.Equal(new[] { "00-first.mdc", "10-middle.mdc", "20-second.mdc" }, rules.Select(r => r.Name)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + [Fact] public async Task GetActiveRulesAsync_ThrowsWhenFolderMissing() { diff --git a/CodeSmellAuditor.Core.Tests/OllamaAiOrchestratorTests.cs b/CodeSmellAuditor.Core.Tests/OllamaAiOrchestratorTests.cs new file mode 100644 index 0000000..d2e3d25 --- /dev/null +++ b/CodeSmellAuditor.Core.Tests/OllamaAiOrchestratorTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Text; +using CodeSmellAuditor.Core; +using Xunit; + +namespace CodeSmellAuditor.Core.Tests; + +public class OllamaAiOrchestratorTests +{ + private static readonly SourceFile SampleFile = new("Sample.cs", "public class Sample { }"); + private static readonly IReadOnlyList SampleRules = + new[] { new AuditRule("rule.mdc", "Prefer interfaces.") }; + + [Fact] + public async Task AnalyzeCodeAsync_StreamsContentAndParsesCompliantStatus() + { + string streamBody = + """{"message":{"content":"Status: "}}""" + "\n" + + """{"message":{"content":"COMPLIANT\nScore: 90"}}""" + "\n"; + + using var httpClient = CreateClient(new FakeHandler(HttpStatusCode.OK, streamBody)); + var orchestrator = new OllamaAiOrchestrator(httpClient, new AuditConfiguration()); + + var tokens = new StringBuilder(); + AuditReport report = await orchestrator.AnalyzeCodeAsync( + SampleFile, + SampleRules, + token => tokens.Append(token)); + + Assert.Equal(AuditOutcome.Passed, report.Outcome); + Assert.Contains("COMPLIANT", report.MarkdownCritique); + Assert.Contains("Status: COMPLIANT", tokens.ToString()); + } + + [Fact] + public async Task AnalyzeCodeAsync_EmptyBody_ReturnsFailed() + { + using var httpClient = CreateClient(new FakeHandler(HttpStatusCode.OK, "\n")); + var orchestrator = new OllamaAiOrchestrator(httpClient, new AuditConfiguration()); + + AuditReport report = await orchestrator.AnalyzeCodeAsync( + SampleFile, + SampleRules, + _ => { }); + + Assert.Equal(AuditOutcome.Failed, report.Outcome); + Assert.Contains("Empty Model Response", report.MarkdownCritique); + } + + [Fact] + public async Task AnalyzeCodeAsync_HttpFailure_ReturnsFailed() + { + using var httpClient = CreateClient(new FakeHandler(HttpStatusCode.BadGateway, "bad gateway")); + var orchestrator = new OllamaAiOrchestrator(httpClient, new AuditConfiguration()); + + AuditReport report = await orchestrator.AnalyzeCodeAsync( + SampleFile, + SampleRules, + _ => { }); + + Assert.Equal(AuditOutcome.Failed, report.Outcome); + Assert.Contains("Local Engine Failure", report.MarkdownCritique); + } + + [Fact] + public async Task AnalyzeCodeAsync_OllamaErrorLine_ReturnsFailed() + { + string streamBody = """{"error":"model not found"}""" + "\n"; + using var httpClient = CreateClient(new FakeHandler(HttpStatusCode.OK, streamBody)); + var orchestrator = new OllamaAiOrchestrator(httpClient, new AuditConfiguration()); + + AuditReport report = await orchestrator.AnalyzeCodeAsync( + SampleFile, + SampleRules, + _ => { }); + + Assert.Equal(AuditOutcome.Failed, report.Outcome); + Assert.Contains("model not found", report.MarkdownCritique); + } + + [Fact] + public async Task AnalyzeCodeAsync_BudgetExceeded_ReturnsFailed() + { + var config = new AuditConfiguration(MaxInputCharacters: 10); + using var httpClient = CreateClient(new FakeHandler(HttpStatusCode.OK, "{}")); + var orchestrator = new OllamaAiOrchestrator(httpClient, config); + + AuditReport report = await orchestrator.AnalyzeCodeAsync( + SampleFile, + SampleRules, + _ => { }); + + Assert.Equal(AuditOutcome.Failed, report.Outcome); + Assert.Contains("Budget Exceeded", report.MarkdownCritique); + } + + private static HttpClient CreateClient(HttpMessageHandler handler) + { + return new HttpClient(handler) + { + BaseAddress = new Uri(AuditConfiguration.DefaultOllamaBaseAddress) + }; + } + + private sealed class FakeHandler : HttpMessageHandler + { + private readonly HttpStatusCode _statusCode; + private readonly string _body; + + public FakeHandler(HttpStatusCode statusCode, string body) + { + _statusCode = statusCode; + _body = body; + } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(_statusCode) + { + Content = new StringContent(_body, Encoding.UTF8, "application/x-ndjson") + }; + return Task.FromResult(response); + } + } +} diff --git a/CodeSmellAuditor.Core/AuditEngine.cs b/CodeSmellAuditor.Core/AuditEngine.cs index 21d8c66..370dac3 100644 --- a/CodeSmellAuditor.Core/AuditEngine.cs +++ b/CodeSmellAuditor.Core/AuditEngine.cs @@ -61,6 +61,39 @@ public async Task> LoadRulesAsync( return combined; } + /// + /// Loads Architecture rule packs (and optional stack packs from the root Rules folder) + /// for sniff-system audits. + /// + public async Task> LoadSystemRulesAsync( + string rulesPath, + string? stackName = null) + { + string architectureRulesPath = ResolveArchitectureRulesPath(rulesPath); + if (!Directory.Exists(architectureRulesPath)) + { + throw new DirectoryNotFoundException( + $"Architecture rules folder not found: {architectureRulesPath}"); + } + + IReadOnlyList architectureRules = await LoadRulesAsync(architectureRulesPath); + if (architectureRules.Count == 0) + { + throw new InvalidOperationException( + $"No architecture rule packs found under {architectureRulesPath}"); + } + + if (string.IsNullOrWhiteSpace(stackName)) + { + return architectureRules; + } + + return await LoadRulesAsync( + architectureRulesPath, + stackName, + stackRootRulesPath: rulesPath); + } + public static string ResolveArchitectureRulesPath(string rulesPath) { return Path.Combine(rulesPath, "Architecture"); @@ -103,7 +136,49 @@ public static string ResolveReportsDirectory(string rulesPath) public static IReadOnlyList EnumerateTargetFiles(string targetsPath) { - return Directory.GetFiles(targetsPath, "*.cs"); + return Directory.GetFiles(targetsPath, "*.cs") + .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + /// + /// Resolves caller paths to existing absolute .cs files. Enforces a minimum count. + /// + public static IReadOnlyList ResolveExistingCsPaths( + IReadOnlyList filePaths, + int minimumCount, + string commandName) + { + if (filePaths.Count < minimumCount) + { + throw new ArgumentException( + minimumCount <= 1 + ? $"{commandName} requires at least one path to a .cs file." + : $"{commandName} requires at least two paths to .cs files.", + nameof(filePaths)); + } + + var absolutePaths = new List(filePaths.Count); + foreach (string filePath in filePaths) + { + string absolutePath = Path.GetFullPath(filePath); + if (!File.Exists(absolutePath)) + { + throw new FileNotFoundException( + $"{commandName} target not found: {absolutePath}", + absolutePath); + } + + if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException( + $"{commandName} path must be a .cs file: {absolutePath}"); + } + + absolutePaths.Add(absolutePath); + } + + return absolutePaths; } public async Task AuditFileAsync( @@ -112,21 +187,31 @@ public async Task AuditFileAsync( string reportsDirectoryPath, Action? onTokenReceived = null) { - string fileName = Path.GetFileName(filePath); - string sourceCodeText = await File.ReadAllTextAsync(filePath); - SourceFile currentTarget = new(filePath, sourceCodeText); + string absolutePath = Path.GetFullPath(filePath); + if (!File.Exists(absolutePath)) + { + throw new FileNotFoundException($"Audit target not found: {absolutePath}", absolutePath); + } + + if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException($"Audit path must be a .cs file: {absolutePath}"); + } + + string fileName = Path.GetFileName(absolutePath); + string sourceCodeText = await File.ReadAllTextAsync(absolutePath); + SourceFile currentTarget = new(absolutePath, sourceCodeText); AuditReport auditReport = await _aiService.AnalyzeCodeAsync( currentTarget, rules, token => onTokenReceived?.Invoke(token)); - string writtenFilePath = await SaveReportToFileSystemAsync( - reportsDirectoryPath, + return await ToFileAuditResultAsync( fileName, - auditReport.MarkdownCritique); - - return new FileAuditResult(fileName, writtenFilePath, auditReport.HasPassed); + fileName, + auditReport, + reportsDirectoryPath); } public async Task AuditSystemAsync( @@ -136,27 +221,16 @@ public async Task AuditSystemAsync( string reportsDirectoryPath, Action? onTokenReceived = null) { - if (filePaths.Count < 2) - { - throw new ArgumentException("System audit requires at least two .cs file paths.", nameof(filePaths)); - } + IReadOnlyList absolutePaths = ResolveExistingCsPaths( + filePaths, + minimumCount: 2, + commandName: "System audit"); - var sources = new List(filePaths.Count); - var fileNames = new List(filePaths.Count); + var sources = new List(absolutePaths.Count); + var fileNames = new List(absolutePaths.Count); - foreach (string filePath in filePaths) + foreach (string absolutePath in absolutePaths) { - string absolutePath = Path.GetFullPath(filePath); - if (!File.Exists(absolutePath)) - { - throw new FileNotFoundException($"System audit target not found: {absolutePath}", absolutePath); - } - - if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) - { - throw new ArgumentException($"System audit path must be a .cs file: {absolutePath}"); - } - string content = await File.ReadAllTextAsync(absolutePath); sources.Add(new SourceFile(absolutePath, content)); fileNames.Add(Path.GetFileName(absolutePath)); @@ -169,13 +243,12 @@ public async Task AuditSystemAsync( token => onTokenReceived?.Invoke(token)); string targetLabel = string.Join(", ", fileNames); - string writtenFilePath = await SaveReportToFileSystemAsync( - reportsDirectoryPath, + return await ToFileAuditResultAsync( + SystemAuditIdentity.ReportBaseName, SystemAuditIdentity.ReportBaseName + ".cs", - auditReport.MarkdownCritique, + auditReport, + reportsDirectoryPath, targetLabel); - - return new FileAuditResult(SystemAuditIdentity.ReportBaseName, writtenFilePath, auditReport.HasPassed); } public async Task RunBatchAsync(string rulesPath, string targetsPath) @@ -198,6 +271,27 @@ public async Task RunBatchAsync(string rulesPath, string targets return new AuditRunResult(fileResults); } + private static async Task ToFileAuditResultAsync( + string resultFileName, + string reportBaseFileName, + AuditReport auditReport, + string reportsDirectoryPath, + string? targetFileLabel = null) + { + if (auditReport.Outcome == AuditOutcome.Failed) + { + return new FileAuditResult(resultFileName, string.Empty, AuditOutcome.Failed); + } + + string writtenFilePath = await SaveReportToFileSystemAsync( + reportsDirectoryPath, + reportBaseFileName, + auditReport.MarkdownCritique, + targetFileLabel); + + return new FileAuditResult(resultFileName, writtenFilePath, auditReport.Outcome); + } + private static async Task SaveReportToFileSystemAsync( string targetFolder, string targetFileName, diff --git a/CodeSmellAuditor.Core/AuditPromptBuilder.cs b/CodeSmellAuditor.Core/AuditPromptBuilder.cs index 7b191b4..7d263f7 100644 --- a/CodeSmellAuditor.Core/AuditPromptBuilder.cs +++ b/CodeSmellAuditor.Core/AuditPromptBuilder.cs @@ -4,9 +4,11 @@ public record AuditConfiguration( string ModelName = AuditConfiguration.DefaultModelName, int NumCtx = 8192, int NumPredict = 1200, - int MaxInputCharacters = 24000) + int MaxInputCharacters = 24000, + string OllamaBaseAddress = AuditConfiguration.DefaultOllamaBaseAddress) { public const string DefaultModelName = "qwen3.5:4b"; + public const string DefaultOllamaBaseAddress = "http://localhost:11434"; /// /// Larger local-model budget for multi-file system audits. diff --git a/CodeSmellAuditor.Core/AuditRunResult.cs b/CodeSmellAuditor.Core/AuditRunResult.cs index d513a1b..6c50e1f 100644 --- a/CodeSmellAuditor.Core/AuditRunResult.cs +++ b/CodeSmellAuditor.Core/AuditRunResult.cs @@ -1,14 +1,38 @@ namespace CodeSmellAuditor.Core; -public record FileAuditResult(string FileName, string ReportPath, bool HasPassed); +public record FileAuditResult(string FileName, string ReportPath, AuditOutcome Outcome) +{ + public bool HasPassed => Outcome == AuditOutcome.Passed; +} public record AuditRunResult(IReadOnlyList FileResults) { public bool AllPassed => FileResults.Count > 0 && FileResults.All(result => result.HasPassed); /// - /// Process exit code for scriptable hosts: 0 when every audited file passed, otherwise 1. - /// Empty runs are treated as failure (nothing to pass). + /// Process exit code: 0 passed, 1 review required, 2 tool/wiring failure. + /// Empty runs exit 1 (nothing to pass). Any Failed dominates over ReviewRequired. /// - public int ExitCode => AllPassed ? 0 : 1; + public int ExitCode + { + get + { + if (FileResults.Count == 0) + { + return 1; + } + + if (FileResults.Any(result => result.Outcome == AuditOutcome.Failed)) + { + return 2; + } + + if (FileResults.Any(result => result.Outcome == AuditOutcome.ReviewRequired)) + { + return 1; + } + + return 0; + } + } } diff --git a/CodeSmellAuditor.Core/MarkdownRuleRepository.cs b/CodeSmellAuditor.Core/MarkdownRuleRepository.cs index 51bd654..0becf5a 100644 --- a/CodeSmellAuditor.Core/MarkdownRuleRepository.cs +++ b/CodeSmellAuditor.Core/MarkdownRuleRepository.cs @@ -12,7 +12,8 @@ public async Task> GetActiveRulesAsync(string rulesFolder var loadedRules = new List(); var ruleFiles = Directory.EnumerateFiles(rulesFolderPath, "*.*") - .Where(file => file.EndsWith(".mdc", StringComparison.OrdinalIgnoreCase)); + .Where(file => file.EndsWith(".mdc", StringComparison.OrdinalIgnoreCase)) + .OrderBy(file => file, StringComparer.OrdinalIgnoreCase); foreach (var filePath in ruleFiles) { diff --git a/CodeSmellAuditor.Core/Models.cs b/CodeSmellAuditor.Core/Models.cs index aaa796a..9167620 100644 --- a/CodeSmellAuditor.Core/Models.cs +++ b/CodeSmellAuditor.Core/Models.cs @@ -2,12 +2,36 @@ namespace CodeSmellAuditor.Core; -// Core Domain Models +/// +/// Result of an audit attempt: model verdict or tool/wiring failure. +/// +public enum AuditOutcome +{ + Passed, + ReviewRequired, + Failed +} + public record SourceFile(string FilePath, string Content); public record AuditRule(string Name, string PromptGuideline); -public record AuditReport(string FilePath, string MarkdownCritique, bool HasPassed); +public record AuditReport(string FilePath, string MarkdownCritique, AuditOutcome Outcome) +{ + public bool HasPassed => Outcome == AuditOutcome.Passed; + + public static AuditReport FromParsedStatus(string filePath, string markdownCritique) + { + bool passed = AuditReportParser.ParsePassStatus(markdownCritique); + return new AuditReport( + filePath, + markdownCritique, + passed ? AuditOutcome.Passed : AuditOutcome.ReviewRequired); + } + + public static AuditReport ToolFailure(string filePath, string markdownCritique) => + new(filePath, markdownCritique, AuditOutcome.Failed); +} /// /// Synthetic report identity for a multi-file system audit (not a single source path). @@ -17,7 +41,6 @@ public static class SystemAuditIdentity public const string ReportBaseName = "SystemAudit"; } -// Ollama API DTOs (Data Transfer Objects) public class OllamaChatRequest { public string Model { get; set; } = string.Empty; @@ -31,7 +54,7 @@ public class OllamaMessage { public string Role { get; set; } = string.Empty; public string Content { get; set; } = string.Empty; - + [JsonPropertyName("thinking")] public string? Thinking { get; set; } -} \ No newline at end of file +} diff --git a/CodeSmellAuditor.Core/OllamaAiOrchestrator.cs b/CodeSmellAuditor.Core/OllamaAiOrchestrator.cs index b9abebf..6463279 100644 --- a/CodeSmellAuditor.Core/OllamaAiOrchestrator.cs +++ b/CodeSmellAuditor.Core/OllamaAiOrchestrator.cs @@ -9,21 +9,12 @@ public class OllamaAiOrchestrator : IAiOrchestrator private readonly HttpClient _httpClient; private readonly AuditConfiguration _config; - public OllamaAiOrchestrator(AuditConfiguration config) + public OllamaAiOrchestrator(HttpClient httpClient, AuditConfiguration config) { - _httpClient = new HttpClient - { - BaseAddress = new Uri("http://localhost:11434"), - Timeout = TimeSpan.FromMinutes(10) - }; + _httpClient = httpClient; _config = config; } - public OllamaAiOrchestrator(string modelName) - : this(new AuditConfiguration(ModelName: modelName)) - { - } - public async Task AnalyzeCodeAsync( SourceFile file, IEnumerable rules, @@ -38,7 +29,9 @@ public async Task AnalyzeCodeAsync( } catch (InvalidOperationException ex) { - return new AuditReport(file.FilePath, $"# Audit Budget Exceeded\n\n{ex.Message}", false); + return AuditReport.ToolFailure( + file.FilePath, + $"# Audit Budget Exceeded\n\n{ex.Message}"); } return await CompleteChatAsync(file.FilePath, systemPrompt, userPrompt, onTokenReceived); @@ -52,10 +45,9 @@ public async Task AnalyzeSystemAsync( { if (files.Count < 2) { - return new AuditReport( + return AuditReport.ToolFailure( SystemAuditIdentity.ReportBaseName, - "# System Audit Error\n\nSystem audit requires at least two source files.", - false); + "# System Audit Error\n\nSystem audit requires at least two source files."); } string systemPrompt = SystemAuditPromptBuilder.BuildSystemPrompt(rules); @@ -67,10 +59,9 @@ public async Task AnalyzeSystemAsync( } catch (InvalidOperationException ex) { - return new AuditReport( + return AuditReport.ToolFailure( SystemAuditIdentity.ReportBaseName, - $"# Audit Budget Exceeded\n\n{ex.Message}", - false); + $"# Audit Budget Exceeded\n\n{ex.Message}"); } return await CompleteChatAsync( @@ -120,7 +111,9 @@ private async Task CompleteChatAsync( Content = contentStream }; - using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); using var networkStream = await response.Content.ReadAsStreamAsync(); @@ -136,6 +129,18 @@ private async Task CompleteChatAsync( } using var doc = JsonDocument.Parse(jsonLine); + + if (doc.RootElement.TryGetProperty("error", out var errorElement)) + { + string errorText = errorElement.ValueKind == JsonValueKind.String + ? errorElement.GetString() ?? "Unknown Ollama error" + : errorElement.ToString(); + + return AuditReport.ToolFailure( + reportPath, + $"# Local Engine Failure\n\nOllama returned an error.\n\n{errorText}"); + } + if (doc.RootElement.TryGetProperty("message", out var messageElement) && messageElement.TryGetProperty("content", out var contentElement)) { @@ -152,27 +157,25 @@ private async Task CompleteChatAsync( string fullTextResult = completeReportBuilder.ToString(); if (string.IsNullOrWhiteSpace(fullTextResult)) { - return new AuditReport( + return AuditReport.ToolFailure( reportPath, "# Empty Model Response\n\n" + - "Ollama returned no report content. If using a reasoning model, ensure thinking mode is disabled.", - false); + "Ollama returned no report content. If using a reasoning model, ensure thinking mode is disabled."); } - bool hasPassed = AuditReportParser.ParsePassStatus(fullTextResult); - - return new AuditReport(reportPath, fullTextResult, hasPassed); + return AuditReport.FromParsedStatus(reportPath, fullTextResult); } catch (HttpRequestException ex) { - return new AuditReport( + return AuditReport.ToolFailure( reportPath, - $"# Local Engine Failure\n\nUnable to reach Ollama API.\n\n{ex.Message}", - false); + $"# Local Engine Failure\n\nUnable to reach Ollama API.\n\n{ex.Message}"); } catch (Exception ex) { - return new AuditReport(reportPath, $"# Internal Audit System Error\n\n{ex.Message}", false); + return AuditReport.ToolFailure( + reportPath, + $"# Internal Audit System Error\n\n{ex.Message}"); } } } diff --git a/README.md b/README.md index 00d4d36..4010b07 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,14 @@ This is an **AI-assisted semantic reviewer**, not a Roslyn/static analyzer: the - `sniff-system` for two or more related files (optional `--manifest`) using `Rules/Architecture/` and one combined report - Local Ollama only — source stays on your machine - Streaming console output with Spectre Status until first token; timestamped reports under `WorkstationStorage/Reports/` -- Pass/fail from the report `Status:` line (exit codes `0`/`1`); `Score` is informational only (not a regression metric) +- Pass/fail from the report `Status:` line (exit `0`/`1`); tool/wiring failures exit `2` and skip report persistence; `Score` is informational only - Layered Core / Cli with interfaces for rule loading and AI orchestration - xUnit tests (fakes; no Ollama in CI) and Ubuntu GitHub Actions ## Architecture ```text -Program.cs (composition root) +Program.cs (composition root: HttpClient + config) -> MarkdownRuleRepository / OllamaAiOrchestrator -> AuditEngine -> CliAuditHost (batch + sniff + sniff-system) diff --git a/docs/architecture.md b/docs/architecture.md index 7413008..96ebf35 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,7 +58,7 @@ Loads active governance rules from a folder. `MarkdownRuleRepository` reads `.md ### IAiOrchestrator -Sends source code and rules to an AI backend and returns an `AuditReport`. Supports single-file `AnalyzeCodeAsync` and multi-file `AnalyzeSystemAsync`. `OllamaAiOrchestrator` streams tokens from Ollama `/api/chat` with a shared request helper and bounded output templates. +Sends source code and rules to an AI backend and returns an `AuditReport` with an `AuditOutcome` (`Passed`, `ReviewRequired`, or `Failed`). Supports single-file `AnalyzeCodeAsync` and multi-file `AnalyzeSystemAsync`. `OllamaAiOrchestrator` takes an injected `HttpClient` (bound at the composition root) and streams tokens from Ollama `/api/chat` with bounded output templates. Transport, empty-response, and budget failures map to `Failed` and are not persisted as critiques. ### AuditPromptBuilder @@ -69,10 +69,11 @@ Assembles system instructions, rule excerpts, and source text. Enforces a charac Presentation-neutral domain API: 1. `LoadRulesAsync` - load compact rules (optional stack name concatenates `Rules/Stacks//`) -2. `EnumerateTargetFiles` / `ResolveReportsDirectory` / `ResolveArchitectureRulesPath` / `ResolveStackRulesPath` - storage helpers -3. `AuditFileAsync` - analyze one file, optional token callback, persist markdown report -4. `AuditSystemAsync` - analyze two or more files as one architecture audit (optional manifest text) -5. `RunBatchAsync` - headless multi-file run for tests and non-interactive callers +2. `LoadSystemRulesAsync` - Architecture packs plus optional stack packs for sniff-system +3. `EnumerateTargetFiles` / `ResolveExistingCsPaths` / `ResolveReportsDirectory` / `ResolveArchitectureRulesPath` / `ResolveStackRulesPath` - storage helpers +4. `AuditFileAsync` - analyze one file, optional token callback, persist markdown report when outcome is not `Failed` +5. `AuditSystemAsync` - analyze two or more files as one architecture audit (optional manifest text) +6. `RunBatchAsync` - headless multi-file run for tests and non-interactive callers ### SystemAuditPromptBuilder @@ -110,9 +111,9 @@ Core does not reference Cli. External AI and file I/O are behind interfaces or i | Mode | Invocation | Source | Exit behavior | |------|------------|--------|---------------| -| Batch | no args | `WorkstationStorage/Targets/*.cs` | Pass/fail summary; exit `0`/`1`; Enter-to-exit unless `--non-interactive` | -| Sniff | `sniff path\to\File.cs [more.cs...]` | one or more external paths in place | no Enter wait; pass/fail summary; exit `0`/`1` from aggregate pass/fail | -| Sniff-system | `sniff-system path\to\A.cs path\to\B.cs [...]` | two or more external paths; optional `--manifest` | one combined architecture report; no Enter wait; exit `0`/`1` | +| Batch | no args | `WorkstationStorage/Targets/*.cs` | Summary; exit `0`/`1`/`2`; Enter-to-exit unless `--non-interactive` | +| Sniff | `sniff path\to\File.cs [more.cs...]` | one or more external paths in place | no Enter wait; exit `0`/`1`/`2` from aggregate outcome | +| Sniff-system | `sniff-system path\to\A.cs path\to\B.cs [...]` | two or more external paths; optional `--manifest` | one combined architecture report; no Enter wait; exit `0`/`1`/`2` | ### Shared options @@ -126,11 +127,20 @@ Core does not reference Cli. External AI and file I/O are behind interfaces or i Flags may appear before or after `sniff` / `sniff-system`. Example: `dotnet run --project CodeSmellAuditor.Cli -- --model qwen3.5:4b --non-interactive`. -Batch and sniff reuse `CliAuditHost` Status wrapping and `AuditEngine.AuditFileAsync` with root `Rules/*.mdc`. Optional `--stack` concatenates `Rules/Stacks//*.mdc` onto that set. Sniff-system loads only `Rules/Architecture/*.mdc` (plus optional stack packs from `Rules/Stacks/`), uses `AuditEngine.AuditSystemAsync` / `IAiOrchestrator.AnalyzeSystemAsync` with a larger `AuditConfiguration.ForSystemAudit` budget, and writes one `SystemAudit_*_Critique.md` report. Rules and reports always come from auditor storage; sniff modes never copy into Targets. Empty batch runs (no `*.cs` targets) exit `1`. Multi-path sniff loads rules once and audits files sequentially. `scripts/sniff.ps1` forwards `-Target` paths plus optional `-Model` / `-Storage` / `-Stack`; `-System` and `-Manifest` map to `sniff-system`. +Batch and sniff reuse `CliAuditHost` Status wrapping and `AuditEngine.AuditFileAsync` with root `Rules/*.mdc`. Optional `--stack` concatenates `Rules/Stacks//*.mdc` onto that set. Sniff-system uses `AuditEngine.LoadSystemRulesAsync` (Architecture packs plus optional stack packs), `AuditSystemAsync` / `AnalyzeSystemAsync` with a larger `AuditConfiguration.ForSystemAudit` budget, and writes one `SystemAudit_*_Critique.md` report when the outcome is not `Failed`. Rules and reports always come from auditor storage; sniff modes never copy into Targets. Empty batch runs (no `*.cs` targets) exit `1`. Wiring and missing-path errors at the CLI exit `2`. Multi-path sniff loads rules once and audits files sequentially. `scripts/sniff.ps1` forwards `-Target` paths plus optional `-Model` / `-Storage` / `-Stack`; `-System` and `-Manifest` map to `sniff-system`. ### Report contract | Field | Role | |-------|------| -| `Status:` | Sole pass/fail signal (`COMPLIANT` or `REVIEW REQUIRED`). Parsed by the CLI for exit codes. | +| `Status:` | Sole model pass/fail signal (`COMPLIANT` or `REVIEW REQUIRED`). Maps to exit `0` or `1`. | +| Tool failure | Ollama unreachable, empty response, budget exceeded, or CLI wiring errors. Maps to `AuditOutcome.Failed` / exit `2`; no critique file written. | | `Score:` | Rough qualitative signal for that run only. Model-emitted, non-deterministic; not a regression metric. Prefer Status and Top Findings over Score deltas between runs. | + +### Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Every audited target passed (`COMPLIANT`) | +| `1` | At least one `REVIEW REQUIRED`, or empty batch (nothing to pass); no tool failures | +| `2` | At least one tool/wiring failure (`Failed`), or CLI could not start the audit (missing Rules/Targets, bad paths, missing stack folder) | diff --git a/docs/sample-report-excerpt.md b/docs/sample-report-excerpt.md index cb78b00..73863b6 100644 --- a/docs/sample-report-excerpt.md +++ b/docs/sample-report-excerpt.md @@ -42,7 +42,7 @@ Score: 72 Extract spawn planning into a dedicated planner class and inject it as a single dependency. ``` -`Status:` is the only pass/fail signal used by the CLI. `Score:` is a rough qualitative signal for that run only - not a regression metric; prefer Status and Top Findings over Score deltas between runs. +`Status:` is the model pass/fail signal used for exit `0`/`1`. Tool failures (Ollama down, empty response, budget exceeded) use exit `2` and do not write a critique file. `Score:` is a rough qualitative signal for that run only - not a regression metric; prefer Status and Top Findings over Score deltas between runs. ## Interrupted reports