Skip to content

Harden audit outcomes, rule loading, and Ollama testability #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TimoKuronen merged 1 commit into from
Sep 30, 2026
Merged

Harden audit outcomes, rule loading, and Ollama testability #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions CodeSmellAuditor.Cli/BatchRunSummary.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,38 @@ public static string Format(AuditRunResult result)
return "Batch summary: 0 files audited.";
}

int passed = result.FileResults.Count(file => file.HasPassed);
int failed = result.FileResults.Count - passed;
int passed = result.FileResults.Count(file => file.Outcome == AuditOutcome.Passed);
int reviewRequired = result.FileResults.Count(file => file.Outcome == AuditOutcome.ReviewRequired);
int failed = result.FileResults.Count(file => file.Outcome == AuditOutcome.Failed);

var builder = new StringBuilder();
builder.Append("Batch summary: ");
builder.Append(result.FileResults.Count);
builder.Append(" files - ");
builder.Append(passed);
builder.Append(" passed, ");
builder.Append(reviewRequired);
builder.Append(" review required, ");
builder.Append(failed);
builder.Append(" failed");

foreach (FileAuditResult file in result.FileResults)
{
builder.AppendLine();
builder.Append(" ");
builder.Append(file.HasPassed ? "PASS" : "FAIL");
builder.Append(FormatOutcome(file.Outcome));
builder.Append(" ");
builder.Append(file.FileName);
}

return builder.ToString();
}

private static string FormatOutcome(AuditOutcome outcome) => outcome switch
{
AuditOutcome.Passed => "PASS",
AuditOutcome.ReviewRequired => "FAIL",
AuditOutcome.Failed => "ERROR",
_ => "UNKNOWN"
};
}
91 changes: 23 additions & 68 deletions CodeSmellAuditor.Cli/CliAuditHost.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public async Task<AuditRunResult> RunAsync(string rulesPath, string targetsPath,
rules,
reportsDirectoryPath);

AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n");
PrintReportPath(result);
fileResults.Add(result);
}

Expand All @@ -58,27 +58,10 @@ public async Task<AuditRunResult> RunSniffAsync(
IReadOnlyList<string> filePaths,
string? stackName = null)
{
if (filePaths.Count == 0)
{
throw new ArgumentException("sniff requires at least one path to a .cs file.");
}

var absolutePaths = new List<string>(filePaths.Count);
foreach (string filePath in filePaths)
{
string absolutePath = Path.GetFullPath(filePath);
if (!File.Exists(absolutePath))
{
throw new FileNotFoundException($"sniff target not found: {absolutePath}", absolutePath);
}

if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException($"sniff path must be a .cs file: {absolutePath}");
}

absolutePaths.Add(absolutePath);
}
IReadOnlyList<string> absolutePaths = AuditEngine.ResolveExistingCsPaths(
filePaths,
minimumCount: 1,
commandName: "sniff");

IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(rulesPath, stackName);
PrintRulesLoaded(rules);
Expand All @@ -95,7 +78,7 @@ public async Task<AuditRunResult> RunSniffAsync(
rules,
reportsDirectoryPath);

AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n");
PrintReportPath(result);
fileResults.Add(result);
}

Expand All @@ -113,27 +96,10 @@ public async Task<AuditRunResult> RunSniffSystemAsync(
string? manifestPath,
string? stackName = null)
{
if (filePaths.Count < 2)
{
throw new ArgumentException("sniff-system requires at least two paths to .cs files.");
}

var absolutePaths = new List<string>(filePaths.Count);
foreach (string filePath in filePaths)
{
string absolutePath = Path.GetFullPath(filePath);
if (!File.Exists(absolutePath))
{
throw new FileNotFoundException($"sniff-system target not found: {absolutePath}", absolutePath);
}

if (!absolutePath.EndsWith(".cs", StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException($"sniff-system path must be a .cs file: {absolutePath}");
}

absolutePaths.Add(absolutePath);
}
IReadOnlyList<string> absolutePaths = AuditEngine.ResolveExistingCsPaths(
filePaths,
minimumCount: 2,
commandName: "sniff-system");

string? manifestText = null;
if (!string.IsNullOrWhiteSpace(manifestPath))
Expand All @@ -148,29 +114,7 @@ public async Task<AuditRunResult> RunSniffSystemAsync(
AnsiConsole.MarkupLine($"[grey]Manifest:[/] [cyan]{Markup.Escape(absoluteManifest)}[/]");
}

string architectureRulesPath = AuditEngine.ResolveArchitectureRulesPath(rulesPath);
if (!Directory.Exists(architectureRulesPath))
{
throw new DirectoryNotFoundException(
$"Architecture rules folder not found: {architectureRulesPath}");
}

// Architecture packs are the primary rules for system audits; optional stack packs
// layer on from the root Rules/Stacks folder (not under Architecture/).
IReadOnlyList<AuditRule> architectureRules = await _engine.LoadRulesAsync(architectureRulesPath);
if (architectureRules.Count == 0)
{
throw new InvalidOperationException(
$"No architecture rule packs found under {architectureRulesPath}");
}

IReadOnlyList<AuditRule> rules = string.IsNullOrWhiteSpace(stackName)
? architectureRules
: await _engine.LoadRulesAsync(
architectureRulesPath,
stackName,
stackRootRulesPath: rulesPath);

IReadOnlyList<AuditRule> rules = await _engine.LoadSystemRulesAsync(rulesPath, stackName);
PrintRulesLoaded(rules);

string reportsDirectoryPath = AuditEngine.ResolveReportsDirectory(rulesPath);
Expand Down Expand Up @@ -211,7 +155,7 @@ await AnsiConsole.Status()
FileAuditResult systemResult = result
?? throw new InvalidOperationException("System audit did not produce a result.");

AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{systemResult.ReportPath}[/]\n");
PrintReportPath(systemResult);

AuditRunResult runResult = new(new[] { systemResult });
PrintBatchSummary(runResult);
Expand Down Expand Up @@ -270,6 +214,17 @@ private static void PrintRulesLoaded(IReadOnlyList<AuditRule> rules)
AnsiConsole.WriteLine();
}

private static void PrintReportPath(FileAuditResult result)
{
if (string.IsNullOrEmpty(result.ReportPath))
{
AnsiConsole.MarkupLine("[grey]└──[/] [yellow]No report saved (tool failure).[/]\n");
return;
}

AnsiConsole.MarkupLine($"[grey]└── Report saved:[/] [underline cyan]{result.ReportPath}[/]\n");
}

private static void PrintBatchSummary(AuditRunResult result)
{
AnsiConsole.WriteLine();
Expand Down
38 changes: 22 additions & 16 deletions CodeSmellAuditor.Cli/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
AnsiConsole.MarkupLine($"[grey]Looked in:[/] {baseStoragePath}");
AnsiConsole.MarkupLine("[grey]Set CODESMELL_STORAGE or --storage, or run from the repo with WorkstationStorage present.[/]");
WaitForEnterIfInteractive(cliArgs);
Environment.ExitCode = 1;
Environment.ExitCode = 2;
return;
}

Expand All @@ -40,7 +40,7 @@
AnsiConsole.MarkupLine($"[grey]Looked in:[/] {baseStoragePath}");
AnsiConsole.MarkupLine("[grey]Set CODESMELL_STORAGE or --storage, or run from the repo with WorkstationStorage present.[/]");
WaitForEnterIfInteractive(cliArgs);
Environment.ExitCode = 1;
Environment.ExitCode = 2;
return;
}

Expand Down Expand Up @@ -108,8 +108,14 @@ static void WaitForEnterIfInteractive(CliArgs parsed)
NumCtx: int.TryParse(Environment.GetEnvironmentVariable("CODESMELL_NUM_CTX"), out int ctx) ? ctx : 8192,
NumPredict: int.TryParse(Environment.GetEnvironmentVariable("CODESMELL_NUM_PREDICT"), out int predict) ? predict : 1200);

using var httpClient = new HttpClient
{
BaseAddress = new Uri(auditConfig.OllamaBaseAddress),
Timeout = TimeSpan.FromMinutes(10)
};

IRuleRepository repository = new MarkdownRuleRepository();
IAiOrchestrator aiService = new OllamaAiOrchestrator(auditConfig);
IAiOrchestrator aiService = new OllamaAiOrchestrator(httpClient, auditConfig);
var engine = new AuditEngine(repository, aiService);
var host = new CliAuditHost(engine);

Expand All @@ -118,20 +124,20 @@ static void WaitForEnterIfInteractive(CliArgs parsed)
AnsiConsole.MarkupLine($"[grey]Stack pack:[/] [cyan]{Markup.Escape(cliArgs.Stack)}[/]");
}

if (cliArgs.Mode == CliMode.Batch)
{
AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath, cliArgs.Stack);
Environment.ExitCode = batchResult.ExitCode;
AnsiConsole.MarkupLine(
batchResult.AllPassed
? "[bold green]Batch processing complete.[/]"
: "[bold yellow]Batch processing complete with failures.[/]");
WaitForEnterIfInteractive(cliArgs);
return;
}

try
{
if (cliArgs.Mode == CliMode.Batch)
{
AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath, cliArgs.Stack);
Environment.ExitCode = batchResult.ExitCode;
AnsiConsole.MarkupLine(
batchResult.AllPassed
? "[bold green]Batch processing complete.[/]"
: "[bold yellow]Batch processing complete with failures.[/]");
WaitForEnterIfInteractive(cliArgs);
return;
}

if (cliArgs.Mode == CliMode.SniffSystem)
{
AuditRunResult systemResult = await host.RunSniffSystemAsync(
Expand All @@ -149,5 +155,5 @@ static void WaitForEnterIfInteractive(CliArgs parsed)
catch (Exception ex) when (ex is FileNotFoundException or ArgumentException or DirectoryNotFoundException or InvalidOperationException)
{
AnsiConsole.MarkupLine($"[bold red]ERROR:[/] {Markup.Escape(ex.Message)}");
Environment.ExitCode = 1;
Environment.ExitCode = 2;
}
121 changes: 117 additions & 4 deletions CodeSmellAuditor.Core.Tests/AuditEngineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,98 @@ await Assert.ThrowsAsync<DirectoryNotFoundException>(() =>
}
}

[Fact]
public async Task LoadSystemRulesAsync_LoadsArchitectureAndOptionalStack()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
string architecturePath = Path.Combine(rulesPath, "Architecture");
string stackPath = Path.Combine(rulesPath, "Stacks", "Unity");
Directory.CreateDirectory(architecturePath);
Directory.CreateDirectory(stackPath);

await File.WriteAllTextAsync(Path.Combine(architecturePath, "arch.mdc"), "Architecture rule.");
await File.WriteAllTextAsync(Path.Combine(stackPath, "unity.mdc"), "Unity rule.");

var engine = new AuditEngine(
new MarkdownRuleRepository(),
new FakeAiOrchestrator("Status: COMPLIANT"));

IReadOnlyList<AuditRule> architectureOnly = await engine.LoadSystemRulesAsync(rulesPath);
IReadOnlyList<AuditRule> withStack = await engine.LoadSystemRulesAsync(rulesPath, "Unity");

Assert.Single(architectureOnly);
Assert.Equal(2, withStack.Count);
Assert.Contains(withStack, r => r.Name == "arch.mdc");
Assert.Contains(withStack, r => r.Name == "unity.mdc");
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

[Fact]
public async Task LoadSystemRulesAsync_ThrowsWhenArchitectureFolderMissing()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
Directory.CreateDirectory(rulesPath);

var engine = new AuditEngine(
new MarkdownRuleRepository(),
new FakeAiOrchestrator("Status: COMPLIANT"));

await Assert.ThrowsAsync<DirectoryNotFoundException>(() =>
engine.LoadSystemRulesAsync(rulesPath));
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

[Fact]
public async Task AuditFileAsync_ToolFailure_SkipsReportPersist()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
string targetsPath = Path.Combine(workspace, "Targets");
Directory.CreateDirectory(rulesPath);
Directory.CreateDirectory(targetsPath);

string filePath = Path.Combine(targetsPath, "Sample.cs");
await File.WriteAllTextAsync(filePath, "public class Sample { }");

var engine = new AuditEngine(
new FakeRuleRepository(new AuditRule("rule.mdc", "Use interfaces.")),
new FailingAiOrchestrator());

string reportsPath = AuditEngine.ResolveReportsDirectory(rulesPath);
FileAuditResult result = await engine.AuditFileAsync(
filePath,
await engine.LoadRulesAsync(rulesPath),
reportsPath);

Assert.Equal(AuditOutcome.Failed, result.Outcome);
Assert.Equal(string.Empty, result.ReportPath);
Assert.Empty(Directory.GetFiles(reportsPath));
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

private static string CreateTempWorkspace()
{
return Path.Combine(Path.GetTempPath(), $"codesmell-engine-{Guid.NewGuid():N}");
Expand Down Expand Up @@ -337,8 +429,7 @@ public Task<AuditReport> AnalyzeCodeAsync(
Action<string> onTokenReceived)
{
onTokenReceived(_critique);
bool hasPassed = AuditReportParser.ParsePassStatus(_critique);
return Task.FromResult(new AuditReport(file.FilePath, _critique, hasPassed));
return Task.FromResult(AuditReport.FromParsedStatus(file.FilePath, _critique));
}

public Task<AuditReport> AnalyzeSystemAsync(
Expand All @@ -350,8 +441,30 @@ public Task<AuditReport> AnalyzeSystemAsync(
LastSystemFileCount = files.Count;
LastManifestText = manifestText;
onTokenReceived(_critique);
bool hasPassed = AuditReportParser.ParsePassStatus(_critique);
return Task.FromResult(new AuditReport(SystemAuditIdentity.ReportBaseName, _critique, hasPassed));
return Task.FromResult(
AuditReport.FromParsedStatus(SystemAuditIdentity.ReportBaseName, _critique));
}
}

private sealed class FailingAiOrchestrator : IAiOrchestrator
{
public Task<AuditReport> AnalyzeCodeAsync(
SourceFile file,
IEnumerable<AuditRule> rules,
Action<string> onTokenReceived)
{
return Task.FromResult(
AuditReport.ToolFailure(file.FilePath, "# Local Engine Failure\n\nUnavailable"));
}

public Task<AuditReport> AnalyzeSystemAsync(
IReadOnlyList<SourceFile> files,
string? manifestText,
IEnumerable<AuditRule> rules,
Action<string> onTokenReceived)
{
return Task.FromResult(
AuditReport.ToolFailure(SystemAuditIdentity.ReportBaseName, "# Local Engine Failure\n\nUnavailable"));
}
}
}
Loading
Loading