Skip to content

Reduce audit noise with stack packs and grounded prompts #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TimoKuronen merged 1 commit into from
Sep 28, 2026
Merged

Reduce audit noise with stack packs and grounded prompts #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions CodeSmellAuditor.Cli/CliArgs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ public sealed record CliArgs(
string? Model = null,
string? Storage = null,
bool NonInteractive = false,
string? Manifest = null)
string? Manifest = null,
string? Stack = null)
{
public static CliArgs Parse(string[] args)
{
string? model = null;
string? storage = null;
string? manifest = null;
string? stack = null;
bool nonInteractive = false;
var positional = new List<string>();

Expand All @@ -45,6 +47,12 @@ public static CliArgs Parse(string[] args)
continue;
}

if (string.Equals(token, "--stack", StringComparison.OrdinalIgnoreCase))
{
stack = RequireOptionValue(args, ref i, "--stack");
continue;
}

if (string.Equals(token, "--non-interactive", StringComparison.OrdinalIgnoreCase))
{
nonInteractive = true;
Expand All @@ -54,7 +62,7 @@ public static CliArgs Parse(string[] args)
if (token.StartsWith('-'))
{
throw new ArgumentException(
$"Unknown option '{token}'. Supported: --model, --storage, --manifest, --non-interactive.");
$"Unknown option '{token}'. Supported: --model, --storage, --manifest, --stack, --non-interactive.");
}

positional.Add(token);
Expand All @@ -67,7 +75,7 @@ public static CliArgs Parse(string[] args)
throw new ArgumentException("--manifest is only valid with sniff-system.");
}

return new CliArgs(CliMode.Batch, Array.Empty<string>(), model, storage, nonInteractive);
return new CliArgs(CliMode.Batch, Array.Empty<string>(), model, storage, nonInteractive, Stack: stack);
}

if (string.Equals(positional[0], "sniff", StringComparison.OrdinalIgnoreCase))
Expand All @@ -78,13 +86,13 @@ public static CliArgs Parse(string[] args)
}

IReadOnlyList<string> sniffPaths = ParseCsPaths(positional, minCount: 1, commandName: "sniff");
return new CliArgs(CliMode.Sniff, sniffPaths, model, storage, nonInteractive);
return new CliArgs(CliMode.Sniff, sniffPaths, model, storage, nonInteractive, Stack: stack);
}

if (string.Equals(positional[0], "sniff-system", StringComparison.OrdinalIgnoreCase))
{
IReadOnlyList<string> systemPaths = ParseCsPaths(positional, minCount: 2, commandName: "sniff-system");
return new CliArgs(CliMode.SniffSystem, systemPaths, model, storage, nonInteractive, manifest);
return new CliArgs(CliMode.SniffSystem, systemPaths, model, storage, nonInteractive, manifest, stack);
}

throw new ArgumentException(
Expand Down
27 changes: 20 additions & 7 deletions CodeSmellAuditor.Cli/CliAuditHost.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ public CliAuditHost(AuditEngine engine)
_engine = engine;
}

public async Task<AuditRunResult> RunAsync(string rulesPath, string targetsPath)
public async Task<AuditRunResult> RunAsync(string rulesPath, string targetsPath, string? stackName = null)
{
IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(rulesPath);
IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(rulesPath, stackName);
PrintRulesLoaded(rules);

IReadOnlyList<string> targetFiles = AuditEngine.EnumerateTargetFiles(targetsPath);
Expand Down Expand Up @@ -53,7 +53,10 @@ public async Task<AuditRunResult> RunAsync(string rulesPath, string targetsPath)
/// <summary>
/// Audits one or more external .cs files in place. Rules load once; reports stay under auditor storage.
/// </summary>
public async Task<AuditRunResult> RunSniffAsync(string rulesPath, IReadOnlyList<string> filePaths)
public async Task<AuditRunResult> RunSniffAsync(
string rulesPath,
IReadOnlyList<string> filePaths,
string? stackName = null)
{
if (filePaths.Count == 0)
{
Expand All @@ -77,7 +80,7 @@ public async Task<AuditRunResult> RunSniffAsync(string rulesPath, IReadOnlyList<
absolutePaths.Add(absolutePath);
}

IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(rulesPath);
IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(rulesPath, stackName);
PrintRulesLoaded(rules);

string reportsDirectoryPath = AuditEngine.ResolveReportsDirectory(rulesPath);
Expand Down Expand Up @@ -107,7 +110,8 @@ public async Task<AuditRunResult> RunSniffAsync(string rulesPath, IReadOnlyList<
public async Task<AuditRunResult> RunSniffSystemAsync(
string rulesPath,
IReadOnlyList<string> filePaths,
string? manifestPath)
string? manifestPath,
string? stackName = null)
{
if (filePaths.Count < 2)
{
Expand Down Expand Up @@ -151,13 +155,22 @@ public async Task<AuditRunResult> RunSniffSystemAsync(
$"Architecture rules folder not found: {architectureRulesPath}");
}

IReadOnlyList<AuditRule> rules = await _engine.LoadRulesAsync(architectureRulesPath);
if (rules.Count == 0)
// Architecture packs are the primary rules for system audits; optional stack packs
// layer on from the root Rules/Stacks folder (not under Architecture/).
IReadOnlyList<AuditRule> architectureRules = await _engine.LoadRulesAsync(architectureRulesPath);
if (architectureRules.Count == 0)
{
throw new InvalidOperationException(
$"No architecture rule packs found under {architectureRulesPath}");
}

IReadOnlyList<AuditRule> rules = string.IsNullOrWhiteSpace(stackName)
? architectureRules
: await _engine.LoadRulesAsync(
architectureRulesPath,
stackName,
stackRootRulesPath: rulesPath);

PrintRulesLoaded(rules);

string reportsDirectoryPath = AuditEngine.ResolveReportsDirectory(rulesPath);
Expand Down
12 changes: 9 additions & 3 deletions CodeSmellAuditor.Cli/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,14 @@ static void WaitForEnterIfInteractive(CliArgs parsed)
var engine = new AuditEngine(repository, aiService);
var host = new CliAuditHost(engine);

if (!string.IsNullOrWhiteSpace(cliArgs.Stack))
{
AnsiConsole.MarkupLine($"[grey]Stack pack:[/] [cyan]{Markup.Escape(cliArgs.Stack)}[/]");
}

if (cliArgs.Mode == CliMode.Batch)
{
AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath);
AuditRunResult batchResult = await host.RunAsync(rulesPath, targetsPath, cliArgs.Stack);
Environment.ExitCode = batchResult.ExitCode;
AnsiConsole.MarkupLine(
batchResult.AllPassed
Expand All @@ -132,12 +137,13 @@ static void WaitForEnterIfInteractive(CliArgs parsed)
AuditRunResult systemResult = await host.RunSniffSystemAsync(
rulesPath,
cliArgs.SniffPaths,
cliArgs.Manifest);
cliArgs.Manifest,
cliArgs.Stack);
Environment.ExitCode = systemResult.ExitCode;
return;
}

AuditRunResult sniffResult = await host.RunSniffAsync(rulesPath, cliArgs.SniffPaths);
AuditRunResult sniffResult = await host.RunSniffAsync(rulesPath, cliArgs.SniffPaths, cliArgs.Stack);
Environment.ExitCode = sniffResult.ExitCode;
}
catch (Exception ex) when (ex is FileNotFoundException or ArgumentException or DirectoryNotFoundException or InvalidOperationException)
Expand Down
126 changes: 126 additions & 0 deletions CodeSmellAuditor.Core.Tests/AuditEngineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,132 @@ public void ResolveArchitectureRulesPath_AppendsArchitectureFolder()
Assert.Equal(Path.Combine(@"D:\Storage\Rules", "Architecture"), resolved);
}

[Fact]
public void ResolveStackRulesPath_AppendsStacksFolderAndName()
{
string resolved = AuditEngine.ResolveStackRulesPath(@"D:\Storage\Rules", "Unity");

Assert.Equal(Path.Combine(@"D:\Storage\Rules", "Stacks", "Unity"), resolved);
}

[Theory]
[InlineData("")]
[InlineData(" ")]
public void ResolveStackRulesPath_RejectsEmptyName(string stackName)
{
Assert.Throws<ArgumentException>(() =>
AuditEngine.ResolveStackRulesPath(@"D:\Storage\Rules", stackName));
}

[Theory]
[InlineData("../Unity")]
[InlineData("Unity/Extra")]
[InlineData(@"Unity\Extra")]
public void ResolveStackRulesPath_RejectsPathTraversal(string stackName)
{
Assert.Throws<ArgumentException>(() =>
AuditEngine.ResolveStackRulesPath(@"D:\Storage\Rules", stackName));
}

[Fact]
public async Task LoadRulesAsync_WithStack_ConcatenatesBaseAndStackRules()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
string stackPath = Path.Combine(rulesPath, "Stacks", "Unity");
Directory.CreateDirectory(rulesPath);
Directory.CreateDirectory(stackPath);

await File.WriteAllTextAsync(
Path.Combine(rulesPath, "base.mdc"),
"Base rule body.");
await File.WriteAllTextAsync(
Path.Combine(stackPath, "unity.mdc"),
"Unity rule body.");

var engine = new AuditEngine(
new MarkdownRuleRepository(),
new FakeAiOrchestrator("Status: COMPLIANT"));

IReadOnlyList<AuditRule> baseOnly = await engine.LoadRulesAsync(rulesPath);
IReadOnlyList<AuditRule> withStack = await engine.LoadRulesAsync(rulesPath, "Unity");

Assert.Single(baseOnly);
Assert.Equal(2, withStack.Count);
Assert.Contains(withStack, r => r.Name == "base.mdc");
Assert.Contains(withStack, r => r.Name == "unity.mdc");
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

[Fact]
public async Task LoadRulesAsync_WithStack_UsesStackRootWhenProvided()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
string architecturePath = Path.Combine(rulesPath, "Architecture");
string stackPath = Path.Combine(rulesPath, "Stacks", "Unity");
Directory.CreateDirectory(architecturePath);
Directory.CreateDirectory(stackPath);

await File.WriteAllTextAsync(
Path.Combine(architecturePath, "arch.mdc"),
"Architecture rule.");
await File.WriteAllTextAsync(
Path.Combine(stackPath, "unity.mdc"),
"Unity rule.");

var engine = new AuditEngine(
new MarkdownRuleRepository(),
new FakeAiOrchestrator("Status: COMPLIANT"));

IReadOnlyList<AuditRule> rules = await engine.LoadRulesAsync(
architecturePath,
"Unity",
stackRootRulesPath: rulesPath);

Assert.Equal(2, rules.Count);
Assert.Contains(rules, r => r.Name == "arch.mdc");
Assert.Contains(rules, r => r.Name == "unity.mdc");
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

[Fact]
public async Task LoadRulesAsync_WithMissingStack_Throws()
{
string workspace = CreateTempWorkspace();

try
{
string rulesPath = Path.Combine(workspace, "Rules");
Directory.CreateDirectory(rulesPath);

var engine = new AuditEngine(
new MarkdownRuleRepository(),
new FakeAiOrchestrator("Status: COMPLIANT"));

await Assert.ThrowsAsync<DirectoryNotFoundException>(() =>
engine.LoadRulesAsync(rulesPath, "Unity"));
}
finally
{
Directory.Delete(workspace, recursive: true);
}
}

private static string CreateTempWorkspace()
{
return Path.Combine(Path.GetTempPath(), $"codesmell-engine-{Guid.NewGuid():N}");
Expand Down
4 changes: 4 additions & 0 deletions CodeSmellAuditor.Core.Tests/AuditPromptBuilderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ public void BuildSystemPrompt_IncludesRulesAndOutputTemplate()
Assert.Contains("EXACTLY one of", prompt);
Assert.Contains("omit Missing Context", prompt);
Assert.Contains("No analysis traces", prompt);
Assert.Contains("Grounding rules:", prompt);
Assert.Contains("visible in the provided source text", prompt);
Assert.Contains("Missing Context as an open question", prompt);
Assert.Contains("not a regression metric", prompt);
}

[Fact]
Expand Down
46 changes: 46 additions & 0 deletions CodeSmellAuditor.Core.Tests/CliArgsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public void Parse_NoArgs_ReturnsBatchMode()
Assert.False(result.NonInteractive);
Assert.Null(result.Model);
Assert.Null(result.Storage);
Assert.Null(result.Stack);
}

[Fact]
Expand Down Expand Up @@ -97,15 +98,60 @@ public void Parse_BatchWithFlags_ParsesOptions()
{
"--model", "qwen3.5:4b",
"--storage", @"D:\Storage",
"--stack", "Unity",
"--non-interactive"
});

Assert.Equal(CliMode.Batch, result.Mode);
Assert.Equal("qwen3.5:4b", result.Model);
Assert.Equal(@"D:\Storage", result.Storage);
Assert.Equal("Unity", result.Stack);
Assert.True(result.NonInteractive);
}

[Fact]
public void Parse_SniffWithStack_ParsesStack()
{
CliArgs result = CliArgs.Parse(new[]
{
"sniff",
"Program.cs",
"--stack",
"Unity"
});

Assert.Equal(CliMode.Sniff, result.Mode);
Assert.Equal("Unity", result.Stack);
}

[Fact]
public void Parse_SniffSystemWithStack_ParsesStack()
{
CliArgs result = CliArgs.Parse(new[]
{
"sniff-system",
"A.cs",
"B.cs",
"--stack",
"Unity",
"--manifest",
@"D:\manifest.md"
});

Assert.Equal(CliMode.SniffSystem, result.Mode);
Assert.Equal("Unity", result.Stack);
Assert.Equal(@"D:\manifest.md", result.Manifest);
}

[Fact]
public void Parse_StackMissingValue_Throws()
{
ArgumentException ex = Assert.Throws<ArgumentException>(
() => CliArgs.Parse(new[] { "--stack" }));

Assert.Contains("--stack requires a value", ex.Message);
}

[Fact]
public void Parse_SniffWithFlagsBeforeAndAfter_ParsesAll()
{
Expand Down
3 changes: 3 additions & 0 deletions CodeSmellAuditor.Core.Tests/SystemAuditPromptBuilderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ public void BuildSystemPrompt_IncludesArchitectureFramingAndTemplate()
Assert.Contains("## Manifest Conformance", prompt);
Assert.Contains("## Cross-File Findings", prompt);
Assert.Contains("REVIEW REQUIRED", prompt);
Assert.Contains("Grounding rules:", prompt);
Assert.Contains("visible in the provided source files", prompt);
Assert.Contains("not a regression metric", prompt);
}

[Fact]
Expand Down
Loading
Loading