diff --git a/README.md b/README.md index e9c90f1..aece55e 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The additional detectors use the same persistent signal model. A threat-surface | Secret | Findings from the embedded `git-pkgs/secrets` rule corpus; output contains `[redacted]` in place of the credential | | Sensitive-Ignore | Effective `.gitignore` rules covering a bounded list of environment, credential, private-key, secret, state, and database paths, with the source pattern and line | | REUSE | Per-file SPDX license declarations, REUSE.toml or DEP5 metadata, and the number of non-empty covered files with both license and copyright information | +| Repository-Ownership | CODEOWNERS rules, owner identities, semantic configuration digests, and incomplete coverage, plus content digests for maintainers and governance documents matched by Brief resource definitions | | Security-Changelog | CVE and GHSA references, security-related entries, and list entries under security sections in changelog files recognized by Brief | | Security-Documentation | Threat models, security audit or review files, and disclosure documents matched by Brief resource definitions outside bundled or derived paths | | Artifact-Signing | GoReleaser artifact, container, binary, and native package signing configuration, including the mechanism, artifact type, and selected IDs | @@ -206,7 +207,7 @@ The additional detectors use the same persistent signal model. A threat-surface The root commit and missing history boundaries receive a full scan. Later commits reuse their first parent's signal state and update the paths changed by that commit. Cross-file detectors, including Agent Skills, dependency-update configuration, REUSE metadata, and sensitive ignore policy, receive a full commit tree when one of their inputs changes. Agent Skills rerun when a `SKILL.md` or a file inside a known skill package changes. Sensitive ignore policy reruns only when a project `.gitignore` changes. -Parser results for changed files are cached by Git blob OID and path for the whole run. Repository files that cannot contain an attestation, certificate, license, SARIF log, SBOM, fuzzer, policy, changelog entry, markup with remote subresources, SAST configuration, or shell program need only a 1 KB prefix for binary classification. Attestation parsing is limited to 8 MB per file. Markup scanning reads at most 8 MB per file. Files without a shell extension are read up to 1 MB when the prefix contains a shell shebang. Security-document paths reuse that prefix pass and need no content read. SARIF candidates use streaming byte, tool, and distinct-value limits, and bundled, derived, test, and example paths are skipped for SARIF detection. Commit reading can run concurrently, while signal-state updates remain ordered. `github.com/git-pkgs/languages` keeps a language inventory so Brief can rerun language-gated detection only when needed. `github.com/git-pkgs/roles` removes bundled and derived paths from attestation, fuzzing, license, security-document, shell, and subresource-integrity detection, and removes test, fixture, example, cache, and build-output paths from certificate detection. Test paths remain eligible for fuzz detection because fuzz targets commonly use them. +Parser results for changed files are cached by Git blob OID and path for the whole run. Repository files that cannot contain an attestation, certificate, license, ownership document, SARIF log, SBOM, fuzzer, policy, changelog entry, markup with remote subresources, SAST configuration, or shell program need only a 1 KB prefix for binary classification. Attestation parsing is limited to 8 MB per file. CODEOWNERS parsing reads at most 3 MB and reports incomplete coverage for larger files; maintainer and governance documents use streaming content digests. Markup scanning reads at most 8 MB per file. Files without a shell extension are read up to 1 MB when the prefix contains a shell shebang. Security-document paths reuse that prefix pass and need no content read. SARIF candidates use streaming byte, tool, and distinct-value limits, and bundled, derived, test, and example paths are skipped for SARIF detection. Commit reading can run concurrently, while signal-state updates remain ordered. `github.com/git-pkgs/languages` keeps a language inventory so Brief can rerun language-gated detection only when needed. `github.com/git-pkgs/roles` removes bundled and derived paths from attestation, fuzzing, license, ownership, security-document, shell, and subresource-integrity detection, and removes test, fixture, example, cache, and build-output paths from certificate detection. Test paths remain eligible for fuzz detection because fuzz targets commonly use them. Brief reads commit trees through `history.TreeFS` and reruns only when Brief's changed-file filter identifies an input to one of the emitted tool signals. Security-tool signals require evidence of configured use, such as a dedicated configuration file, declared dependency, action, or content match. A language manifest that only makes a scanner applicable does not emit a `Security-Tool` signal. Package files are parsed when their paths change, and the parse result supplies dependency constraints, pins, Actions references, container images, lockfile resolution, integrity records, and manifest-declared license values or file references. Results are cached by blob OID and path. Shell parsing follows downloads into checksum checks and later execution, while package commands use the command definitions loaded from `github.com/git-pkgs/managers`. The same parser handles shell files, Dockerfile `RUN` instructions, and Actions `run` steps. Package-manager detection uses `managers.AffectedDirectories` to rescan only the affected projects. Secret scanning updates changed paths and caches results by blob OID and path because some rules depend on the filename. REUSE metadata changes trigger a full REUSE scan; ordinary changes update the affected file state and coverage count. diff --git a/detector.go b/detector.go index 86b6b27..cd1429a 100644 --- a/detector.go +++ b/detector.go @@ -16,6 +16,7 @@ const ( checkDependencyUpdateTool = "Dependency-Update-Tool" checkFuzzing = "Fuzzing" checkLicense = "License" + checkRepositoryOwnership = "Repository-Ownership" checkSARIF = "SARIF" checkSAST = "SAST" checkSBOM = "SBOM" diff --git a/ownership.go b/ownership.go new file mode 100644 index 0000000..4e33503 --- /dev/null +++ b/ownership.go @@ -0,0 +1,195 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "regexp" + "sort" + "strings" + + "github.com/git-pkgs/brief/kb" +) + +const ( + ownershipKindCodeowners = "codeowners" + ownershipKindOwner = "owner" + ownershipKindMaintainers = "maintainers" + ownershipKindGovernance = "governance" + ownershipKindCoverage = "coverage" + ownershipTruncatedName = "truncated" + ownershipSummaryName = "summary" + codeownersContentBytes = 3 << 20 +) + +var codeownerEmailPattern = regexp.MustCompile("^[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,63}$") + +type ownershipRules struct { + codeowners *kb.ResourceInfo + maintainers *kb.ResourceInfo + governance *kb.ResourceInfo +} + +func newOwnershipRules(knowledge *kb.KnowledgeBase) *ownershipRules { + return &ownershipRules{ + codeowners: knowledge.Resource("community", ownershipKindCodeowners), + maintainers: knowledge.Resource("community", ownershipKindMaintainers), + governance: knowledge.Resource("community", ownershipKindGovernance), + } +} + +func (rules *ownershipRules) kind(filePath string) string { + if rules == nil { + return "" + } + var kind string + switch { + case matchesResource(rules.codeowners, filePath): + kind = ownershipKindCodeowners + case matchesResource(rules.maintainers, filePath): + kind = ownershipKindMaintainers + case matchesResource(rules.governance, filePath): + kind = ownershipKindGovernance + } + if kind == "" || !repositoryProjectEvidencePath(filePath) { + return "" + } + return kind +} + +func codeownersSignals(filePath string, data []byte, size int64) []Signal { + owners := make(map[string]bool) + var statements []string + ruleCount := 0 + gitlabSyntax := strings.HasPrefix(strings.ToLower(filePath), ".gitlab/") + for _, line := range bytes.Split(data, []byte{'\n'}) { + statement := codeownersStatement(string(line), gitlabSyntax) + if statement == "" { + continue + } + fields := strings.Fields(statement) + hasOwner := false + for _, field := range fields[1:] { + if codeownerIdentity(field) { + owners[field] = true + hasOwner = true + } + } + section := codeownersSection(statement) + if !section && !gitlabSyntax && !hasOwner { + continue + } + statements = append(statements, strings.Join(fields, " ")) + if !section { + ruleCount++ + } + } + summary := fmt.Sprintf( + "rules=%d; owners=%d; config=%s", + ruleCount, + len(owners), + contentDigest([]byte(strings.Join(statements, "\n"))), + ) + signals := []Signal{{ + Check: checkRepositoryOwnership, + Kind: ownershipKindCodeowners, + Name: ownershipSummaryName, + Value: summary, + Path: filePath, + }} + ownerNames := make([]string, 0, len(owners)) + for owner := range owners { + ownerNames = append(ownerNames, owner) + } + sort.Strings(ownerNames) + for _, owner := range ownerNames { + signals = append(signals, Signal{ + Check: checkRepositoryOwnership, + Kind: ownershipKindOwner, + Name: owner, + Path: filePath, + }) + } + if size > codeownersContentBytes { + signals = append(signals, Signal{ + Check: checkRepositoryOwnership, + Kind: ownershipKindCoverage, + Name: ownershipTruncatedName, + Value: fmt.Sprintf("bytes=%d; limit=%d", size, codeownersContentBytes), + Path: filePath, + }) + } + return signals +} + +func repositoryOwnershipSignals( + tree fs.FS, + filePath string, + size int64, + rules *ownershipRules, +) ([]Signal, error) { + kind := rules.kind(filePath) + if kind == "" { + return nil, nil + } + if kind == ownershipKindCodeowners { + data, err := readRepositoryFile(tree, filePath, codeownersContentBytes) + if err != nil { + return nil, err + } + return codeownersSignals(filePath, data, size), nil + } + digest, err := ownershipDocumentDigest(tree, filePath) + if err != nil { + return nil, err + } + return []Signal{{ + Check: checkRepositoryOwnership, + Kind: kind, + Value: digest, + Path: filePath, + }}, nil +} + +func ownershipDocumentDigest(tree fs.FS, filePath string) (string, error) { + hasher := sha256.New() + err := scanRepositoryFileError(tree, filePath, func(reader io.Reader) error { + _, err := io.Copy(hasher, reader) + return err + }) + digest := hasher.Sum(nil) + return "sha256:" + hex.EncodeToString(digest[:6]), err +} + +func codeownersStatement(line string, gitlabSyntax bool) string { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, `\#`) { + return "" + } + if gitlabSyntax { + return line + } + fields := strings.Fields(line) + for index, field := range fields { + if strings.HasPrefix(field, "#") { + fields = fields[:index] + break + } + } + return strings.Join(fields, " ") +} + +func codeownerIdentity(value string) bool { + return (len(value) > 1 && strings.HasPrefix(value, "@")) || codeownerEmailPattern.MatchString(value) +} + +func codeownersSection(statement string) bool { + fields := strings.Fields(statement) + if len(fields) == 0 { + return false + } + return strings.HasPrefix(strings.TrimPrefix(fields[0], "^"), "[") +} diff --git a/ownership_test.go b/ownership_test.go new file mode 100644 index 0000000..c60298f --- /dev/null +++ b/ownership_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + testCodeownersPath = ".gitlab/CODEOWNERS" + testMaintainersPath = "MAINTAINERS.md" + testOwnershipLogCommand = "log" +) + +func TestRepositoryOwnershipSignals(t *testing.T) { + data := []byte(`# project owners +* @example/core security@example.com +/docs/ @example/docs +escaped\#name @example/core # retained pattern, ignored comment +[Backend][2] @example/backend # @example/comment-owner +`) + signals := repositorySignals(t, testCodeownersPath, data) + summary := findOwnershipSignal(t, signals, ownershipKindCodeowners, "summary") + if !strings.HasPrefix(summary.Value, "rules=3; owners=5; config=") { + t.Fatalf("summary = %#v", summary) + } + for _, owner := range []string{ + "@example/backend", + "@example/comment-owner", + "@example/core", + "@example/docs", + "security@example.com", + } { + findOwnershipSignal(t, signals, ownershipKindOwner, owner) + } + + for _, test := range []struct { + path string + kind string + }{ + {path: testMaintainersPath, kind: ownershipKindMaintainers}, + {path: "docs/GOVERNANCE.md", kind: ownershipKindGovernance}, + } { + data := []byte("project document\n") + found := repositorySignals(t, test.path, data) + signal := findOwnershipSignal(t, found, test.kind, "") + if signal.Value != contentDigest(data) { + t.Errorf("%s digest = %q, want %q", test.path, signal.Value, contentDigest(data)) + } + changed := findOwnershipSignal(t, repositorySignals(t, test.path, []byte("updated document\n")), test.kind, "") + if changed.Value == signal.Value { + t.Errorf("%s digest did not change with content", test.path) + } + } +} + +func TestRepositoryOwnershipSignalsUseGitHubComments(t *testing.T) { + data := []byte("\\#not-a-pattern @example/ignored\nescaped\\#name @example/escaped\n* @example/actual # @example/comment\n") + signals := repositorySignals(t, ".github/CODEOWNERS", data) + summary := findOwnershipSignal(t, signals, ownershipKindCodeowners, "summary") + if !strings.HasPrefix(summary.Value, "rules=2; owners=2; config=") { + t.Fatalf("summary = %#v", summary) + } + findOwnershipSignal(t, signals, ownershipKindOwner, "@example/actual") + findOwnershipSignal(t, signals, ownershipKindOwner, "@example/escaped") + for _, ignored := range []string{"@example/ignored", "@example/comment"} { + for _, signal := range signals { + if signal.Check == checkRepositoryOwnership && signal.Kind == ownershipKindOwner && signal.Name == ignored { + t.Errorf("GitHub comment owner was reported: %#v", signal) + } + } + } +} + +func TestRepositoryOwnershipSignalsSupportsLongRules(t *testing.T) { + data := append(bytes.Repeat([]byte{'x'}, int(repositoryFileContentBytes)-len(" @example/owner")), []byte(" @example/owner")...) + signals := repositorySignals(t, ".github/CODEOWNERS", data) + summary := findOwnershipSignal(t, signals, ownershipKindCodeowners, "summary") + if !strings.HasPrefix(summary.Value, "rules=1; owners=1; config=") { + t.Fatalf("summary = %#v", summary) + } + findOwnershipSignal(t, signals, ownershipKindOwner, "@example/owner") +} + +func TestRepositoryOwnershipSignalsReportTruncatedCodeowners(t *testing.T) { + prefix := []byte("* @example/owner\n") + data := append([]byte(nil), prefix...) + data = append(data, bytes.Repeat([]byte{'x'}, codeownersContentBytes+1-len(prefix))...) + signals := repositorySignals(t, ".github/CODEOWNERS", data) + findOwnershipSignal(t, signals, ownershipKindOwner, "@example/owner") + coverage := findOwnershipSignal(t, signals, ownershipKindCoverage, ownershipTruncatedName) + want := fmt.Sprintf("bytes=%d; limit=%d", len(data), codeownersContentBytes) + if coverage.Value != want { + t.Fatalf("coverage value = %q, want %q", coverage.Value, want) + } +} + +func TestRepositoryOwnershipSignalsExcludeBundledDocuments(t *testing.T) { + for _, filePath := range []string{ + "vendor/dependency/CODEOWNERS", + "testdata/project/MAINTAINERS.md", + "fixtures/repository/GOVERNANCE.md", + } { + if signals := repositorySignals(t, filePath, []byte("* @dependency/owners\n")); len(signals) != 0 { + t.Errorf("%s signals = %#v", filePath, signals) + } + } +} + +func TestLogTracksRepositoryOwnershipChanges(t *testing.T) { + repo := t.TempDir() + gitRun(t, repo, "init", "-b", "main") + gitRun(t, repo, "config", "user.name", "Signals Test") + gitRun(t, repo, "config", "user.email", "signals@example.com") + gitRun(t, repo, "config", "commit.gpgsign", "false") + + writeFile(t, repo, "README.md", "# example\n") + gitRun(t, repo, "add", ".") + gitRun(t, repo, "commit", "-m", "Initial commit") + + writeFile(t, repo, testCodeownersPath, "* @example/old\n") + writeFile(t, repo, testMaintainersPath, "Old Maintainer \n") + gitRun(t, repo, "add", ".") + gitRun(t, repo, "commit", "-m", "Add ownership") + + writeFile(t, repo, testCodeownersPath, "* @example/new\n") + writeFile(t, repo, testMaintainersPath, "New Maintainer \n") + gitRun(t, repo, "add", ".") + gitRun(t, repo, "commit", "-m", "Change ownership") + + if err := os.Remove(filepath.Join(repo, testCodeownersPath)); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(repo, testMaintainersPath)); err != nil { + t.Fatal(err) + } + gitRun(t, repo, "add", "-u") + gitRun(t, repo, "commit", "-m", "Remove ownership") + + var stdout, stderr bytes.Buffer + if err := run([]string{testOwnershipLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) + } + var records []jsonlRecord + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + var record jsonlRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatalf("decode JSONL: %v\n%s", err, line) + } + records = append(records, record) + } + added := findJSONLRecord(t, records, "Add ownership") + changed := findJSONLRecord(t, records, "Change ownership") + removed := findJSONLRecord(t, records, "Remove ownership") + assertOwnershipChange(t, added.Changes, "added", "@example/old") + assertOwnershipChange(t, changed.Changes, "removed", "@example/old") + assertOwnershipChange(t, changed.Changes, "added", "@example/new") + assertOwnershipChange(t, removed.Changes, "removed", "@example/new") + oldMaintainers := contentDigest([]byte("Old Maintainer \n")) + newMaintainers := contentDigest([]byte("New Maintainer \n")) + assertOwnershipDocumentChange(t, added.Changes, "added", oldMaintainers) + assertOwnershipDocumentChange(t, changed.Changes, "removed", oldMaintainers) + assertOwnershipDocumentChange(t, changed.Changes, "added", newMaintainers) + assertOwnershipDocumentChange(t, removed.Changes, "removed", newMaintainers) + if stderr.Len() != 0 { + t.Fatalf("unexpected detection errors:\n%s", stderr.String()) + } +} + +func findOwnershipSignal(t *testing.T, signals []Signal, kind, name string) Signal { + t.Helper() + for _, signal := range signals { + if signal.Check == checkRepositoryOwnership && signal.Kind == kind && signal.Name == name { + return signal + } + } + t.Fatalf("ownership signal kind=%q name=%q not found: %#v", kind, name, signals) + return Signal{} +} + +func assertOwnershipChange(t *testing.T, changes []jsonlChange, change, name string) { + t.Helper() + for _, candidate := range changes { + if candidate.Change == change && candidate.Check == checkRepositoryOwnership && + candidate.Kind == ownershipKindOwner && candidate.Name == name && candidate.Path == testCodeownersPath { + return + } + } + t.Errorf("ownership change=%q name=%q not found: %#v", change, name, changes) +} + +func assertOwnershipDocumentChange(t *testing.T, changes []jsonlChange, change, value string) { + t.Helper() + for _, candidate := range changes { + if candidate.Change == change && candidate.Check == checkRepositoryOwnership && + candidate.Kind == ownershipKindMaintainers && candidate.Value == value && + candidate.Path == testMaintainersPath { + return + } + } + t.Errorf("maintainers change=%q value=%q not found: %#v", change, value, changes) +} diff --git a/repository_files.go b/repository_files.go index 3a54199..1879a08 100644 --- a/repository_files.go +++ b/repository_files.go @@ -124,6 +124,7 @@ type repositoryFileCacheKey struct { type repositoryPathRules struct { changelog *kb.ResourceInfo + ownership *ownershipRules policy *kb.ResourceInfo sbom *kb.ResourceInfo qodana *kb.ToolDef @@ -134,6 +135,7 @@ type repositoryPathRules struct { func newRepositoryPathRules(knowledge *kb.KnowledgeBase) *repositoryPathRules { return &repositoryPathRules{ changelog: knowledge.Resource("", "changelog"), + ownership: newOwnershipRules(knowledge), policy: knowledge.Resource("security", "policy"), sbom: knowledge.Resource("metadata", "sbom"), qodana: knowledge.ByName["Qodana"], @@ -237,7 +239,11 @@ func repositoryFileSignalsFromFS( if signal, ok := binaryArtifactSignal(filePath, prefix, true); ok { signals = append(signals, signal) } - signals = append(signals, rules.securityDocumentSignals(filePath)...) + documents, err := repositoryDocumentSignals(tree, filePath, size, rules) + if err != nil { + return nil, err + } + signals = append(signals, documents...) certificates, err := repositoryCertificateSignals(ctx, tree, filePath, prefix, size) if err != nil { @@ -281,12 +287,6 @@ func repositoryFileSignalsFromFS( signals = append(signals, parsed...) } - parsed, err := repositoryChangelogSignals(tree, filePath, rules) - if err != nil { - return nil, err - } - signals = append(signals, parsed...) - if subresourceMarkupPath(filePath) { parsed, err := scanRepositoryFile(tree, filePath, func(reader io.Reader) ([]Signal, error) { return subresourceIntegritySignals(filePath, reader) @@ -319,7 +319,7 @@ func repositoryFileSignalsFromFS( } } - parsed, err = shellRepositoryFileSignals(tree, filePath, prefix, shell) + parsed, err := shellRepositoryFileSignals(tree, filePath, prefix, shell) if err != nil { return nil, err } @@ -1085,6 +1085,22 @@ func (rules *repositoryPathRules) securityDocumentSignals(filePath string) []Sig return rules.securityDocuments.signals(filePath) } +func repositoryDocumentSignals( + tree fs.FS, + filePath string, + size int64, + rules *repositoryPathRules, +) ([]Signal, error) { + signals := rules.securityDocumentSignals(filePath) + ownership, err := repositoryOwnershipSignals(tree, filePath, size, rules.ownership) + if err != nil { + return nil, err + } + signals = append(signals, ownership...) + changelog, err := repositoryChangelogSignals(tree, filePath, rules) + return append(signals, changelog...), err +} + func repositoryChangelogSignals(tree fs.FS, filePath string, rules *repositoryPathRules) ([]Signal, error) { if !rules.changelogPath(filePath) { return nil, nil @@ -1314,7 +1330,8 @@ func (d *localDetectors) runRepositoryFiles( delete(current.errors, detectorRepositoryFiles) checks := []string{ - checkArtifactSigning, checkAttestation, checkBinaryArtifacts, checkCertificate, checkFuzzing, checkLicense, checkSARIF, + checkArtifactSigning, checkAttestation, checkBinaryArtifacts, checkCertificate, checkFuzzing, checkLicense, + checkRepositoryOwnership, checkSARIF, checkSAST, checkSBOM, checkSBOMGeneration, checkSecurityChangelog, checkSecurityDocumentation, checkSecurityPolicy, checkShellDependency, checkSubresourceIntegrity,