diff --git a/README.md b/README.md index aece55e..7705629 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ The additional detectors use the same persistent signal model. A threat-surface | Dependency-Update-Config | Dependabot and Renovate configuration, including package managers, directories, schedules, cooldowns, groups, and semantic configuration changes | | Package-Manager | The selected package manager for each detected package ecosystem and project directory | | Package-File | Manifest and lockfile presence, including files recognized by package-manager definitions before dependency parsing is available | -| Package-Provenance | Explicit npm publication provenance settings in `publishConfig.provenance` or `.npmrc` | +| Package-Provenance | Explicit npm provenance settings in parsed manifests or `.npmrc`, npm publish flags or environment settings in workflows, and PyPI publication configured with an OIDC token and no password or uv token | | Dependency-Constraint | Ranged, unparsed, and unversioned direct dependency constraints as written in parsed manifests | | Dependency-Pin | Exact direct dependency constraints found in parsed manifests | | Dependency-Pinning | Exact, ranged, unparsed, and resolved dependency counts from manifests and lockfiles | @@ -190,7 +190,6 @@ The additional detectors use the same persistent signal model. A threat-surface | Cache-Mode | Explicit workflow and job cache access modes parsed from Actions workflows | | Job-Timeout | Configured Actions job timeouts, including expression-backed values | | Packaging | Literal packaging and publication commands or known publication Actions; presence does not show that a workflow ran | -| Package-Provenance | Explicit npm provenance flags or environment settings, and PyPI publication configured with an OIDC token | | Runner-Selection | Explicit self-hosted runner labels, runner groups, and dynamic runner expressions in GitHub Actions jobs | | SARIF | Bounded summaries of committed SARIF tools, result counts and levels, baselines, rules, suppressions, and CWE taxa with names and categories | | Package-Parse | Package files that were identified but could not be parsed | @@ -218,6 +217,7 @@ The local detectors share parsers and changed-path caches across related signals - [`github.com/git-pkgs/attestation`](https://github.com/git-pkgs/attestation) parses claims from in-toto statements, DSSE envelopes, and Sigstore bundles without performing verification. - [`github.com/git-pkgs/magic`](https://github.com/git-pkgs/magic) detects binary content. - [`github.com/git-pkgs/certs`](https://github.com/git-pkgs/certs) scans bounded certificate files, trust stores, and small archives. +- [`github.com/git-pkgs/changelog`](https://github.com/git-pkgs/changelog) parses headings and list entries from changelog files. - [`github.com/git-pkgs/gitignore`](https://github.com/git-pkgs/gitignore) resolves effective ignore rules and returns their source pattern and line. - [`github.com/git-pkgs/integrity`](https://github.com/git-pkgs/integrity) parses and normalizes Subresource Integrity metadata. - [`github.com/git-pkgs/licenses`](https://github.com/git-pkgs/licenses) identifies license text, notices, and SPDX declarations with its embedded ScanCode corpus. @@ -229,10 +229,12 @@ The local detectors share parsers and changed-path caches across related signals - [`github.com/git-pkgs/cwe`](https://github.com/git-pkgs/cwe) adds names and categories to CWE taxa referenced by SARIF results. - [`github.com/git-pkgs/brief`](https://github.com/git-pkgs/brief) detects CI systems, build tools, security tools, and pre-commit runners from an `fs.FS` commit tree. - [`github.com/git-pkgs/languages`](https://github.com/git-pkgs/languages) detects source languages from paths and blob content. +- [`github.com/git-pkgs/mailmap`](https://github.com/git-pkgs/mailmap) canonicalizes contributor identities for commit metrics. - [`github.com/git-pkgs/reuse`](https://github.com/git-pkgs/reuse) resolves SPDX headers, sidecars, REUSE.toml annotations, and DEP5 metadata from an `fs.FS` commit tree. - [`github.com/git-pkgs/roles`](https://github.com/git-pkgs/roles) classifies CI, fuzz, legal, generated, vendored, and packaging paths. - [`github.com/git-pkgs/secrets`](https://github.com/git-pkgs/secrets) scans changed blob content with its embedded local rule corpus. - [`github.com/git-pkgs/skills`](https://github.com/git-pkgs/skills) discovers and parses Agent Skill packages from commit trees. +- [`github.com/git-pkgs/vers`](https://github.com/git-pkgs/vers) parses native package constraints for dependency classification. Actionlint parses each changed Actions workflow once for Artifact-Provenance, Dangerous-Workflow, SAST, Token-Permissions, Cache-Mode, Job-Timeout, Packaging, Package-Provenance, and Runner-Selection signals. Package parsing separately records Actions references and their pinning state. Packaging observations cover known Actions and literal commands in `run:` steps without treating their presence as evidence of a successful workflow run. Package provenance requires an explicit npm setting, or an OIDC-enabled PyPI publication without a configured password or uv token. Runner observations record configuration without determining which machine handled a particular job. diff --git a/agent_skills.go b/agent_skills.go index 453e79f..a6aee50 100644 --- a/agent_skills.go +++ b/agent_skills.go @@ -26,7 +26,7 @@ func agentSkillSignals(tree fs.FS) ([]Signal, error) { if parseErr != nil { signals = append(signals, Signal{ Check: checkAgentSkill, - Kind: "invalid", + Kind: signalInvalid, Name: parseErr.Error(), }) return nil diff --git a/commit_stream.go b/commit_stream.go index a3e8e07..18ed8d1 100644 --- a/commit_stream.go +++ b/commit_stream.go @@ -280,31 +280,7 @@ func (s *commitStream) evictBefore(cutoff time.Time) { write++ continue } - s.commits-- - if entry.signed { - s.signedCommits-- - } - if entry.human { - s.humanCommits-- - if entry.contributor != "" { - s.contributors[entry.contributor]-- - if s.contributors[entry.contributor] == 0 { - delete(s.contributors, entry.contributor) - if entry.domain != "" { - s.domains[entry.domain]-- - if s.domains[entry.domain] == 0 { - delete(s.domains, entry.domain) - } - } - } - } - if len(entry.reviewProxies) > 0 { - s.reviewedHumanCommits-- - } - for _, proxy := range entry.reviewProxies { - s.reviewProxyCounts[proxy]-- - } - } + s.removeEntry(entry) } for i := write; i < len(s.entries); i++ { s.entries[i] = commitWindowEntry{} @@ -312,6 +288,35 @@ func (s *commitStream) evictBefore(cutoff time.Time) { s.entries = s.entries[:write] } +func (s *commitStream) removeEntry(entry commitWindowEntry) { + s.commits-- + if entry.signed { + s.signedCommits-- + } + if !entry.human { + return + } + s.humanCommits-- + if entry.contributor != "" { + s.contributors[entry.contributor]-- + if s.contributors[entry.contributor] == 0 { + delete(s.contributors, entry.contributor) + if entry.domain != "" { + s.domains[entry.domain]-- + if s.domains[entry.domain] == 0 { + delete(s.domains, entry.domain) + } + } + } + } + if len(entry.reviewProxies) > 0 { + s.reviewedHumanCommits-- + } + for _, proxy := range entry.reviewProxies { + s.reviewProxyCounts[proxy]-- + } +} + func commitEvents(commit *object.Commit) []jsonlSignal { var events []jsonlSignal if tool := dependencyBot(commit.Author.Name, commit.Author.Email); tool != "" { diff --git a/commit_stream_test.go b/commit_stream_test.go index fa6cd19..0ec2aaa 100644 --- a/commit_stream_test.go +++ b/commit_stream_test.go @@ -36,7 +36,7 @@ func TestLogWritesCommitEvents(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{ - "log", testJSONLFlag, "--after", "2024-02-01", "--before", "2024-03-01", repo, + testLogCommand, testJSONLFlag, testAfterFlag, testAfterDate, testBeforeFlag, testBeforeDate, repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -96,8 +96,8 @@ func TestLogSnapshotsIncludeCommitMetrics(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{ - "log", testJSONLFlag, "--snapshots", - "--after", "2024-02-01", "--before", "2024-03-01", repo, + testLogCommand, testJSONLFlag, testSnapshotsFlag, + testAfterFlag, testAfterDate, testBeforeFlag, testBeforeDate, repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -145,8 +145,8 @@ func TestLogSnapshotsHandleNonMonotonicCommitterDates(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{ - "log", testJSONLFlag, "--snapshots", - "--after", "2024-04-10", "--before", "2024-05-01", repo, + testLogCommand, testJSONLFlag, testSnapshotsFlag, + testAfterFlag, "2024-04-10", testBeforeFlag, "2024-05-01", repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -182,7 +182,7 @@ func TestLogFormatsTextCommitDatesInUTC(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{ - "log", "--snapshots", "--after", "2024-02-01", "--before", "2024-02-02", repo, + testLogCommand, testSnapshotsFlag, testAfterFlag, testAfterDate, testBeforeFlag, "2024-02-02", repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -243,11 +243,19 @@ func TestDependencyBotRecognizesHistoricalDependabotIdentity(t *testing.T) { func TestCommitStreamEvictsOutsideOrderAndKeepsRepositoryAge(t *testing.T) { stream := newCommitStream(mailmap.New()) stream.addEntry(commitEntry(stream.mailmap, testCommitAt(t, "2024-04-02T00:00:00Z", "Bob", "bob@example.org", "Recent commit"))) - stream.addEntry(commitEntry(stream.mailmap, testCommitAt(t, "2024-01-01T00:00:00Z", "Alice", "alice@example.org", "Initial commit"))) + old := testCommitAt( + t, "2024-01-01T00:00:00Z", "Alice", "alice@example.org", + "Initial commit\n\nReviewed-by: Carol ", + ) + old.Signature = "signed" + stream.addEntry(commitEntry(stream.mailmap, old)) metrics := stream.metricsAt(mustTime(t, "2024-04-03T00:00:00Z")) assertCountMetric(t, metrics, checkMaintained, "commits", "", 1) assertCountMetric(t, metrics, checkContributors, "non-bot-authors", "", 1) + assertCountMetric(t, metrics, checkCodeReview, "reviewed-by", "", 0) + assertRatioMetric(t, metrics, checkCodeReview, "proxy-ratio", 0, 1) + assertRatioMetric(t, metrics, checkSignedCommit, "presence-ratio", 0, 1) assertDaysMetric(t, metrics, "days-since-commit", 1) assertDaysMetric(t, metrics, "repository-age", 93) } @@ -356,7 +364,7 @@ func decodeSnapshots(t *testing.T, output []byte) []jsonlSnapshotRecord { if err := decoder.Decode(&record); err != nil { t.Fatal(err) } - if record.Type == "snapshot" { + if record.Type == testSnapshotRecordType { snapshots = append(snapshots, record) } } diff --git a/detector.go b/detector.go index cd1429a..8aa72a8 100644 --- a/detector.go +++ b/detector.go @@ -35,6 +35,7 @@ const ( signalKindLicense = "license" signalKindNotice = "notice" signalKindTool = "tool" + signalInvalid = "invalid" sastToolSonar = "Sonar" ) @@ -54,7 +55,8 @@ func sortSignals(signals []Signal) { } func deduplicateSignals(signals []Signal) []Signal { - if len(signals) < 2 { + const minimumInput = 2 + if len(signals) < minimumInput { return signals } result := signals[:1] diff --git a/goreleaser_test.go b/goreleaser_test.go index a4227f7..d6383ec 100644 --- a/goreleaser_test.go +++ b/goreleaser_test.go @@ -11,9 +11,7 @@ import ( const ( testGoReleaserPath = ".goreleaser.yaml" - testGoReleaserChangeAdded = "added" testGoReleaserChangeRemoved = "removed" - testGoReleaserLogCommand = "log" ) func TestGoReleaserSignals(t *testing.T) { @@ -135,7 +133,7 @@ func TestLogTracksGoReleaserSupplyChainConfiguration(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove release evidence") var stdout, stderr bytes.Buffer - if err := run([]string{testGoReleaserLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } var records []jsonlRecord @@ -150,9 +148,9 @@ func TestLogTracksGoReleaserSupplyChainConfiguration(t *testing.T) { changed := findJSONLRecord(t, records, "Change selected release build") removed := findJSONLRecord(t, records, "Remove release evidence") for _, expected := range []jsonlChange{ - {Change: testGoReleaserChangeAdded, Check: checkArtifactSigning, Kind: goreleaserKindSign, Name: goreleaserGPGCommand, Value: "archive; ids=go", Path: testGoReleaserPath}, - {Change: testGoReleaserChangeAdded, Check: checkSBOMGeneration, Kind: goreleaserKindSBOM, Name: "syft", Value: "archive; ids=go", Path: testGoReleaserPath}, - {Change: testGoReleaserChangeAdded, Check: checkArtifactSigning, Kind: goreleaserKindPackageSign, Name: goreleaserNFPMCommand, Value: "deb; id=packages; ids=go", Path: testGoReleaserPath}, + {Change: testChangeAdded, Check: checkArtifactSigning, Kind: goreleaserKindSign, Name: goreleaserGPGCommand, Value: "archive; ids=go", Path: testGoReleaserPath}, + {Change: testChangeAdded, Check: checkSBOMGeneration, Kind: goreleaserKindSBOM, Name: "syft", Value: "archive; ids=go", Path: testGoReleaserPath}, + {Change: testChangeAdded, Check: checkArtifactSigning, Kind: goreleaserKindPackageSign, Name: goreleaserNFPMCommand, Value: "deb; id=packages; ids=go", Path: testGoReleaserPath}, } { if !containsExactJSONLChange(added.Changes, expected) { t.Errorf("added record missing %#v:\n%s", expected, stdout.String()) diff --git a/history.go b/history.go index 44ed420..eecee78 100644 --- a/history.go +++ b/history.go @@ -199,6 +199,18 @@ func (s *commitStateStore) release(hash plumbing.Hash) { delete(s.states, hash) } +type logScan struct { + repo *githistory.Repo + local *localDetectors + cache *blobCache + stats *scanStats + states *commitStateStore + refEvents map[plumbing.Hash][]jsonlSignal + opts LogOptions + stdout io.Writer + stderr io.Writer +} + func logSignals(path string, opts LogOptions, stdout, stderr io.Writer) (err error) { repo, err := githistory.Open(path) if err != nil { @@ -232,67 +244,18 @@ func logSignals(path string, opts LogOptions, stdout, stderr io.Writer) (err err if err := snapshots.writeStart(stdout, opts.JSONL); err != nil { return err } - err = repo.WalkCommits(walk, func(commit githistory.Commit) error { - forceFull := len(commit.Object.ParentHashes) == 0 - parent := emptyCommitState() - var parentHash plumbing.Hash - if len(commit.Object.ParentHashes) > 0 { - parentHash = commit.Object.ParentHashes[0] - var ok bool - parent, ok = states.get(parentHash) - if !ok { - parentCommit, err := repo.Repository().CommitObject(parentHash) - if err != nil { - return err - } - parent, err = detectFullCommitState(parentCommit, local, cache, stats) - if err != nil { - return err - } - states.retain(parentHash, parent) - forceFull = true - } - } - - current, err := detectCommitState( - commit.Object, - parent, - commit.Changes, - forceFull, - local, - cache, - stats, - ) - if err != nil { - return err - } - if !parentHash.IsZero() { - states.release(parentHash) - } - states.retain(commit.Object.Hash, current) - stats.commits++ - if opts.Progress > 0 && stats.commits%opts.Progress == 0 { - if _, err := fmt.Fprintf(stderr, "signals: scanned %d commits\n", stats.commits); err != nil { - return err - } - } - if !dateInRange(commit.Object.Committer.When, opts.After, opts.Before) { - return nil - } - - var changes []signalChange - var errorChanges []errorChange - if len(commit.Object.ParentHashes) <= 1 { - changes = diffSignals(parent.signals, current.signals) - errorChanges = diffErrors(parent.errors, current.errors) - } - events := append(commitEvents(commit.Object), refEvents[commit.Object.Hash]...) - sortEventSignals(events) - if err := writeCommitRecords(stdout, stderr, commit.Object, changes, errorChanges, events, opts.JSONL); err != nil { - return err - } - return nil - }) + scan := logScan{ + repo: repo, + local: local, + cache: cache, + stats: stats, + states: states, + refEvents: refEvents, + opts: opts, + stdout: stdout, + stderr: stderr, + } + err = repo.WalkCommits(walk, scan.visit) if err == nil { err = snapshots.finish(stdout, opts.JSONL) } @@ -302,6 +265,72 @@ func logSignals(path string, opts LogOptions, stdout, stderr io.Writer) (err err return err } +func (s *logScan) visit(commit githistory.Commit) error { + parent, parentHash, forceFull, err := s.parentState(commit) + if err != nil { + return err + } + current, err := detectCommitState( + commit.Object, + parent, + commit.Changes, + forceFull, + s.local, + s.cache, + s.stats, + ) + if err != nil { + return err + } + if !parentHash.IsZero() { + s.states.release(parentHash) + } + s.states.retain(commit.Object.Hash, current) + s.stats.commits++ + if s.opts.Progress > 0 && s.stats.commits%s.opts.Progress == 0 { + if _, err := fmt.Fprintf(s.stderr, "signals: scanned %d commits\n", s.stats.commits); err != nil { + return err + } + } + if !dateInRange(commit.Object.Committer.When, s.opts.After, s.opts.Before) { + return nil + } + return s.writeRecords(commit, parent, current) +} + +func (s *logScan) parentState(commit githistory.Commit) (commitState, plumbing.Hash, bool, error) { + if len(commit.Object.ParentHashes) == 0 { + return emptyCommitState(), plumbing.ZeroHash, true, nil + } + parentHash := commit.Object.ParentHashes[0] + parent, ok := s.states.get(parentHash) + if ok { + return parent, parentHash, false, nil + } + parentCommit, err := s.repo.Repository().CommitObject(parentHash) + if err != nil { + return commitState{}, plumbing.ZeroHash, false, err + } + parent, err = detectFullCommitState(parentCommit, s.local, s.cache, s.stats) + if err != nil { + return commitState{}, plumbing.ZeroHash, false, err + } + s.states.retain(parentHash, parent) + return parent, parentHash, true, nil +} + +func (s *logScan) writeRecords(commit githistory.Commit, parent, current commitState) error { + var changes []signalChange + var errorChanges []errorChange + if len(commit.Object.ParentHashes) <= 1 { + changes = diffSignals(parent.signals, current.signals) + errorChanges = diffErrors(parent.errors, current.errors) + } + events := append(commitEvents(commit.Object), s.refEvents[commit.Object.Hash]...) + sortEventSignals(events) + return writeCommitRecords(s.stdout, s.stderr, commit.Object, changes, errorChanges, events, s.opts.JSONL) +} + func loadBoundarySnapshot( repo *git.Repository, local *localDetectors, diff --git a/local.go b/local.go index 1ea45eb..975a265 100644 --- a/local.go +++ b/local.go @@ -183,15 +183,10 @@ func (d *localDetectors) detect( managerFull := forceFull || parent.errors[detectorPackageManagers] != "" managerRun := managerFull || managerImpactErr != nil || len(managerDirectories) > 0 secretFull := forceFull || parent.errors[checkSecret] != "" - reuseFull := forceFull || parent.errors[checkREUSE] != "" || reuseMetadataChanged(changes) - if !reuseFull && !parent.reuseActive { - var err error - reuseFull, err = d.reuseChangeMayActivate(changes) - if err != nil { - return commitState{}, err - } + reuseFull, reuseRun, err := d.reuseScanMode(parent, changes, forceFull) + if err != nil { + return commitState{}, err } - reuseRun := reuseFull || parent.reuseActive && len(changes) > 0 var tree fs.FS if anyDetectorRuns(agentSkillRun, briefRun, dependencyUpdateRun, packageRun, repositoryFileRun, sensitiveIgnoreRun, workflowRun, managerRun, reuseRun) { @@ -223,6 +218,22 @@ func (d *localDetectors) detect( return current, nil } +func (d *localDetectors) reuseScanMode( + parent commitState, + changes []githistory.Change, + forceFull bool, +) (bool, bool, error) { + full := forceFull || parent.errors[checkREUSE] != "" || reuseMetadataChanged(changes) + if !full && !parent.reuseActive { + var err error + full, err = d.reuseChangeMayActivate(changes) + if err != nil { + return false, false, err + } + } + return full, full || parent.reuseActive && len(changes) > 0, nil +} + func (d *localDetectors) runAgentSkills( tree fs.FS, run bool, diff --git a/main_test.go b/main_test.go index 6bfba1d..85a4bb8 100644 --- a/main_test.go +++ b/main_test.go @@ -53,7 +53,7 @@ jobs: gitRun(t, repo, "commit", "-m", "Remove security policy") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -109,7 +109,7 @@ jobs: gitRun(t, repo, "commit", "-m", "Move workflow lines") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } output := stdout.String() @@ -158,7 +158,7 @@ jobs: gitRun(t, repo, "commit", "-m", "Remove build provenance") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -237,7 +237,7 @@ func TestLogTracksSecurityDocumentation(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove audit") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -284,7 +284,7 @@ func TestLogUsesBriefSecurityPathDefinitions(t *testing.T) { gitRun(t, repo, "commit", "-m", "Add shared security paths") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -331,7 +331,7 @@ func TestLogDetectsStableSecurityChangelogEntries(t *testing.T) { gitRun(t, repo, "commit", "-m", "Document security advisory") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -377,7 +377,7 @@ func TestLogAcceptsLongSecurityChangelogLines(t *testing.T) { gitRun(t, repo, "commit", "-m", "Document security guidance") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -409,7 +409,7 @@ func TestLogDetectsSubresourceIntegrityChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Protect CDN script") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -469,7 +469,7 @@ updates: gitRun(t, repo, "commit", "-m", "Configure Scala Steward") var stdout, stderr bytes.Buffer - if err := run([]string{"log", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -510,7 +510,7 @@ func TestLogDetectsRenovateInPackageJSON(t *testing.T) { gitRun(t, repo, "commit", "-m", "Configure Renovate") var stdout, stderr bytes.Buffer - if err := run([]string{"log", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -572,7 +572,7 @@ func TestLogDetectsPackagePostureChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Update exact dependency without lockfile") var stdout, stderr bytes.Buffer - if err := run([]string{"log", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -614,7 +614,7 @@ func TestLogDetectsPackageProvenanceChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Disable package provenance") var stdout, stderr bytes.Buffer - if err := run([]string{"log", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ @@ -668,7 +668,7 @@ license = "Apache-2.0" gitRun(t, repo, "commit", "-m", "Update licensing") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -736,7 +736,7 @@ jobs: gitRun(t, repo, "commit", "-m", "Repair workflow") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, testStatsFlag, "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testStatsFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -811,7 +811,7 @@ func TestLogDetectsPackageManagerChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove Nimble project") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -854,7 +854,7 @@ func TestLogComparesBranchCommitsWithTheirFirstParent(t *testing.T) { gitRunAt(t, repo, "2016-12-18T00:03:00Z", "merge", "--no-ff", "feature", "-m", "Merge feature") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -890,7 +890,7 @@ func TestLogUsesLanguagesAndRolesForFuzzing(t *testing.T) { gitRun(t, repo, "commit", "-m", "Add project fuzz target") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -928,7 +928,7 @@ func TestLogTracksThreatSurfaceAcrossHistory(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove Rails") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } var records []jsonlRecord @@ -980,7 +980,7 @@ func TestLogRequiresConfiguredSecurityToolEvidence(t *testing.T) { gitRun(t, repo, "commit", "-m", "Declare Brakeman") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } var records []jsonlRecord @@ -1058,7 +1058,7 @@ func TestLogDetectsLocalToolingSecretsAndREUSEAcrossHistory(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove local security evidence") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } if strings.Contains(stdout.String(), token) { @@ -1077,13 +1077,13 @@ func TestLogDetectsLocalToolingSecretsAndREUSEAcrossHistory(t *testing.T) { removed := findJSONLRecord(t, records, "Remove local security evidence") expectedAdditions := []jsonlChange{ - {Change: "added", Check: checkCI, Kind: "GitHub Actions", Path: ".github/workflows/"}, - {Change: "added", Check: checkPreCommit, Kind: "pre-commit", Path: ".pre-commit-config.yaml"}, - {Change: "added", Check: checkSecretsManagement, Kind: "SOPS", Path: ".sops.yaml"}, - {Change: "added", Check: checkSecurityTool, Kind: testSecurityScanKind, Name: testSemgrepName, Path: testSemgrepConfigPath}, - {Change: "added", Check: checkBuild, Kind: "Make"}, - {Change: "added", Check: checkSecret, Kind: "github-pat", Value: "[redacted]", Path: "config.yml", Line: 1}, - {Change: "added", Check: checkREUSE, Kind: "declaration", Value: testMITLicenseID, Path: "licensed.txt"}, + {Change: testChangeAdded, Check: checkCI, Kind: "GitHub Actions", Path: ".github/workflows/"}, + {Change: testChangeAdded, Check: checkPreCommit, Kind: "pre-commit", Path: ".pre-commit-config.yaml"}, + {Change: testChangeAdded, Check: checkSecretsManagement, Kind: "SOPS", Path: ".sops.yaml"}, + {Change: testChangeAdded, Check: checkSecurityTool, Kind: testSecurityScanKind, Name: testSemgrepName, Path: testSemgrepConfigPath}, + {Change: testChangeAdded, Check: checkBuild, Kind: "Make"}, + {Change: testChangeAdded, Check: checkSecret, Kind: "github-pat", Value: "[redacted]", Path: "config.yml", Line: 1}, + {Change: testChangeAdded, Check: checkREUSE, Kind: "declaration", Value: testMITLicenseID, Path: "licensed.txt"}, } for _, expected := range expectedAdditions { if !containsExactJSONLChange(added.Changes, expected) { @@ -1272,7 +1272,7 @@ func TestLogScansEveryDetectedLanguage(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove JavaScript") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -1318,7 +1318,7 @@ func TestLogTracksBriefLanguagesAcrossSymlinkTransitions(t *testing.T) { gitRun(t, repo, "commit", "-m", "Restore Ruby file") var stdout, stderr bytes.Buffer - if err := run([]string{"log", "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -1366,7 +1366,7 @@ func TestLogRebuildsLanguagesAtShallowBoundary(t *testing.T) { writeFile(t, repo, ".git/shallow", boundary+"\n") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testStatsFlag, "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testStatsFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -1397,7 +1397,7 @@ func TestLogRebuildsBriefLanguagesForManifestChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Add TypeScript") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testStatsFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testStatsFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -1429,7 +1429,7 @@ func TestLogWritesJSONL(t *testing.T) { gitRunAt(t, repo, "2024-02-01T12:00:00Z", "commit", "-m", "Add security policy") var stdout, stderr bytes.Buffer - if err := run([]string{"log", testJSONLFlag, "--workers", "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -1501,12 +1501,12 @@ func TestLogWritesBoundarySnapshots(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{ - "log", + testLogCommand, testJSONLFlag, - "--snapshots", - "--after", "2024-02-01", - "--before", "2024-03-01", - "--workers", "1", + testSnapshotsFlag, + testAfterFlag, testAfterDate, + testBeforeFlag, testBeforeDate, + testWorkersFlag, "1", repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) @@ -1528,7 +1528,7 @@ func TestLogWritesBoundarySnapshots(t *testing.T) { t.Fatalf("decode end snapshot: %v", err) } - if start.Type != "snapshot" || start.Boundary != "start" || start.Date != "2024-02-01" { + if start.Type != testSnapshotRecordType || start.Boundary != "start" || start.Date != testAfterDate { t.Errorf("start snapshot metadata = %#v", start) } if start.Author == nil || start.Author.Name != "Signals Test" || start.Author.Email != "signals@example.com" { @@ -1540,7 +1540,7 @@ func TestLogWritesBoundarySnapshots(t *testing.T) { if change.Type != "change" || change.Subject != "Remove policy after funding" { t.Errorf("change record = %#v", change) } - if end.Type != "snapshot" || end.Boundary != "end" || end.Date != "2024-03-01" { + if end.Type != testSnapshotRecordType || end.Boundary != "end" || end.Date != testBeforeDate { t.Errorf("end snapshot metadata = %#v", end) } if containsJSONLSignal(end.Signals, checkSecurityPolicy, "SECURITY.md") { @@ -1584,10 +1584,10 @@ func TestLogFiltersOutputByCommitterDateWithoutDiscardingEarlierState(t *testing var stdout, stderr bytes.Buffer if err := run([]string{ - "log", - "--after", "2024-02-01", - "--before", "2024-03-01", - "--workers", "1", + testLogCommand, + testAfterFlag, testAfterDate, + testBeforeFlag, testBeforeDate, + testWorkersFlag, "1", repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) @@ -1618,22 +1618,22 @@ func TestRunRejectsInvalidDates(t *testing.T) { }{ { name: "after format", - args: []string{"log", "--after", "2024/02/01"}, + args: []string{testLogCommand, testAfterFlag, "2024/02/01"}, want: `invalid --after date "2024/02/01": use YYYY-MM-DD`, }, { name: "before format", - args: []string{"log", "--before", "tomorrow"}, + args: []string{testLogCommand, testBeforeFlag, "tomorrow"}, want: `invalid --before date "tomorrow": use YYYY-MM-DD`, }, { name: "empty range", - args: []string{"log", "--after", "2024-02-01", "--before", "2024-02-01"}, + args: []string{testLogCommand, testAfterFlag, testAfterDate, testBeforeFlag, testAfterDate}, want: "after date must be earlier than before date", }, { name: "snapshots without range", - args: []string{"log", "--snapshots"}, + args: []string{testLogCommand, testSnapshotsFlag}, want: "snapshots require both --after and --before", }, } @@ -1649,7 +1649,7 @@ func TestRunRejectsInvalidDates(t *testing.T) { } func TestRunRejectsInvalidWorkers(t *testing.T) { - err := run([]string{"log", "--workers", "0"}, &bytes.Buffer{}, &bytes.Buffer{}) + err := run([]string{testLogCommand, testWorkersFlag, "0"}, &bytes.Buffer{}, &bytes.Buffer{}) if err == nil || err.Error() != "workers must be positive" { t.Fatalf("got %v", err) } @@ -1672,10 +1672,10 @@ func TestRunPropagatesWriterErrors(t *testing.T) { stderr io.Writer }{ {name: "version", args: []string{"version"}, stdout: failingWriter{}, stderr: io.Discard}, - {name: "text", args: []string{"log", repo}, stdout: failingWriter{}, stderr: io.Discard}, - {name: "JSONL", args: []string{"log", testJSONLFlag, repo}, stdout: failingWriter{}, stderr: io.Discard}, - {name: "progress", args: []string{"log", "--progress", "1", repo}, stdout: io.Discard, stderr: failingWriter{}}, - {name: "stats", args: []string{"log", testStatsFlag, repo}, stdout: io.Discard, stderr: failingWriter{}}, + {name: "text", args: []string{testLogCommand, repo}, stdout: failingWriter{}, stderr: io.Discard}, + {name: "JSONL", args: []string{testLogCommand, testJSONLFlag, repo}, stdout: failingWriter{}, stderr: io.Discard}, + {name: "progress", args: []string{testLogCommand, "--progress", "1", repo}, stdout: io.Discard, stderr: failingWriter{}}, + {name: "stats", args: []string{testLogCommand, testStatsFlag, repo}, stdout: io.Discard, stderr: failingWriter{}}, } for _, test := range tests { diff --git a/ownership_test.go b/ownership_test.go index c60298f..e4f779c 100644 --- a/ownership_test.go +++ b/ownership_test.go @@ -11,9 +11,8 @@ import ( ) const ( - testCodeownersPath = ".gitlab/CODEOWNERS" - testMaintainersPath = "MAINTAINERS.md" - testOwnershipLogCommand = "log" + testCodeownersPath = ".gitlab/CODEOWNERS" + testMaintainersPath = "MAINTAINERS.md" ) func TestRepositoryOwnershipSignals(t *testing.T) { @@ -142,7 +141,7 @@ func TestLogTracksRepositoryOwnershipChanges(t *testing.T) { gitRun(t, repo, "commit", "-m", "Remove ownership") var stdout, stderr bytes.Buffer - if err := run([]string{testOwnershipLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } var records []jsonlRecord diff --git a/packages.go b/packages.go index 4a82ec8..7d1b2e3 100644 --- a/packages.go +++ b/packages.go @@ -140,7 +140,7 @@ func packageFileSignals(path string, data []byte) []Signal { Check: checkPackageParse, Kind: string(match.Kind), Name: match.Ecosystem, - Value: "invalid", + Value: signalInvalid, Path: path, }) } diff --git a/packages_test.go b/packages_test.go index 24e3a16..790b6c4 100644 --- a/packages_test.go +++ b/packages_test.go @@ -67,7 +67,7 @@ func TestManifestLicenseExpressionSignalsClassifyDeclarations(t *testing.T) { {name: "strict", declaration: "MIT OR Apache-2.0", kind: licenseExpressionKind, expression: "MIT OR Apache-2.0", status: licenseExpressionStrict}, {name: "future identifier", declaration: "Future-License-1.0 OR MIT", kind: licenseExpressionKind, expression: "Future-License-1.0 OR MIT", status: licenseExpressionSyntax}, {name: "informal", declaration: "Apache 2", kind: licenseExpressionKind, expression: "Apache-2.0", status: licenseExpressionNormalized}, - {name: "invalid", declaration: "SEE LICENSE IN COPYING", kind: licenseExpressionInvalid, expression: "SEE LICENSE IN COPYING"}, + {name: signalInvalid, declaration: "SEE LICENSE IN COPYING", kind: licenseExpressionInvalid, expression: "SEE LICENSE IN COPYING"}, } for _, test := range tests { diff --git a/repository_files_test.go b/repository_files_test.go index 08a680a..d4531c9 100644 --- a/repository_files_test.go +++ b/repository_files_test.go @@ -52,7 +52,12 @@ func TestRepositoryFileSignalsDetectsSecurityDocuments(t *testing.T) { } const ( + testAfterFlag = "--after" + testAfterDate = "2024-02-01" testAttestationPath = "provenance.intoto.jsonl" + testBeforeFlag = "--before" + testBeforeDate = "2024-03-01" + testChangeAdded = "added" testLicensePath = "LICENSE" testMITLicenseID = "MIT" testFuzzPath = "fuzz_test.go" @@ -65,6 +70,8 @@ const ( testSARIFPath = "results.sarif" testSecurityPolicyPath = ".github/SECURITY.md" testShellNoExtensionPath = "install" + testSnapshotRecordType = "snapshot" + testSnapshotsFlag = "--snapshots" testStatsFlag = "--stats" testWorkersFlag = "--workers" ) @@ -404,7 +411,7 @@ func TestRepositoryFileSignalsSummarizesSARIF(t *testing.T) { if !containsExactSignal(invalid, Signal{ Check: checkSARIF, Kind: "candidate", - Name: "invalid", + Name: sarifInvalidName, Path: "invalid.sarif.json", }) { t.Fatalf("invalid SARIF signals = %#v", invalid) diff --git a/repository_refs_test.go b/repository_refs_test.go index 6047537..704def5 100644 --- a/repository_refs_test.go +++ b/repository_refs_test.go @@ -9,8 +9,6 @@ import ( "time" ) -const testRepositoryRefsLogCommand = "log" - func TestLogWritesSignedReleaseAndGittufEvents(t *testing.T) { repo := t.TempDir() gitRun(t, repo, "init", "-b", "main") @@ -42,7 +40,7 @@ func TestLogWritesSignedReleaseAndGittufEvents(t *testing.T) { gitRun(t, repo, "update-ref", gittufRSLRef, rslTip) var stdout, stderr bytes.Buffer - if err := run([]string{testRepositoryRefsLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testJSONLFlag, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } @@ -102,7 +100,7 @@ RSL Reference Entry } gitRun(t, repo, "update-ref", gittufRSLRef, strings.TrimSpace(string(output))) if err := run([]string{ - testRepositoryRefsLogCommand, "--ref", "v0.8.0", testJSONLFlag, testWorkersFlag, "1", repo, + testLogCommand, "--ref", "v0.8.0", testJSONLFlag, testWorkersFlag, "1", repo, }, &stdout, &stderr); err != nil { t.Fatalf("run log at annotated tag: %v\nstderr:\n%s", err, stderr.String()) } diff --git a/shell.go b/shell.go index 821417a..f86b08a 100644 --- a/shell.go +++ b/shell.go @@ -23,6 +23,11 @@ const ( shellOperationUpdate = "update" shellStateUnverified = "unverified" shellCommandCurl = "curl" + commandNameEnv = "env" + commandNamePython = "python" + commandNamePython3 = "python3" + commandNameSudo = "sudo" + shellDownloadDynamic = "dynamic" ) var ( @@ -32,7 +37,7 @@ var ( shellLanguageName = map[string]bool{"bash": true, "dash": true, "ksh": true, "sh": true, "zsh": true} shellInterpreterName = map[string]bool{ "node": true, "nodejs": true, - "perl": true, "php": true, "python": true, "python2": true, "python3": true, + "perl": true, "php": true, commandNamePython: true, "python2": true, commandNamePython3: true, "ruby": true, } ) @@ -307,12 +312,12 @@ func normalizeShellCommand(args []string) []string { switch name { case "command", "exec": args = args[1:] - case "env", "sudo": + case commandNameEnv, commandNameSudo: args = args[1:] for len(args) > 0 && (strings.HasPrefix(args[0], "-") || strings.Contains(args[0], "=")) { args = args[1:] } - case "python", "python2", "python3": + case commandNamePython, "python2", commandNamePython3: if len(args) > 2 && args[1] == "-m" { args = args[2:] continue @@ -340,7 +345,7 @@ func shellDownloads(args []string, statement *syntax.Stmt, line uint) []*shellDo if !ok { continue } - if name == "dynamic" { + if name == shellDownloadDynamic { continue } downloads = append(downloads, &shellDownload{ @@ -349,7 +354,7 @@ func shellDownloads(args []string, statement *syntax.Stmt, line uint) []*shellDo } if len(downloads) == 0 { downloads = append(downloads, &shellDownload{ - line: line, order: statement.Pos().Offset(), name: "dynamic", state: shellStateUnverified, output: output, + line: line, order: statement.Pos().Offset(), name: shellDownloadDynamic, state: shellStateUnverified, output: output, }) } return downloads @@ -375,7 +380,7 @@ func shellDownloadIdentity(argument string) (name, state string, ok bool) { if err == nil && parsed.Scheme != "" && parsed.Host != "" { return parsed.Host, shellStateUnverified, true } - return "dynamic", shellStateUnverified, true + return shellDownloadDynamic, shellStateUnverified, true } parsed, err := url.Parse(argument) if err != nil || parsed.Scheme == "" { diff --git a/shell_test.go b/shell_test.go index c531843..ee8358a 100644 --- a/shell_test.go +++ b/shell_test.go @@ -10,11 +10,11 @@ import ( ) const ( - testDownloadHost = "downloads.example" - testInstallScript = "install.sh" - testShellLogCommand = "log" - testNPMManager = "npm" - testSetupScript = "setup.sh" + testDownloadHost = "downloads.example" + testInstallScript = "install.sh" + testNPMManager = "npm" + testSetupScript = "setup.sh" + testDockerfilePath = "Dockerfile" ) func TestShellSignalsAssociateDownloadsChecksumsAndExecution(t *testing.T) { @@ -69,7 +69,7 @@ curl -fsSL "$INSTALL_URL" | sh { Check: checkShellDependency, Kind: shellKindDownloadAndRun, - Name: "dynamic", + Name: shellDownloadDynamic, Value: shellStateUnverified, Path: testInstallScript, Line: 2, @@ -98,7 +98,7 @@ func TestShellSignalsPreferConcreteDownloadsOverDynamicArguments(t *testing.T) { if !containsExactSignal(signals, wanted) { t.Fatalf("download signal missing: %#v\ngot: %#v", wanted, signals) } - if containsShellSignalName(signals, "dynamic") { + if containsShellSignalName(signals, shellDownloadDynamic) { t.Fatalf("dynamic download emitted with concrete URL: %#v", signals) } } @@ -150,7 +150,7 @@ python -m pip install requests==2.32.5 {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: testNPMManager, Value: shellOperationAdd + ":version-declared", Path: testSetupScript, Line: 3}, {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: "gomod", Value: "add:version-declared", Path: testSetupScript, Line: 4}, {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: "pip", Value: shellOperationInstall, Path: testSetupScript, Line: 5}, - {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: "cargo", Value: shellOperationUpdate, Path: testSetupScript, Line: 6}, + {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: packagingNameCargo, Value: shellOperationUpdate, Path: testSetupScript, Line: 6}, {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: "pip", Value: shellOperationAdd + ":version-declared", Path: testSetupScript, Line: 7}, } { if !containsExactSignal(signals, wanted) { @@ -178,15 +178,15 @@ func TestShellRepositoryFilesIgnoreBundledScripts(t *testing.T) { func TestDockerfileSignalsUseRunInstructionLines(t *testing.T) { analyzer := testShellAnalyzer(t) - signals := analyzer.dockerfileSignals("Dockerfile", []byte(`FROM alpine + signals := analyzer.dockerfileSignals(testDockerfilePath, []byte(`FROM alpine RUN curl -o /tmp/tool https://downloads.example/tool && \ echo '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef /tmp/tool' | sha256sum -c - RUN /tmp/tool `)) for _, wanted := range []Signal{ - {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: "checksum-verified", Path: "Dockerfile", Line: 2}, - {Check: checkShellDependency, Kind: shellKindChecksum, Name: "sha256sum", Value: "/tmp/tool", Path: "Dockerfile", Line: 3}, + {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: "checksum-verified", Path: testDockerfilePath, Line: 2}, + {Check: checkShellDependency, Kind: shellKindChecksum, Name: "sha256sum", Value: "/tmp/tool", Path: testDockerfilePath, Line: 3}, } { if !containsExactSignal(signals, wanted) { t.Errorf("signal missing: %#v\ngot: %#v", wanted, signals) @@ -216,7 +216,7 @@ INSTALL func TestDockerfileSignalsKeepContinuedRunsAcrossComments(t *testing.T) { analyzer := testShellAnalyzer(t) - signals := analyzer.dockerfileSignals("Dockerfile", []byte(`FROM alpine + signals := analyzer.dockerfileSignals(testDockerfilePath, []byte(`FROM alpine RUN apk add \ # retain the download in this instruction @@ -229,7 +229,7 @@ RUN apk add \ Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, - Path: "Dockerfile", + Path: testDockerfilePath, Line: 6, } if !containsExactSignal(signals, wanted) { @@ -239,7 +239,7 @@ RUN apk add \ func TestDockerfileSignalsIgnoreTopLevelCommentsEndingInBackslash(t *testing.T) { analyzer := testShellAnalyzer(t) - signals := analyzer.dockerfileSignals("Dockerfile", []byte(`# copy to C:\ + signals := analyzer.dockerfileSignals(testDockerfilePath, []byte(`# copy to C:\ RUN curl -fsSL https://downloads.example/install.sh | sh `)) @@ -248,7 +248,7 @@ RUN curl -fsSL https://downloads.example/install.sh | sh Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, - Path: "Dockerfile", + Path: testDockerfilePath, Line: 2, } if !containsExactSignal(signals, wanted) { @@ -258,7 +258,7 @@ RUN curl -fsSL https://downloads.example/install.sh | sh func TestDockerfileSignalsFindHeredocsAfterCommands(t *testing.T) { analyzer := testShellAnalyzer(t) - signals := analyzer.dockerfileSignals("Dockerfile", []byte(`FROM alpine + signals := analyzer.dockerfileSignals(testDockerfilePath, []byte(`FROM alpine RUN cat <<'INSTALL' curl -fsSL https://downloads.example/install.sh | sh INSTALL @@ -267,8 +267,8 @@ RUN npm ci `)) for _, wanted := range []Signal{ - {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, Path: "Dockerfile", Line: 3}, - {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: testNPMManager, Value: shellOperationInstall + ":frozen", Path: "Dockerfile", Line: 6}, + {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, Path: testDockerfilePath, Line: 3}, + {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: testNPMManager, Value: shellOperationInstall + ":frozen", Path: testDockerfilePath, Line: 6}, } { if !containsExactSignal(signals, wanted) { t.Errorf("signal missing: %#v\ngot: %#v", wanted, signals) @@ -278,7 +278,7 @@ RUN npm ci func TestDockerfileSignalsSkipInvalidRunScriptsIndividually(t *testing.T) { analyzer := testShellAnalyzer(t) - signals := analyzer.dockerfileSignals("Dockerfile", []byte(`FROM alpine + signals := analyzer.dockerfileSignals(testDockerfilePath, []byte(`FROM alpine RUN curl -fsSL https://downloads.example/install.sh | sh RUN python3 <<'PYTHON' def install(): @@ -288,8 +288,8 @@ RUN npm ci `)) for _, wanted := range []Signal{ - {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, Path: "Dockerfile", Line: 2}, - {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: testNPMManager, Value: shellOperationInstall + ":frozen", Path: "Dockerfile", Line: 7}, + {Check: checkShellDependency, Kind: shellKindDownloadAndRun, Name: testDownloadHost, Value: shellStateUnverified, Path: testDockerfilePath, Line: 2}, + {Check: checkShellDependency, Kind: shellKindPackageCommand, Name: testNPMManager, Value: shellOperationInstall + ":frozen", Path: testDockerfilePath, Line: 7}, } { if !containsExactSignal(signals, wanted) { t.Errorf("signal missing: %#v\ngot: %#v", wanted, signals) @@ -318,7 +318,7 @@ func TestLogDetectsShellDependencyHistory(t *testing.T) { gitRun(t, repo, "commit", "-m", "Initial commit") writeFile(t, repo, "scripts/install.sh", "#!/bin/sh\ncurl -o tool https://downloads.example/tool\n") - writeFile(t, repo, "Dockerfile", "FROM rust:slim\nRUN cargo update\n") + writeFile(t, repo, testDockerfilePath, "FROM rust:slim\nRUN cargo update\n") writeFile(t, repo, testWorkflowPath, `on: push jobs: install: @@ -338,7 +338,7 @@ sh tool gitRun(t, repo, "commit", "-m", "Verify downloaded tool") var stdout, stderr bytes.Buffer - if err := run([]string{testShellLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { + if err := run([]string{testLogCommand, testWorkersFlag, "1", repo}, &stdout, &stderr); err != nil { t.Fatalf("run log: %v\nstderr:\n%s", err, stderr.String()) } for _, expected := range []string{ diff --git a/subresource_integrity.go b/subresource_integrity.go index 96cb55c..6e86b10 100644 --- a/subresource_integrity.go +++ b/subresource_integrity.go @@ -95,7 +95,7 @@ func subresourceIntegritySignal(filePath string, token html.Token) (Signal, bool } parsed, err := integrity.ParseSRI(metadata) if err != nil { - return Signal{Check: checkSubresourceIntegrity, Kind: "invalid", Name: url, Path: filePath}, true + return Signal{Check: checkSubresourceIntegrity, Kind: signalInvalid, Name: url, Path: filePath}, true } return Signal{ Check: checkSubresourceIntegrity, diff --git a/subresource_integrity_test.go b/subresource_integrity_test.go index cb07aef..4c9477a 100644 --- a/subresource_integrity_test.go +++ b/subresource_integrity_test.go @@ -29,7 +29,7 @@ func TestSubresourceIntegritySignals(t *testing.T) { for _, wanted := range []Signal{ {Check: checkSubresourceIntegrity, Kind: "protected", Name: testCDNScript, Value: testSRI, Path: testMarkupPath}, {Check: checkSubresourceIntegrity, Kind: "missing", Name: "//cdn.example/missing.js", Path: testMarkupPath}, - {Check: checkSubresourceIntegrity, Kind: "invalid", Name: "https://cdn.example/invalid.js", Path: testMarkupPath}, + {Check: checkSubresourceIntegrity, Kind: signalInvalid, Name: "https://cdn.example/invalid.js", Path: testMarkupPath}, {Check: checkSubresourceIntegrity, Kind: "missing", Name: "https://cdn.example/app.css", Path: testMarkupPath}, } { if !containsExactSignal(signals, wanted) { diff --git a/workflow_observations.go b/workflow_observations.go index fd49be7..0810c0a 100644 --- a/workflow_observations.go +++ b/workflow_observations.go @@ -19,6 +19,7 @@ const ( workflowObservationJobLevel = "jobLevel" packagingKindPackage = "package" packagingKindPublish = "publish" + packagingNameCargo = "cargo" packagingNameNPM = "npm" packagingNameContainer = "container image" packagingNameGoReleaser = "GoReleaser" @@ -309,11 +310,11 @@ func packagingCommand(fields []string) (string, string) { return packagingVerb(command, args[1], "pack", "publish") } return packagingVerb(command, args[0], "pack", "publish") - case "cargo": - return packagingVerb("cargo", args[0], "package", "publish") + case packagingNameCargo: + return packagingVerb(packagingNameCargo, args[0], "package", "publish") case "gem": return packagingVerb("RubyGems", args[0], buildName, "push") - case "python", "python3", "py": + case commandNamePython, commandNamePython3, "py": return pythonPackagingCommand(args) case "twine": if args[0] == "upload" { @@ -371,7 +372,7 @@ func unwrapCommand(fields []string) []string { fields = fields[1:] continue } - if fields[0] == "sudo" || fields[0] == "env" { + if fields[0] == commandNameSudo || fields[0] == commandNameEnv { fields = fields[1:] continue } diff --git a/workflow_observations_test.go b/workflow_observations_test.go index 2b96346..21806d8 100644 --- a/workflow_observations_test.go +++ b/workflow_observations_test.go @@ -136,7 +136,7 @@ jobs: for _, wanted := range []Signal{ {Check: checkRunnerSelection, Kind: runnerKindSelfHosted, Name: "Private runner", Value: "Linux,X64,self-hosted", Path: testWorkflowPath}, {Check: checkRunnerSelection, Kind: runnerKindGroup, Name: "grouped", Value: "secure-runners", Path: testWorkflowPath}, - {Check: checkRunnerSelection, Kind: runnerKindDynamic, Name: "dynamic", Value: "expression:matrix.runner", Path: testWorkflowPath}, + {Check: checkRunnerSelection, Kind: runnerKindDynamic, Name: runnerKindDynamic, Value: "expression:matrix.runner", Path: testWorkflowPath}, } { if !hasWorkflowSignal(signals, wanted) { t.Errorf("signal missing: %#v\ngot: %#v", wanted, signals) @@ -152,7 +152,7 @@ func TestPackagingCommandDistinguishesBuildFromPackage(t *testing.T) { }{ {command: "cargo build"}, {command: "npm run build"}, - {command: "cargo package", kind: packagingKindPackage, name: "cargo"}, + {command: "cargo package", kind: packagingKindPackage, name: packagingNameCargo}, {command: "gem build example.gemspec", kind: packagingKindPackage, name: "RubyGems"}, {command: "mvn clean package", kind: packagingKindPackage, name: packagingNameMaven}, {command: "mvn -B clean deploy", kind: packagingKindPublish, name: packagingNameMaven}, diff --git a/workflow_provenance.go b/workflow_provenance.go index fb300e7..aa24238 100644 --- a/workflow_provenance.go +++ b/workflow_provenance.go @@ -151,7 +151,7 @@ func inlineEnvValue(fields []string, name string) (string, bool) { var result string var found bool for _, field := range fields { - if field == "env" || field == "sudo" { + if field == commandNameEnv || field == commandNameSudo { continue } key, value, ok := strings.Cut(field, "=") diff --git a/workflows.go b/workflows.go index 4a3c95d..a4cc189 100644 --- a/workflows.go +++ b/workflows.go @@ -22,6 +22,8 @@ const ( permissionKindReusableCall = "reusableCall" permissionKindReusableJob = "reusableJob" permissionKindReusableTop = "reusableTopLevel" + workflowExpressionOpening = "${{" + workflowExpressionClosing = "}}" ) var dangerousToJSONPattern = regexp.MustCompile(`(?i)tojson\s*\(\s*github(?:\.event)?\s*\)`) @@ -233,12 +235,12 @@ func shellWorkflowSignals( func sanitizeWorkflowExpressions(script string) string { data := []byte(script) for search := 0; search < len(data); { - relative := bytes.Index(data[search:], []byte("${{")) + relative := bytes.Index(data[search:], []byte(workflowExpressionOpening)) if relative < 0 { break } start := search + relative - end := workflowExpressionEnd(data, start+3) + end := workflowExpressionEnd(data, start+len(workflowExpressionOpening)) if end < 0 { break } @@ -276,7 +278,7 @@ func workflowExpressionEnd(data []byte, start int) int { continue } if current == '}' && index+1 < len(data) && data[index+1] == '}' { - return index + 2 + return index + len(workflowExpressionClosing) } } return -1 diff --git a/workflows_test.go b/workflows_test.go index acca68b..c12e319 100644 --- a/workflows_test.go +++ b/workflows_test.go @@ -372,7 +372,7 @@ jobs: { Check: checkShellDependency, Kind: shellKindPackageCommand, - Name: "cargo", + Name: packagingNameCargo, Value: shellOperationUpdate, Path: testWorkflowPath, Line: 13, @@ -448,7 +448,7 @@ jobs: if !containsExactSignal(signals, wanted) { t.Fatalf("workflow download signal missing: %#v\ngot: %#v", wanted, signals) } - if containsShellSignalName(signals, "dynamic") { + if containsShellSignalName(signals, shellDownloadDynamic) { t.Fatalf("dynamic workflow download emitted with concrete URL: %#v", signals) } }