diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5fbe03a..bb3a01c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -76,13 +76,39 @@ npm run test:github Do not start with the evaluator core (`src/evaluator-core.ts`, `src/contract/`, `schemas/`) or the GitHub adapter privileged lane (`src/github/`, Action privileged workflows). Those paths are the trust boundary. -Current contribution issues, if they are still open: +The safest contribution ladder is: + +```text +run the Case Lab + -> report a confusing result + -> add a redacted governance scenario + -> improve remediation or documentation + -> contribute an adapter fixture + -> propose a trust-boundary change with maintainer review +``` + +Good-first work should normally be one of these focused packets: + +- **Scenario and fixture:** add one real governance edge case and its expected + receipt to [the Case Lab](../docs/case-lab.md). +- **First-use documentation:** improve a command, error explanation, or + captured output without changing the evaluator contract. +- **Consumer workflow:** improve the clean consumer-repository fixture or its + documentation while preserving the no-PR-checkout privileged lane. +- **Adapter evidence:** extend a recorded GitHub fixture only when the source, + SHA binding, permissions, and redaction rules are documented. + +Every packet must state the user problem, allowed files, expected output, and +the exact verification command. Start a discussion or issue before any change +that would alter a rule, schema, trust boundary, or Action permission. -- [Issue #5 — beta release and rollback documentation](https://github.com/daichunghy/patchgate/issues/5) -- [Issue #6 — CODEOWNERS subset conformance fixtures](https://github.com/daichunghy/patchgate/issues/6) -- [Issue #7 — clean consumer-repository Action fixture](https://github.com/daichunghy/patchgate/issues/7) +For currently available work, search the live issue list for `good first issue` +or `help wanted` rather than relying on a numbered list that can become stale. +If no suitable issue exists, open one with the same packet fields above. -Issue #6 is fixture coverage for the **documented** CODEOWNERS subset. Do not start matching `?` or other undocumented syntax without fixtures and an explicit contract change. +Fixture coverage must remain within the **documented** CODEOWNERS subset. Do +not start matching `?` or other undocumented syntax without fixtures and an +explicit contract change. ## 7. License diff --git a/README.md b/README.md index 7e20455..32c6567 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,10 @@ snapshot` and `support-bundle` write files with `--output` only. Giving `evaluate` both flags with different paths exits 2 (`REPORT_OUTPUT_CONFLICT`). `--fail-on` defaults to `blocked`, matching the Action. -Longer walkthrough: [Getting started](docs/getting-started.md). +Longer walkthrough: [Getting started](docs/getting-started.md). The +[Case Lab](docs/case-lab.md) contains ready, blocked, missing-evidence, +human-gate, and policy-ambiguity scenarios that can be replayed from the +fixture manifest with `npm run case-lab`. ## GitHub Action candidate diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 21008ef..4f2d569 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Feedback-driven ownership and SHA reporting — 2026-09-03 +- Local preflight now surfaces root Prow `OWNERS` and `OWNERS_ALIASES` files as + `needs_confirmation` discovery findings instead of silently presenting a + repository with no supported ownership source. +- Action summaries and logs now show base, head, tested, and target-kind values + separately, with an explicit warning when the tested SHA differs from the PR + head. +- Locked `fast-uri` to `3.1.7` so the high-severity audit finding reported on + Dependabot PRs #70–#72 is removed; PR #73 carries the same dependency fix. + +### First-use Case Lab and contribution paths — 2026-09-01 +- Added a three-minute README path that surfaces the first useful local result + before the longer architecture and release notes. +- Added a replayable [Case Lab](case-lab.md) and `npm run case-lab` alias for + the 53-entry fixture compatibility manifest. +- Added a contributor ladder with focused scenario, documentation, consumer, + and adapter-evidence work packets outside the trust boundary. +- Added first-use labels and an adoption scorecard that separates external + receipts and repeat use from stars, downloads, releases, and bot activity. ### First-use probe and live-smoke entrypoint — 2026-08-27 - Added a read-only first-use probe that distinguishes a valid trusted local policy from a missing `patchgate.yml` without treating discovery guidance as diff --git a/docs/adoption-loop.md b/docs/adoption-loop.md index e61a28d..78d929e 100644 --- a/docs/adoption-loop.md +++ b/docs/adoption-loop.md @@ -1,9 +1,23 @@ -# User adoption loop — 2026-08-28 +# User adoption loop — 2026-09-01 The goal for this workspace is not to produce four busy repositories. It is to get real people to a first useful result, learn where the workflow fails, and fix that failure without weakening the product boundaries. +For PatchGate, the first-use journey is: + +```text +install or clone + -> run preflight or Case Lab + -> receive one understandable result + -> apply the remediation + -> run again on another pull request +``` + +The Case Lab is a local activation aid. A real adoption signal starts only when +an outside maintainer reaches a receipt in their own repository and gives +consented feedback. + ## What counts as progress For each repository, the strongest next signal is one consented outside user @@ -56,6 +70,24 @@ entry point to a verifiable first result, followed by a consented outside workflow. If activation is successful but feedback conversion is weak, improve the result and the form before adding features. +## Metrics that guide the next change + +Track these as a small weekly scorecard. Stars, releases, self-authored issues, +CI runs, and npm downloads remain discoverability or maintenance signals; they +are not adoption proof. + +| Metric | Definition | Decision it informs | +| --- | --- | --- | +| First receipt | An outside repository produces its first PatchGate receipt | Whether the onboarding path is understandable | +| Time to first receipt | Elapsed time from the first documented command to the first usable result | Which setup step deserves removal or better output | +| Remediation completion | The user fixes the reported issue and reaches the intended next state | Whether the result is actionable rather than merely diagnostic | +| Repeat use | The same outside repository evaluates a second pull request | Whether PatchGate solves a recurring workflow problem | +| First response time | Time until a human maintainer responds to an outside issue or PR | Whether the community feels attended to | +| Scenario conversion | Outside feedback that becomes a redacted fixture or documentation change | Whether community input improves the product | + +Do not report any metric as public adoption until its repository, user context, +consent, and evidence class are recorded. + ## Working cadence 1. Check live GitHub and package signals before writing the weekly status. diff --git a/docs/architecture.md b/docs/architecture.md index 9fef57a..d76085b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,6 +115,13 @@ authentication proof: a caller that fabricates a complete local snapshot can fabricate both claims. Authenticated GitHub retrieval remains a G3 adapter responsibility. +Some repositories use Prow `OWNERS` files instead of `CODEOWNERS`. PatchGate +surfaces root `OWNERS` and `OWNERS_ALIASES` files during local discovery so a +maintainer can see that the ownership model is outside the supported parser. +They remain discovery-only: Prow labels and `/lgtm` or `/approve` state do not +become approval evidence until a versioned adapter contract can authenticate +and verify them. + ## Evidence strength An evidence item must carry: diff --git a/docs/case-lab.md b/docs/case-lab.md new file mode 100644 index 0000000..3d3ff75 --- /dev/null +++ b/docs/case-lab.md @@ -0,0 +1,78 @@ +# PatchGate Case Lab + +The Case Lab is a small, replayable set of governance situations. It shows +what PatchGate can decide today without requiring GitHub credentials, a live +pull request, or execution of contributor-controlled code. + +## Run the lab + +From a fresh clone on Node.js 20 or later: + +```bash +npm ci +npm run case-lab +``` + +`case-lab` runs the same 53-entry fixture manifest used by the repository's +compatibility tests. Every entry has a named scenario, an expected status or +contract diagnostic, and an oracle assertion. A green run proves deterministic +fixture compatibility; it does not prove live GitHub integration or external +adoption. + +To see a complete receipt directly: + +```bash +npm run build +node dist/src/cli.js evaluate --event fixtures/pr-ready.json +``` + +The command above is the smallest successful path. It produces +`ready_for_review` from a normalized snapshot and does not contact GitHub. + +## Core scenarios + +| Scenario | Fixture manifest entry | Expected result | What a maintainer learns | +| --- | --- | --- | --- | +| Ready contribution | `valid-ready` | `ready_for_review` | The required check, issue linkage, policy revision, and reviewability evidence are present. | +| Missing issue linkage | `complete-zero-linked-issues` | `blocked` | A valid check cannot compensate for a required linked issue that is absent. | +| Incomplete check observation | `incomplete-checks-with-success` | `evidence_missing` | A green-looking check is not enough when the check collection is incomplete. | +| Human gate not satisfied | `same-actor-duplicate-approval` | `human_review_required` | Duplicate approval by one actor does not satisfy a two-person sensitive-path gate. | +| Policy digest mismatch | `policy-object-digest-mismatch` | `policy_ambiguous` | PatchGate refuses to treat a policy object as trusted when its digest does not match. | +| Unsupported input | `unsupported-input-version` | contract rejection | An unsupported schema version fails closed instead of being guessed. | + +The JSON files under `fixtures/**` are scenario descriptors. They are not raw +evaluation snapshots to be passed directly to `evaluate`; the fixture test +harness derives each normalized input from a trusted base fixture and checks +the expected oracle in [fixtures/manifest.json](../fixtures/manifest.json). + +## Add a scenario + +Add a scenario only when it represents a real governance edge case or a +documented product boundary: + +1. Add a descriptor with a unique `scenario` value under the appropriate + `fixtures/` directory. +2. Add the derived input and expected outcome to + [test/fixture.test.ts](../test/fixture.test.ts). +3. Add a manifest entry with the status, reason IDs, requirement results, or + contract diagnostic that must remain stable. +4. Run `npm run case-lab` and `npm run verify`. +5. Explain the user-facing remediation in the pull request. + +Scenario contributions are intentionally safer than changes to the evaluator +or privileged GitHub adapter. Do not turn a fixture or prose suggestion into a +new blocking rule without an explicit contract change and maintainer review. + +## What to record from a real first use + +When a person runs the Case Lab or the Action in a repository outside this +project, record only the minimum redacted evidence: + +- version or immutable commit; +- time to the first useful result; +- the first confusing or failed step; +- whether the receipt changed what the person reviewed; +- consent to publish a redacted summary. + +Those observations belong in the first-use feedback path. A local Case Lab run +is a product demonstration, not a pilot or an adoption claim. diff --git a/docs/first-use.md b/docs/first-use.md index c33dff1..3595e88 100644 --- a/docs/first-use.md +++ b/docs/first-use.md @@ -4,6 +4,13 @@ The first useful result is a non-blocking check on a real pull request. It should tell a maintainer what evidence is present and what still needs a human decision without changing merge eligibility. +## See the result before connecting GitHub + +The [Case Lab](case-lab.md) is the fastest way to understand the product +contract. Run `npm ci && npm run case-lab` to replay ready, blocked, +missing-evidence, human-gate, and policy-ambiguity cases locally. This is a +demonstration and compatibility check, not external adoption evidence. + ## Shadow setup For a local policy-only first result from this repository, run: @@ -54,6 +61,16 @@ Use the [first-use feedback form](https://github.com/daichunghy/patchgate/issues for a redacted report. Do not include tokens, private repository data, or unredacted pull-request contents. +For comparable results, use these exact labels: + +- `worked` — the first result was reached without a workaround; +- `worked_with_workaround` — a result was reached after an unexpected manual + step; +- `did_not_reach_first_result` — setup or permissions prevented a result. + +The most useful signal is not a star or a download. It is an outside maintainer +reaching a first receipt, understanding the remediation, and choosing whether +to run PatchGate again on another pull request. If the first-use question is worth observing on a real public repository, use the [shadow pilot interest form](https://github.com/daichunghy/patchgate/issues/new?template=pilot-interest.yml) and read the [pilot intake guide](community/pilot-intake.md) before changing diff --git a/docs/getting-started.md b/docs/getting-started.md index c5f1e36..b123084 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,6 +11,10 @@ This walkthrough uses a clone and a local build. Do not run `npx patchgate`: that npm name is a different project. The direct GitHub install is available for the beta release, while the CLI remains an unpublished npm package. +If you only want to see the decision contract first, run the +[Case Lab](case-lab.md) with `npm ci && npm run case-lab`. It replays local +fixtures and does not contact GitHub. + ## 1. Clone and build Requires Node.js 20 or later. diff --git a/docs/github-adapter-contract.md b/docs/github-adapter-contract.md index 06d0b5a..8553902 100644 --- a/docs/github-adapter-contract.md +++ b/docs/github-adapter-contract.md @@ -50,6 +50,12 @@ GitHub control was bypassed. - Policy and CODEOWNERS are read at the PR base SHA, never from the proposed head. +- A root `OWNERS` or `OWNERS_ALIASES` file is surfaced by local discovery as + `needs_confirmation`, not parsed as enforceable ownership. Prow repositories + may keep additional `OWNERS` files in subdirectories and use `/lgtm`, + `/approve`, or labels; this version does not treat those signals as + qualified GitHub approval evidence. Confirm the intended boundary in + `patchgate.yml` or a supported native GitHub control. - Head target evidence is bound to the PR head SHA. Merge target evidence is bound to the immutable merge SHA returned by GitHub. - Initial identity and decision-bearing observations are re-read during diff --git a/docs/reviews/2026-09-03-feedback-improvements.md b/docs/reviews/2026-09-03-feedback-improvements.md new file mode 100644 index 0000000..415bc43 --- /dev/null +++ b/docs/reviews/2026-09-03-feedback-improvements.md @@ -0,0 +1,42 @@ +# Feedback-driven improvement record — 2026-09-03 + +This record connects the current implementation changes to feedback that was +checked in email, GitHub discussions, and open pull-request checks. It does not +claim an external pilot or an independent review of PatchGate. + +## Evidence reviewed + +| Source | Observed feedback | Product implication | +| --- | --- | --- | +| Email thread “Question on security-tooling ownership evidence” | Falco maintainers resolve ownership from Prow `OWNERS` files per changed path; `/lgtm` and `/approve` are separate gates. Their repositories do not use `CODEOWNERS`, and cross-repository work is handled as separate PRs. | A CODEOWNERS-only parser can miss a real ownership model. Detect the alternate file as discovery-only and do not guess at Prow approval semantics. | +| [PatchGate Discussion #29](https://github.com/daichunghy/patchgate/discussions/29) | The response recommends showing the tested SHA in the check output and only posting success when the workflow SHA matches the intended PR head. | Make `headSha`, `testedSha`, and `targetKind` visible together in the Action summary and log. | +| [PR #69](https://github.com/daichunghy/patchgate/pull/69) and its [Full Verify run](https://github.com/daichunghy/patchgate/actions/runs/33526195564) | The documentation restructure passed the platform and test jobs but failed the consumer-documentation assertion for snapshot-rejection Check Run wording. | Keep the failure as a release-surface issue to resolve on that PR; do not call the PR green from its partial checks. | +| [PRs #70–#73](https://github.com/daichunghy/patchgate/pulls) | #70–#72 failed the high-severity audit because `fast-uri@3.1.5` was installed; #73 updates it to `3.1.7` and its current checks are green. | Apply the lockfile security update locally and retain the Dependabot PR as the public merge path. | + +No human review or inline review comment was present on the open PatchGate PRs +checked on this date. Dependabot notifications and CI results are maintenance +signals, not independent approval evidence. + +## Decisions implemented + +1. Add root `OWNERS` and `OWNERS_ALIASES` to discovery. A present file is + classified `needs_confirmation` with `authority: discovery_only` and the + `prow_owners` signal. It cannot create a blocking requirement. +2. Keep the enforceable ownership contract unchanged: `CODEOWNERS`, explicit + `patchgate.yml`, and authenticated native GitHub controls remain the only + supported ownership sources for this version. +3. Show `Base SHA`, `Head SHA`, `Tested SHA`, and `Evidence Target` in the Action + summary and console output. When the tested and head SHAs differ, the + summary states that the result is bound to the declared target. +4. Apply the exact `fast-uri@3.1.7` lockfile update already proposed by PR #73. + +## Verification target + +The local change must pass: + +```bash +npm run verify +``` + +The fixture for Prow discovery is synthetic and proves only classification and +non-enforcement. It is not a recording of Falco repository data. diff --git a/fixtures/repositories/prow-owners/OWNERS b/fixtures/repositories/prow-owners/OWNERS new file mode 100644 index 0000000..412fdf5 --- /dev/null +++ b/fixtures/repositories/prow-owners/OWNERS @@ -0,0 +1,6 @@ +approvers: +- security-team +reviewers: +- maintainer +labels: +- lgtm diff --git a/fixtures/repositories/prow-owners/patchgate.yml b/fixtures/repositories/prow-owners/patchgate.yml new file mode 100644 index 0000000..b825518 --- /dev/null +++ b/fixtures/repositories/prow-owners/patchgate.yml @@ -0,0 +1 @@ +version: 1 diff --git a/package.json b/package.json index 3520448..d730bd0 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "prepublishOnly": "npm run verify", "build": "tsc -p tsconfig.json", "first-use": "npm run build && node dist/src/cli.js preflight --repo . --base main", + "case-lab": "npm run test:fixtures", "bundle:action": "ncc build src/action/index.ts -o dist/action --minify --transpile-only", "audit": "npm audit --audit-level=high", "typecheck": "tsc -p tsconfig.json --noEmit", diff --git a/src/action/index.ts b/src/action/index.ts index 2218d94..9cafc32 100644 --- a/src/action/index.ts +++ b/src/action/index.ts @@ -304,6 +304,10 @@ export async function runAction(env: NodeJS.ProcessEnv = process.env): Promise { const unsupportedFixture = runCommand(["preflight", "--base", resolve("fixtures/repositories/unsupported-guidance/patchgate.yml"), "--json"]); expect(unsupportedFixture.exit).toBe(0); expect(JSON.parse(unsupportedFixture.stdout).guidance).toEqual(expect.arrayContaining([expect.objectContaining({ diagnosticId: "DISCOVERY_UNSUPPORTED", classification: "unsupported" })])); + const prowOwnersFixture = runCommand(["preflight", "--base", resolve("fixtures/repositories/prow-owners/patchgate.yml"), "--json"]); + expect(prowOwnersFixture.exit).toBe(0); + expect(JSON.parse(prowOwnersFixture.stdout).guidance).toEqual(expect.arrayContaining([expect.objectContaining({ path: "OWNERS", classification: "needs_confirmation", diagnosticId: "DISCOVERY_NEEDS_CONFIRMATION", signals: ["prow_owners", "ownership"] })])); const doctor = runCommand(["doctor", "--base", resolve("fixtures/repositories/missing-policy"), "--json"]); expect(doctor.exit).toBe(1); expect(JSON.parse(doctor.stdout)).toMatchObject({ status: "attention", mode: "local" });