diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 985b8ba8f031d..907aa8fad9573 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -80,6 +80,7 @@ fork-ci/actions/gradle-cache/gradle_cache.test.py fork-ci/actions/handoff-put/handoff.test.py ~16s fork-ci/actions/head-check/head_check.test.py ~69s fork-ci/actions/head-check/head_gate.test.py +fork-ci/actions/head-check/review_guide.test.py fork-ci/actions/job-changed/job-changed.test.py ~21s fork-ci/actions/pinned/pinned.test.py fork-ci/actions/pr-queue/lens.test.py diff --git a/.github/scripts/fork-contracts.test.py b/.github/scripts/fork-contracts.test.py index 74bfc380a7f08..03d3b8b67d65c 100644 --- a/.github/scripts/fork-contracts.test.py +++ b/.github/scripts/fork-contracts.test.py @@ -4171,6 +4171,10 @@ def result_step(jobs, name): # Needs a checkout that is not a merge commit; CI's pull request checkout is one. "fork-ci/actions/head-check/head_check.test.py": [ "SameChangeBare.test_a_checkout_that_is_not_a_merge_commit_is_refused"], + # Need the ci/vN tags; CI's checkout fetches none (only its Releases class runs there). + "fork-ci/actions/head-check/review_guide.test.py": ["ReleaseTags." + t for t in ( + "test_override_authors_is_first_in_the_release_after_the_one_the_notes_name", + "test_the_path_rules_are_first_in_the_release_the_guide_names")], } check("records: the tests a part run may skip are exactly the pinned ones, each by name (MAY_SKIP)", {f: sorted(n) if isinstance(n, list) else n for f, n in DECLARED_SKIPS.items()} == MAY_SKIP, diff --git a/fork-ci/README.md b/fork-ci/README.md index 795745599f771..26137f9531655 100644 --- a/fork-ci/README.md +++ b/fork-ci/README.md @@ -80,6 +80,13 @@ Two habits every action expects: To move to a new release, change the SHA and the comment together; read the new tag's release notes first ([RELEASE-NOTES.md](RELEASE-NOTES.md): what changed, and what to do before or after the move). +### Review rules + +The queue and the head gate can also judge which lenses a change needs, its findings and its review rounds, from a +`review` block in the project's queue config. Turn it on in this order: release, pin, then config +([pr-queue README](actions/pr-queue/README.md#turning-it-on-release-pin-then-config)), in `shadow` first: a release +that does not know the key refuses the whole config. + ### Outside a workflow A project's tools that run outside a workflow (a deploy script, a local check) use the same code from the same diff --git a/fork-ci/RELEASE-NOTES.md b/fork-ci/RELEASE-NOTES.md index 85d5465757d8f..f3ec50d85c707 100644 --- a/fork-ci/RELEASE-NOTES.md +++ b/fork-ci/RELEASE-NOTES.md @@ -13,6 +13,22 @@ nothing. ## After ci/v14 +### pr-queue and head-check: review findings, rounds and `override_authors` (#740) + +With a `review` block, the gate also judges the review itself from the lens comments. With `findings: true`, a lens's +newest report needs a findings section (`blocker:` and `major:` lines, or `findings: none`), and a blocker or major in +it fails the head. A round is a lens read of a different own change (a queue update adds none): from +`rounds.approach_at` the approach lens is required, and past `rounds.max` a person decides. The new optional key +`review.override_authors` lists the logins whose override comment counts as a person's (empty or left out: none +does). In `shadow` the check summary only adds a row "findings and rounds (shadow): they would have said: ...". Without +a `review` block nothing changes. First in the release after `ci/v20`. + +**Upgrade:** `ci/v20` and older refuse a config with `override_authors`: move every pin first, then add the key +(pr-queue README, "Turning it on: release, pin, then config"). A config already in `enforce` with `findings: true`: +after the move, a required lens whose newest report has no findings section fails the head, so have the lens prompts +write `blocker: none` and `major: none` first, or switch to `shadow` before the move. In both modes a head whose +rounds cannot be judged (an error while reading the lens comments) now fails instead of passing on the old rule. + ### handoff-put and handoff-get: a file handed from one job to a later job of the same run (#247) - New actions [`handoff-put`](actions/handoff-put/README.md) and [`handoff-get`](actions/handoff-get/README.md). put diff --git a/fork-ci/actions/head-check/gate-caller.example.yml b/fork-ci/actions/head-check/gate-caller.example.yml index cbe3815af808c..c1da2093bfeb2 100644 --- a/fork-ci/actions/head-check/gate-caller.example.yml +++ b/fork-ci/actions/head-check/gate-caller.example.yml @@ -3,6 +3,10 @@ # the pull request to the gate as a number, nothing more. Reviews and inline review comments start it through the # review trigger (review-trigger.example.yml, copied next to this file; README.md, "Run again after a review"): on its # finished run the gate looks the pull request up by that run's head. +# In a public repository GitHub blocks pull_request_target (from 2026-12-02) unless a repository Actions policy allows +# it for exactly this file (README.md, "The gate", setup step 4): keep the file name the policy names. +# Review rules (a `review` block in the config): release, pin, then config, as in ../pr-queue/README.md, "Turning it on: +# release, pin, then config". Pin the release first: an older one refuses the whole config. name: Head gate on: @@ -50,5 +54,9 @@ jobs: pr: ${{ format('{0}', github.event.pull_request.number || github.event.issue.number || inputs.pr) }} config: .github/pr-queue.json check-name: CHECK_NAME + # In shadow (or with no review block) these names decide. In enforce the config's path rules name the lenses and + # these are required as well; empty this only after the base's config is in enforce. Empty, it is no floor: a + # change only zero-lens rules match (`**/*.md` in review.recommended.json, AGENTS.md too) merges with no lens + # (the guide, step 5). required-lenses: | # one lens name per line; none = the gate refuses, unless the config's review block is in enforce LENS_NAME diff --git a/fork-ci/actions/head-check/review_guide.test.py b/fork-ci/actions/head-check/review_guide.test.py new file mode 100644 index 0000000000000..8b193f12ef6ff --- /dev/null +++ b/fork-ci/actions/head-check/review_guide.test.py @@ -0,0 +1,527 @@ +"""Tests the review rules' turn-on guide against the code it describes (run: python3 fork-ci/actions/head-check/review_guide.test.py). + +The guide is the pr-queue README's section "Turning it on: release, pin, then config". The fork-ci README and the +example gate caller point to it. Here: the links resolve to its heading, its steps keep their order, the summary rows +it quotes are the rows the gate writes, the releases it names agree with RELEASE-NOTES.md and (where this clone has +the ci/vN tags) with the tags, and each "stopped half-way" state it promises is what head_check and head_gate do: a +config without the block is the gate of before, a key a release does not know refuses the whole config (a gate on a +release that knows the block refuses the pull request that adds it), an empty required-lenses input fails unless the +base is in enforce and is no floor once it may be empty, the rows never fail a head in shadow, what cannot be judged +fails it in both modes, and a pin back is checked by nothing but review. Each such test also reads the guide's own +words for that state, so a guide that promises something else goes red. + +No network, no GitHub: the gate's reads are replaced by fakes (git only for a throwaway repository, and for the tags); +the rules and the row text are the real code. +""" +import json, os, re, subprocess, sys, tempfile, unittest +from unittest import mock + +sys.dont_write_bytecode = True # no __pycache__ next to the scripts +HERE = os.path.dirname(os.path.abspath(__file__)) +FORK_CI = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, HERE) +import head_check as H # noqa: E402 (puts ../pr-queue on the path) +import head_gate as G # noqa: E402 +import config # noqa: E402 +import review_config # noqa: E402 +import review_judge # noqa: E402 + +QUEUE = os.path.join(FORK_CI, 'actions', 'pr-queue') +GUIDE_README = os.path.join(QUEUE, 'README.md') +FORK_CI_README = os.path.join(FORK_CI, 'README.md') +NOTES = os.path.join(FORK_CI, 'RELEASE-NOTES.md') +EXAMPLE = os.path.join(HERE, 'gate-caller.example.yml') +HEADING = 'Turning it on: release, pin, then config' +BASE = { + 'required_checks': ['build', 'test'], 'merge_method': 'squash', 'bot_logins': ['review-bot'], + 'lens_author_associations': ['OWNER'], 'override': {'prefix': 'tool ready --head', 'author_association': 'OWNER'}, + 'ready_command': 'tool ready {pr}', 'push_command': 'tool push', 'lens_format_doc': 'docs/review.md', + 'head_check': 'test', +} + + +def read(path): + with open(path, encoding='utf-8') as f: + return f.read() + + +def slug(heading): + """GitHub's anchor of a Markdown heading: lower case, punctuation dropped, spaces to dashes.""" + return re.sub(r'[^\w\- ]', '', heading.strip().lower()).replace(' ', '-') + + +def guide(): + """The guide's text: from its heading to the next heading of the same or a higher level.""" + text = read(GUIDE_README) + m = re.search(r'^### ' + re.escape(HEADING) + r'\n(.*?)(?=^#{2,3} )', text, re.S | re.M) + if not m: + raise AssertionError(f'no "### {HEADING}" section in {GUIDE_README}') + return m.group(1) + + +def steps(text): + """The guide's numbered steps: [(number, bold title)].""" + return [(int(n), t) for n, t in re.findall(r'^(\d+)\. \*\*([^*]+)\*\*', text, re.M)] + + +def flat(text): + return ' '.join(text.split()) + + +def step(n): + """Step n of the guide, whitespace folded.""" + text = guide() + start = re.search(rf'^{n}\. \*\*', text, re.M) + end = re.search(rf'^({n + 1}\. \*\*|\*\*Stopped half-way)', text, re.M) + return flat(text[start.start():end.start()]) + + +def halfway(opening): + """The guide's "Stopped half-way" bullet that starts with `opening`, whitespace folded.""" + part = guide().split('**Stopped half-way, or run again.**')[1].split('**The owner step.**')[0] + found = [flat(b) for b in re.split(r'^- ', part, flags=re.M)[1:] if flat(b).startswith(opening)] + if len(found) != 1: + raise AssertionError(f'one "Stopped half-way" bullet starting {opening!r}, found {len(found)}') + return found[0] + + +def owner_step(): + return flat(guide().split('**The owner step.**')[1]) + + +def recommended(): + return json.loads(read(os.path.join(QUEUE, 'review.recommended.json'))) + + +def parsed(mode): + return config.parse(json.dumps(dict(BASE, review=dict(recommended(), mode=mode)))) + + +class Links(unittest.TestCase): + def test_the_guide_has_its_heading(self): + self.assertIn(f'\n### {HEADING}\n', read(GUIDE_README)) + + def test_the_fork_ci_readme_links_to_the_heading(self): + want = 'actions/pr-queue/README.md#' + slug(HEADING) + self.assertEqual(slug(HEADING), 'turning-it-on-release-pin-then-config') + self.assertIn(f']({want})', read(FORK_CI_README)) + + def test_the_example_caller_names_the_heading_and_the_order(self): + header = ' '.join(x.lstrip('# ').strip() for x in read(EXAMPLE).splitlines() if x.startswith('#')) + self.assertIn(f'../pr-queue/README.md, "{HEADING}"', header) + self.assertIn('Pin the release first: an older one refuses the whole config', header) + + def test_the_example_caller_names_the_pull_request_target_policy(self): + header = ' '.join(x.lstrip('# ').strip() for x in read(EXAMPLE).splitlines() if x.startswith('#')) + self.assertIn('repository Actions policy allows it for exactly this file', header) + self.assertIn('keep the file name the policy names', header) + + def test_the_example_caller_says_what_required_lenses_does_in_each_mode(self): + lines = read(EXAMPLE).splitlines() + at = next(i for i, x in enumerate(lines) if x.strip().startswith('required-lenses:')) + said = ' '.join(x.strip().lstrip('# ') for x in lines[at - 4:at]) + self.assertIn('In shadow (or with no review block) these names decide', said) + self.assertIn("In enforce the config's path rules name the lenses and these are required as well", said) + self.assertIn("empty this only after the base's config is in enforce", said) + # The floor the empty input drops (EmptyInputFloor checks that it is so). + self.assertIn('Empty, it is no floor', said) + self.assertIn('AGENTS.md too) merges with no lens (the guide, step 5)', said) + + def test_every_relative_link_of_the_guide_resolves(self): + text = guide() + links = re.findall(r'\]\(([^)#\s]+)(#[^)\s]+)?\)', text) + self.assertTrue(links) + for path, anchor in links: + if path.startswith('http'): + continue + target = os.path.normpath(os.path.join(QUEUE, path)) + self.assertTrue(os.path.exists(target), path) + if anchor: + heads = [slug(h) for h in re.findall(r'^#+ (.+)$', read(target), re.M)] + self.assertIn(anchor[1:], heads, f'{path}{anchor}') + + +class Order(unittest.TestCase): + def test_release_pin_config_read_enforce_roll_back(self): + self.assertEqual(steps(guide()), [(1, 'Release.'), (2, 'Pin.'), (3, 'Config in `shadow`.'), + (4, 'Read the results.'), (5, 'Enforce.'), (6, 'Roll back')]) + + def test_the_pin_step_moves_every_caller(self): + pin = guide().split('2. **Pin.**')[1].split('3. **')[0] + for caller in ('`fork-pr-queue.yml`', '`pr-queue` action', 'gate-caller.example.yml', '`head-check`'): + self.assertIn(caller, pin) + self.assertIn('dispatch the gate', pin.replace('\n', ' ').replace("gate's", 'gate')) + + def test_the_pin_step_moves_the_pin_config_of_tools_outside_a_workflow_too(self): + """A tool outside a workflow runs the queue's config.py from its own pin config (actions/pinned): on an older + release it refuses the new key like an old caller does (StoppedHalfWay, a key a release does not know).""" + pin = step(2) + self.assertIn("Move the `sha` of the pin config of every tool outside a workflow that runs the queue's code", + pin) + self.assertIn('](../pinned/README.md)', pin) + self.assertIn('](../../README.md#outside-a-workflow)', pin) + self.assertIn('"sha"', read(os.path.join(FORK_CI, 'actions', 'pinned', 'README.md'))) + + def test_enforce_dispatches_the_backfill_before_arming_and_empties_the_input_only_after(self): + enforce = step(5) + self.assertLess(enforce.index('Dispatch the backfill right after it merges'), + enforce.index('arm no pull request before the backfill has run')) + self.assertIn('Nothing checks that gap', enforce) + self.assertIn("only once the base's config is in `enforce`", enforce) + bullet = halfway('`enforce` merged, backfill not yet dispatched') + self.assertIn('the queue can merge on it', bullet) + self.assertIn('arm nothing before it has run', bullet) + + def test_roll_back_is_the_reverse(self): + step = ' '.join(guide().split('6. **Roll back**')[1].split('**Stopped')[0].split()) + self.assertLess(step.index('emptied input'), step.index('back to `shadow`')) + self.assertLess(step.index('back to `shadow`'), step.index('a pin back')) + + +class Releases(unittest.TestCase): + """The releases the guide names agree with the release notes.""" + + def entry(self, pr): + notes = read(NOTES) + found = [e for e in notes.split('\n### ')[1:] if f'(#{pr})' in e.split('\n')[0]] + self.assertEqual(len(found), 1, f'one release notes entry for #{pr}') + return ' '.join(found[0].split()) + + def test_findings_rounds_and_override_authors_come_after_the_same_release(self): + entry = self.entry(740) + first = re.search(r'First in the release after `(ci/v\d+)`', entry) + self.assertIsNotNone(first, entry) + text = ' '.join(guide().split()) + self.assertIn(f'Findings, rounds and `override_authors` are in the first release after `{first[1]}`', text) + self.assertIn(f'`{first[1]}` or older for `override_authors`', text) + self.assertIn(f'`{first[1]}` and older refuse a config with `override_authors`', entry) + + def test_the_release_notes_upgrade_points_to_the_guide(self): + self.assertIn(f'pr-queue README, "{HEADING}"', self.entry(740)) + + +def git(*a, cwd=FORK_CI, check=False): + env = dict(os.environ, GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT='0') + return subprocess.run(['git', '-c', 'user.name=t', '-c', 'user.email=t@example.invalid', *a], cwd=cwd, env=env, + text=True, capture_output=True, check=check) + + +REVIEW_CONFIG = 'fork-ci/actions/pr-queue/review_config.py' + + +# fork-ci: may skip ReleaseTags.test_the_path_rules_are_first_in_the_release_the_guide_names +# fork-ci: may skip ReleaseTags.test_override_authors_is_first_in_the_release_after_the_one_the_notes_name +# (CI checks out without the ci/vN tags; these run in a clone that has them.) +class ReleaseTags(unittest.TestCase): + """The releases the guide and the notes name, against the ci/vN tags. CI checks out without tags (fetch-depth 2, + no tags), so there these skip and only Releases runs; run them in a clone with the tags (git fetch --tags).""" + + def setUp(self): + tags = git('tag', '-l', 'ci/v*').stdout.split() + self.tags = sorted(int(t[len('ci/v'):]) for t in tags if t[len('ci/v'):].isdigit()) + if not self.tags: + self.skipTest('no ci/vN tags in this clone (CI fetches none): git fetch --tags to run these') + + def source(self, n): + r = git('show', f'ci/v{n}:{REVIEW_CONFIG}') + return r.stdout if r.returncode == 0 else None + + def test_the_path_rules_are_first_in_the_release_the_guide_names(self): + n = int(re.search(r'path rules \(`review`, `shadow` and `enforce`\) are in `ci/v(\d+)` and later', + flat(guide()))[1]) + self.assertIn(n, self.tags) + self.assertIsNotNone(self.source(n), f'ci/v{n} has no review_config.py') + self.assertTrue(all(self.source(k) is None for k in self.tags if k < n), f'review_config.py before ci/v{n}') + self.assertIn(f'`ci/v{n - 1}` or older for `review`', step(6)) + + def test_override_authors_is_first_in_the_release_after_the_one_the_notes_name(self): + n = int(re.search(r'First in the release after `ci/v(\d+)`', Releases().entry(740))[1]) + self.assertIn(n, self.tags) + for k in self.tags: + text = self.source(k) + self.assertEqual(text is not None and 'override_authors' in text, k > n, f'ci/v{k}') + self.assertIn('override_authors', read(os.path.join(QUEUE, 'review_config.py'))) + + +class Rows(unittest.TestCase): + """The rows the guide quotes are the rows the gate writes.""" + + def judgement(self): + return review_judge.Judgement('critical', ('data-1', 'data-2'), + (review_judge.Hit('data', 'critical', ('src/storage/a.py',)),), (), 1) + + def quoted(self, label): + text = ' '.join(guide().split()) + m = re.search(r'`' + re.escape(label) + r'`: `([^`]+)`', text) + self.assertIsNotNone(m, label) + return m[1] + + def as_pattern(self, quoted): + """A quoted row as a regex: `<...>`, `(...)`, `` and `N`/`M` stand for any text.""" + out = re.escape(quoted) + for hole in (re.escape(''), re.escape('<...>'), re.escape('(...)')): + out = out.replace(hole, '.+?') + return out.replace(r'round\ N\ of\ M', r'round \d+ of \d+').replace('round N of M', r'round \d+ of \d+') + + def test_the_path_rules_row(self): + name, good, text = H.path_row('shadow', self.judgement()) + self.assertEqual(name, 'path rules (shadow)') + self.assertIs(good, True) + self.assertRegex(text, '^' + self.as_pattern(self.quoted('path rules (shadow)')) + '$') + + def test_a_config_warning_is_added_as_the_guide_says(self): + _, _, text = H.path_row('shadow', self.judgement(), ('rule a above rule b',)) + self.assertTrue(text.endswith('; config warning: rule a above rule b'), text) + self.assertIn('`; config warning: ...`', guide()) + + def test_the_findings_and_rounds_row(self): + quoted = self.quoted('findings and rounds (shadow)') + rows = run_decide('shadow', rounds=(review_judge.Rounds(2, False, False, (), ()), None))[3] + name, good, text = rows[1] + self.assertEqual(name, 'findings and rounds (shadow)') + self.assertIs(good, True) + self.assertRegex(text, '^' + self.as_pattern(quoted)) + self.assertTrue(text.endswith('; no blocker or major open'), text) + self.assertIn('`; no blocker or major open`', guide()) + + def test_the_findings_row_parts_in_the_order_the_guide_gives(self): + """The approach lens part and the findings: false part, quoted as the guide quotes them, in its order.""" + m = re.search(r'`they would have said: round N of M`, then `([^`]+)` once the round reaches ' + r'`rounds.approach_at`, then `([^`]+)` with `findings: false`', flat(guide())) + self.assertIsNotNone(m) + approach, unjudged = m[1], m[2] + for name in ('', ''): + self.assertIn(name, approach) + approach = re.escape(approach).replace(re.escape(''), r'\S+').replace(re.escape(''), r'\d+') + review = parsed('shadow').review + no_findings = config.parse(json.dumps(dict(BASE, review=dict(recommended(), mode='shadow', findings=False)))) + text = review_judge.Rounds(3, True, False, (), ()).text(no_findings.review) + self.assertRegex(text, '^round 3 of 4' + approach + re.escape(unjudged) + '; no blocker or major open$') + self.assertEqual(review_judge.Rounds(1, False, False, (), ()).text(review), + 'round 1 of 4; no blocker or major open') + + +def run_decide(mode, rounds, verdict=(True, 'every lens read the head', [('code', True, 'read abc')]), + block=True, required=None, judgement=None, paths=None, base=None, seen=None): + """head_check.decide with the gate's reads faked: (code, ok, reason, rows). judgement: the path rules' Judgement; + paths: what path_judgement says instead ((None, why): it cannot judge); base: base_config's side effect instead of + the parsed config; seen: a list that gets the lenses decide requires (what it hands verdict).""" + cfg = parsed(mode) if block else config.parse(json.dumps(BASE)) + hit = review_judge.Hit('code', 'normal', ('x.py',)) + judgement = judgement or review_judge.Judgement('normal', ('code',), (hit,), (), 1) + pr = {'isDraft': False, 'headRefOid': 'a' * 40, 'comments': [], 'reviews': []} + + def no_gh(*a, **k): + raise AssertionError('GitHub must not be called') + + def judged(pr, review, checkout, required, *a): + if seen is not None: + seen.append(dict(required)) + return verdict + with mock.patch.object(H, 'base_config', **({'side_effect': base} if base else {'return_value': cfg})), \ + mock.patch.object(H, 'merged_config_problem', return_value=None), \ + mock.patch.object(H, 'path_judgement', return_value=paths or (judgement, None)), \ + mock.patch.object(H, 'read_pr', return_value=(pr, None)), \ + mock.patch.object(H, 'review_rounds', return_value=rounds), \ + mock.patch.object(H, 'verdict', side_effect=judged): + return H.decide('o/p', 7, '.github/pr-queue.json', object(), required or {'code': 'code'}, False, no_gh, + lambda s: None, lambda line: None, []) + + +CONFIG_PATH = '.github/pr-queue.json' + + +def repository(d, base, head): + """A throwaway repository with two commits, base then head ({path: text}, each the whole tree): their shas.""" + git('init', '-q', d, check=True) + shas = [] + for files in (base, head): + for f in git('ls-files', cwd=d).stdout.split(): + os.remove(os.path.join(d, f)) + for path, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(d, path)), exist_ok=True) + with open(os.path.join(d, path), 'w', encoding='utf-8') as f: + f.write(text) + git('add', '-A', cwd=d, check=True) + git('commit', '-q', '--allow-empty', '-m', 'x', cwd=d, check=True) + shas.append(git('rev-parse', 'HEAD', cwd=d, check=True).stdout.strip()) + return tuple(shas) + + +def merged_problem(base, head): + """head_check.merged_config_problem, bare mode, on a throwaway repository of these two trees.""" + with tempfile.TemporaryDirectory(prefix='review-guide-') as d: + shas = repository(d, base, head) + return H.merged_config_problem(H.Checkout(os.path.join(d, '.git'), bare=shas), CONFIG_PATH) + + +class StoppedHalfWay(unittest.TestCase): + """Each state the guide's "Stopped half-way, or run again" promises.""" + + def test_pins_moved_no_block_yet_is_the_gate_of_before(self): + self.assertIn('the gate judges exactly as before', halfway('Pins moved, no config change yet')) + cfg = config.parse(json.dumps(BASE)) + self.assertIsNone(cfg.review) + code, ok, _, rows = run_decide('shadow', rounds=None, block=False) + self.assertEqual((code, ok, rows), (0, True, [('code', True, 'read abc')])) + + def test_a_key_a_release_does_not_know_refuses_the_whole_config(self): + """What an older release does to a newer key: the whole config is refused, naming the key.""" + self.assertIn('That caller refuses the whole config', halfway('The block added while one caller still pins')) + with self.assertRaises(config.ConfigError) as e: + config.parse(json.dumps(dict(BASE, review=dict(recommended(), later_key=True)))) + self.assertIn('later_key', str(e.exception)) + with self.assertRaises(config.ConfigError) as e: + config.parse(json.dumps(dict(BASE, review=recommended(), later_key=True))) + self.assertIn('later_key', str(e.exception)) + + def test_the_shadow_step_starts_from_the_recommended_block_without_override_authors(self): + cfg = parsed('shadow') + self.assertEqual(cfg.review.mode, 'shadow') + self.assertNotIn('override_authors', recommended()) + self.assertEqual(cfg.review.override_authors, ()) + + def test_a_gate_that_knows_the_block_refuses_the_pull_request_that_adds_a_key_it_does_not_know(self): + """The guide: a gate on ci/v15 or later refuses the block pull request itself. The gate parses the config the + merge would leave with its own parser; a key it does not know (here `later_key`, as `review` is to ci/v14 or + `override_authors` to ci/v20) does not parse, so the pull request is refused before it can merge.""" + bullet = halfway('The block added while one caller still pins') + self.assertIn('a gate on `ci/v15` or later refuses the block pull request itself', bullet) + self.assertIn('the owner step below', bullet) + base = {CONFIG_PATH: json.dumps(BASE)} + why = merged_problem(base, {CONFIG_PATH: json.dumps(dict(BASE, review=dict(recommended(), later_key=1)))}) + self.assertIn('later_key', why or '') + self.assertIsNone(merged_problem(base, {CONFIG_PATH: json.dumps(dict(BASE, review=recommended()))})) + + def test_a_pin_back_is_checked_by_nothing_but_review(self): + """Step 6: a pull request that only moves a pin (a workflow file) leaves the config as it was, so the gate's + merged-config check passes it whatever release it pins; the config may hold a key the old release refuses.""" + self.assertIn('The gate does not check this; it is a review rule', step(6)) + self.assertIn('only the owner step below gets out', step(6)) + with_key = json.dumps(dict(BASE, review=dict(recommended(), override_authors=['owner']))) + self.assertEqual(config.parse(with_key).review.override_authors, ('owner',)) + caller = '.github/workflows/gate.yml' + base = {CONFIG_PATH: with_key, caller: 'uses: o/r/fork-ci/actions/head-check@' + 'b' * 40 + ' # ci/v21\n'} + head = dict(base, **{caller: 'uses: o/r/fork-ci/actions/head-check@' + 'a' * 40 + ' # ci/v20\n'}) + self.assertIsNone(merged_problem(base, head)) + + def test_the_enforce_pull_request_needs_only_the_input_lenses(self): + """Step 5: the pull request that sets enforce is judged under the base's shadow, so the gate requires only the + input's lenses, not the critical ones its own change (the config, a protected path) would need in enforce.""" + self.assertIn('so ask for the critical lenses: the gate does not require them yet', step(5)) + judgement = review_judge.judge(parsed('shadow').review, [('M', (CONFIG_PATH,))], CONFIG_PATH) + self.assertEqual((judgement.weight, judgement.lenses), ('critical', ('data-1', 'data-2'))) + for mode, want in (('shadow', {'code'}), ('enforce', {'code', 'data-1', 'data-2'})): + seen = [] + run_decide(mode, rounds=(review_judge.Rounds(1, False, False, (), ()), None), judgement=judgement, + seen=seen) + self.assertEqual(set(seen[0].values()), want, mode) + + def test_an_empty_input_fails_unless_the_base_is_in_enforce(self): + bullet = halfway('The input emptied while the base is not in `enforce`') + self.assertIn('every head fails, the pull request that puts a lens name back too', bullet) + self.assertIn('The owner step below', bullet) + for cfg, want in ((config.parse(json.dumps(BASE)), 'fails'), (parsed('shadow'), 'fails'), + (parsed('enforce'), None)): + with mock.patch.object(H, 'base_config', return_value=cfg): + problem = G.empty_lenses_problem(object(), '.github/pr-queue.json') + if want is None: + self.assertIsNone(problem) + else: + self.assertIn('required-lenses names no lens (it is empty)', problem) + + def test_an_unreadable_base_config_fails_an_empty_input_too(self): + def unreadable(checkout, path, problems): + problems.append('the config cannot be read') + return None + with mock.patch.object(H, 'base_config', side_effect=unreadable): + self.assertEqual(G.empty_lenses_problem(object(), '.github/pr-queue.json'), 'the config cannot be read') + + def test_the_owner_step_reverts_by_hand_then_dispatches_the_backfill(self): + said = owner_step() + self.assertIn('An owner pushes the fix straight to the default branch, past the branch ruleset', said) + self.assertLess(said.index('the revert of the commit that broke it'), said.index('Then dispatch the backfill')) + self.assertIn('the fix by hand', read(GUIDE_README).split('**Shadow and enforce.**')[1].split('\n\n')[1]) + + def test_the_rows_never_fail_a_head_in_shadow(self): + self.assertIn('The rows never fail a head in `shadow`', step(4)) + open_finding = review_judge.Rounds(5, True, True, (('code', 'major', 'major: x'),), + ('lens code reported a major',)) + code, ok, reason, rows = run_decide('shadow', rounds=(open_finding, None)) + self.assertEqual((code, ok, reason), (0, True, 'every lens read the head')) + self.assertEqual([good for _, good, _ in rows], [True, True, True]) + self.assertTrue(rows[1][2].startswith('they would have said: round 5 of 4'), rows[1][2]) + + def test_the_same_rows_fail_it_in_enforce(self): + self.assertIn('`enforce` would have failed a head that passed', step(4)) + open_finding = review_judge.Rounds(2, False, False, (('code', 'major', 'major: x'),), + ('lens code reported a major',)) + code, ok, reason, rows = run_decide('enforce', rounds=(open_finding, None)) + self.assertEqual((code, ok), (1, False)) + self.assertIn('lens code reported a major', reason) + self.assertIs(rows[1][1], False) + + def test_each_thing_the_guide_says_cannot_be_judged_fails_in_both_modes(self): + """Step 4's list, item by item: a list item with no case here, or a case the list leaves out, goes red.""" + listed = re.search(r'a change the rules cannot judge \(([^)]+)\) fails it in both modes', step(4)) + self.assertIsNotNone(listed) + items = [x.strip() for x in re.split(r', ', listed[1])] + + def unparsable(checkout, path, problems): + problems.append('the config does not parse') + return None + rounds = (review_judge.Rounds(1, False, False, (), ()), None) + cases = { + 'no merge base': dict(rounds=rounds, paths=(None, 'no merge base of the base and the head')), + 'a failed diff': dict(rounds=rounds, paths=(None, 'git diff failed')), + 'a config that does not parse': dict(rounds=rounds, base=unparsable), + 'lens comments that cannot be read': dict(rounds=(None, 'GitHub said no')), + } + self.assertEqual(items, list(cases)) + for item, kw in cases.items(): + for mode in ('shadow', 'enforce'): + code, ok, _, _ = run_decide(mode, **kw) + self.assertEqual((code in (1, H.REFUSED), ok), (True, False), f'{item}, {mode}') + code, ok, reason, _ = run_decide('shadow', rounds=(None, 'GitHub said no')) + self.assertIn('cannot be judged, so the head is not passed', reason) + self.assertIn('lens comments that cannot be read (the rounds cannot be judged)', + flat(read(GUIDE_README).split('**Fail closed.**')[1].split('\n\n')[0])) + + +class EmptyInputFloor(unittest.TestCase): + """Step 5: once the input may be empty (enforce), it is no floor. A change that only zero-lens rules match needs no + lens; the guide's rule for agent instruction files keeps them at one.""" + + def snippet(self): + m = re.search(r'```json\n(.*?)\n\s*```', guide(), re.S) + self.assertIsNotNone(m) + return json.loads(m[1]) + + def judged(self, review, path): + return review_judge.judge(review, [('M', (path,))], CONFIG_PATH) + + def test_with_the_input_empty_a_markdown_change_needs_no_lens(self): + enforce = step(5) + self.assertIn('a change that only rules with `"lenses": []` match needs no lens at all', enforce) + self.assertIn('matches agent instruction files such as `AGENTS.md` and `CLAUDE.md`', enforce) + with mock.patch.object(H, 'base_config', return_value=parsed('enforce')): + self.assertIsNone(G.empty_lenses_problem(object(), CONFIG_PATH)) # the empty input may stand + review = parsed('enforce').review + for path in ('AGENTS.md', 'tools/CLAUDE.md', 'docs/notes.txt'): + judgement = self.judged(review, path) + self.assertEqual((judgement.weight, judgement.lenses), ('light', ()), path) + + def test_the_guide_rule_keeps_agent_instruction_files_at_one_lens(self): + self.assertIn('put a rule for the agent instruction files above the `docs` rule', step(5)) + raw = recommended() + rules = raw['rules'] + at = next(i for i, r in enumerate(rules) if r['name'] == 'docs') + raw = dict(raw, mode='enforce', rules=rules[:at] + [self.snippet()] + rules[at:]) + review = config.parse(json.dumps(dict(BASE, review=raw))).review # enforce refuses a bad rule order + for path in ('AGENTS.md', 'tools/CLAUDE.md'): + self.assertEqual(self.judged(review, path).lenses, ('code',), path) + self.assertEqual(self.judged(review, 'README.md').lenses, ()) + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index 1dc8e28344e6f..73f1731530b4c 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -66,7 +66,8 @@ the findings, the approach lens and the round cap from the lens comments (below, "Findings and rounds"). The `state_line` key is parsed but not enforced yet. `review.recommended.json` (next to this file) is a complete example with made-up paths; a test parses it. Real paths, lens names and prompt files belong -in the project's own config (below, "Turning it on"). +in the project's own config (below, "Turning it on: release, pin, then +config"). Every key is required except `override_authors`, and an unknown key is an error, as in the rest of the config. A wrong block makes the whole config invalid, naming every problem at @@ -284,16 +285,6 @@ to the config itself and to a lens prompt need the critical lenses: the extra review. A project whose gate or queue code lives in its own repository (the fork's `fork-ci/`) adds a critical rule for those paths. -**Turning it on.** A project opts in through its own config, never through -the fork: add the `review` block to the queue config its head gate reads -(`config` input), with its real paths, lens names and prompt files, in -`shadow` first. Read a few gate summaries, fix every config warning, then set -`mode` to `enforce` and dispatch the backfill (head-check README). The fork's -own config is [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), -in `shadow`, without `override_authors` (no person yet) until issue #78 is done; its lens -prompts (`.github/fork-lens-*.md`) tell each lens the exact header and findings -lines, so a switch to `enforce` does not fail every pull request. - **The empty input.** The gate refuses an empty `required-lenses` input (a head with no lens comment would pass), except when the config at the base tip has a `review` block in `enforce`: then the path rules name the lenses. A name that @@ -301,16 +292,151 @@ names no lens (``, `r4`) is refused in every mode. **Fail closed.** No merge base (the base was force-pushed, or the histories are unrelated), a failed `git diff`, a diff line the judge cannot read, a base or -head that cannot be fetched, or a `review` block that does not parse: the gate +head that cannot be fetched, a `review` block that does not parse, or lens +comments that cannot be read (the rounds cannot be judged): the gate posts a failure that says why, in `shadow` as in `enforce`. Never a pass, never "no lenses needed", and never today's rule instead. A config without the `review` key gives exactly the gate of before this block. -**Adding the block.** A release that does not know the `review` key refuses the -whole config (an unknown key), so: release first, then pin that release in every -caller of the queue and the gate, then add the key. Roll back in the reverse order: -remove the key, then the pin. The same holds for `override_authors`: a -release from before it refuses a config that has it. +### Turning it on: release, pin, then config + +A project opts in through its own config, never through the fork. The order +matters: a release that does not know a key refuses the whole config (an +unknown key), and a refused config stops the queue and makes the gate refuse +every head, the pull request that would fix it too. So: release first, then pin +that release in every caller, then the config. Each step after the release is a +pull request of its own on the default branch. + +1. **Release.** The fork makes the releases + ([fork-ci README, "How a release is made"](../../README.md#how-a-release-is-made)); + a project never makes one. The path rules (`review`, `shadow` and `enforce`) + are in `ci/v15` and later. Findings, rounds and `override_authors` are in + the first release after `ci/v20` + ([RELEASE-NOTES.md](../../RELEASE-NOTES.md), #740). Take the newest + `ci/vN` and its release sha as in + [Pinning](../../README.md#pinning). +2. **Pin.** In one pull request, move every `uses:` of fork-ci in the + project to that sha, with the tag as the comment: the queue's caller + (`fork-pr-queue.yml` or the `pr-queue` action), the gate's caller + ([`gate-caller.example.yml`](../head-check/gate-caller.example.yml)) and + any job that runs the `head-check` action. Move the `sha` of the pin + config of every tool outside a workflow that runs the queue's code too + ([`actions/pinned`](../pinned/README.md), fork-ci README, + ["Outside a workflow"](../../README.md#outside-a-workflow)): on an older + release it refuses the new key the same way. Read the release notes between + the old and the new tag first. After it merges, dispatch the gate's + backfill (head-check README, "After a change to the caller, a new pin"). + The config does not change yet, so the gate judges as before. A project + without the gate sets it up first (head-check README, "The gate", setup + steps). In a public repository the caller needs a repository Actions + policy that allows `pull_request_target` for exactly its file: keep the + file name the policy names. +3. **Config in `shadow`.** Add the `review` block to the queue config the + gate reads (its `config` input) with `"mode": "shadow"`. Start from + `review.recommended.json` and put in the real paths, lens names and prompt + files. Leave `override_authors` out until a person, not an agent, posts + overrides from a login of their own (issue #78). The lens prompts ask for + the lens header and the findings section (sections "Lens comments" and + "Findings and rounds"; the fork's own prompts, `.github/fork-lens-*.md`, + are an example), so a later switch to `enforce` does not fail every pull + request. Keep the caller's `required-lenses` as it is: in `shadow` it + still decides. This pull request is judged by the old rules (the gate + reads the config from the base tip), and its merged config must parse, or + the gate refuses it. The fork's own config, + [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), is at + this step. +4. **Read the results.** From now on every gate run adds rows to the + summary of the gate App's check run on the head ("Details" on the pull + request, or + `gh api "repos/OWNER/REPO/commits/HEAD_SHA/check-runs?check_name=CHECK_NAME&filter=latest" --jq '.check_runs[0].output.summary'`): + - `path rules (shadow)`: + `the path rules would have said: , lenses <...> (...); the required-lenses input decides (mode shadow)`, + and `; config warning: ...` for each problem `enforce` would refuse (the + run log has it as a warning too); + - `findings and rounds (shadow)`: `they would have said: round N of M`, + then `, approach lens required from round ` once the round + reaches `rounds.approach_at`, then + `; findings are not judged (findings: false)` with `findings: false`, + then the blockers and majors still open, a lens report with no findings + section or the round cap, or `; no blocker or major open`. + + Compare them with the real verdict of each pull request that merges. A + difference means `enforce` would have failed a head that passed (a lens + the input does not name, an open finding, a report without a findings + section) or the rules are wrong: fix the config or the lens prompts. The + rows never fail a head in `shadow`; a change the rules cannot judge (no + merge base, a failed diff, a config that does not parse, lens comments + that cannot be read) fails it in both modes. + +5. **Enforce.** When the rows have matched the real verdicts over the + number of merged pull requests the project picked, and no row shows a + config warning, one pull request sets `"mode": "enforce"`. It changes the + config, so ask for the critical lenses: the gate does not require them yet + (the base is still in `shadow`, where only the input decides). Dispatch + the backfill right after it merges, and arm no pull request before the + backfill has run: each open pull request is judged by the new rules only at + its next run, so until then the queue can merge one on a pass it got under + `shadow`. Nothing checks that gap. Now the path rules' lenses are required, + and the `required-lenses` input is required as well (it can add lenses, + never remove them). + + A later pull request may empty the input (the path rules name the + lenses), but only once the base's config is in `enforce`: an empty input + fails every head under `shadow` or without the block (above, "The empty + input"). Emptying it also drops a floor: while the input names a lens, + every change needs at least that lens; once it is empty, a change that only + rules with `"lenses": []` match needs no lens at all. In + `review.recommended.json` that is the `docs` rule (`docs/**`, `**/*.md`), + which also matches agent instruction files such as `AGENTS.md` and + `CLAUDE.md`: an edit to them would merge unread. A project that wants a + lens on every change keeps the input, or gives no rule zero lenses. At + least, put a rule for the agent instruction files above the `docs` rule: + + ```json + { + "name": "agent-instructions", + "weight": "normal", + "paths": ["**/AGENTS.md", "**/CLAUDE.md"], + "lenses": ["code"] + } + ``` + +6. **Roll back** in the reverse order: lens names back into an emptied + input, then `mode` back to `shadow` (or the block out), and only then a + pin back. Never pin a release that does not know a key the config still + has (`ci/v14` or older for `review`, `ci/v20` or older for + `override_authors`). The gate does not check this; it is a review rule. + The pin-back pull request is judged by the current gate, which knows the + key, so it can merge; after it, the old release refuses the config, every + head fails, and only the owner step below gets out. + +**Stopped half-way, or run again.** Every step leaves a state that works or +fails closed: + +- Pins moved, no config change yet: the gate judges exactly as before (no + `review` key is the gate of before). +- The block added while one caller still pins an older release: a gate on + `ci/v15` or later refuses the block pull request itself (the config it would + merge does not parse), so this happens only when the old caller is the + queue or a `ci/v14` gate, or the block was merged by hand. That caller + refuses the whole config. The queue stops: the gate still passes the pull + request that moves the queue's pin, so merge that one by hand. A `ci/v14` + gate fails every head, the pull request that moves its pin too: the owner + step below. That is why step 2 moves every caller. +- `enforce` merged, backfill not yet dispatched: a pass posted under + `shadow` stays on its head until the next event on it, and the queue can + merge on it. Dispatch the backfill and arm nothing before it has run; a + second dispatch judges the same heads again and changes nothing else. +- The input emptied while the base is not in `enforce`: every head fails, + the pull request that puts a lens name back too (the base's caller judges + it, with the empty input). The owner step below. + +**The owner step.** When every head fails, the pull request that would fix it +too, no pull request gets out; nobody waits for one. An owner pushes the fix +straight to the default branch, past the branch ruleset (as for a base config +that does not parse, "Shadow and enforce" above): the revert of the commit that +broke it (a lens name back into the input, the pin moved forward, or the block +taken out), nothing else. Then dispatch the backfill. ## Lens comments