diff --git a/.github/fork-lens-approach.md b/.github/fork-lens-approach.md new file mode 100644 index 000000000000..af3f4556efa9 --- /dev/null +++ b/.github/fork-lens-approach.md @@ -0,0 +1,26 @@ +# Lens: approach + +This pull request has reached its third review round or later. Do not look for +more small findings. Ask whether the whole approach is right: + +- Does it solve the problem the issue names, at the root? +- Is there a simpler way, or an existing mechanism it should build on? +- Will more rounds converge, or should it be split, restarted or closed? + +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: approach · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-code.md b/.github/fork-lens-code.md new file mode 100644 index 000000000000..3d4c47ad682a --- /dev/null +++ b/.github/fork-lens-code.md @@ -0,0 +1,27 @@ +# Lens: code + +You review one pull request of this repository: is it correct, tested and as +simple as it can be? Read the diff and the files it touches; run its tests. + +- Correctness first: wrong results, crashes, missed cases. +- Ask: what happens when this runs again, stops half-way, or meets existing, + old or empty state? +- Tests: do they cover the change, and do they fail on the old code? + +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: code · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-merge-path.md b/.github/fork-lens-merge-path.md new file mode 100644 index 000000000000..436e0ea0cfed --- /dev/null +++ b/.github/fork-lens-merge-path.md @@ -0,0 +1,31 @@ +# Lens: merge path and security + +You review one pull request of this repository from one angle: can it let a +change merge that should not, or leak or weaken anything? Read the diff and +the files it touches; run its tests. + +- Every gate, check, ruleset and workflow it touches: can a head pass that no + review read, or a check pass without running? +- Fail closed: an unknown, unreadable or empty answer must block, never pass. +- Tokens and secrets: none printed, stored or handed to code the pull request + controls. +- This repository is public: nothing personal (names, addresses, paths of a + person's machine, tokens) in files, commits or comments. + +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: merge-path · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-state.md b/.github/fork-lens-state.md new file mode 100644 index 000000000000..27624dd28413 --- /dev/null +++ b/.github/fork-lens-state.md @@ -0,0 +1,28 @@ +# Lens: state and tests + +You review one pull request of this repository from one angle: does it hold +when it meets real state, and do its tests prove it? Read the diff and the +files it touches; run its tests. + +- Ask every time: what happens when this runs again, stops half-way, or meets + existing, old or empty state? +- Each fix line should turn a test red when it is removed. Spot-check some. +- New tests should fail on the old code. Check one. + +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: state · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-pr-queue.json b/.github/fork-pr-queue.json new file mode 100644 index 000000000000..a77cb870e66f --- /dev/null +++ b/.github/fork-pr-queue.json @@ -0,0 +1,39 @@ +{ + "required_checks": [ + "fork-ci-result", + "No database schema changes", + "Outside pull request", + "head_check" + ], + "merge_method": "squash", + "bot_logins": [], + "lens_author_associations": ["OWNER"], + "override": { "prefix": "override for head", "author_association": "OWNER" }, + "ready_command": "gh pr merge {pr} --auto --squash", + "push_command": "git push", + "lens_format_doc": "fork-ci/actions/pr-queue/README.md, section Lens comments", + "head_check": "head_check", + "review": { + "version": 1, + "mode": "shadow", + "rules": [ + { + "name": "gate", + "weight": "critical", + "paths": [".github/**", "fork-ci/**"], + "lenses": ["merge-path", "state"] + }, + { "name": "code", "weight": "normal", "paths": ["**"], "lenses": ["code"] } + ], + "lenses": { + "merge-path": { "prompt": ".github/fork-lens-merge-path.md" }, + "state": { "prompt": ".github/fork-lens-state.md" }, + "code": { "prompt": ".github/fork-lens-code.md" }, + "approach": { "prompt": ".github/fork-lens-approach.md" } + }, + "min_lenses": { "critical": 2, "normal": 1, "light": 0 }, + "rounds": { "approach_at": 3, "max": 4, "approach_lens": "approach" }, + "findings": true, + "state_line": false + } +} diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index ef5c1c2a284b..2ecab8ea1985 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -50,11 +50,28 @@ All of them are the queue's, imported from `../pr-queue`, never copied: summary only adds a row "the path rules would have said: ...". A change the rules cannot judge (no merge base, a failed diff) refuses, in either mode. Without the block, nothing of this runs. +- **Findings and rounds** (`review_judge.judge_rounds`, the pr-queue README, + "Findings and rounds"): with a `review` block in `enforce`, a lens whose + newest report has a blocker or major finding (a `blocker:` or `major:` line, + a `### Blocking` or `### Major` heading with items, a table row ...) fails + the head until a later round of that lens clears it or a person's override + newer than the report covers it (`findings: true`); a report with no findings + section (a blocker label and a major label, `none` allowed) fails too. A person is a login in `override_authors`; none + listed (the default) means no override clears a finding or passes the cap + (issue #78). A round is a read of a different own change (`own_change`: a + head that only merges the base in is no new round); from round + `rounds.approach_at` the approach lens is required; a round past + `rounds.max` fails the head until a person's override for it, newer than + every lens comment, lets it through on its own. In `shadow` the summary only + adds a row "they would have said: ...". Comments that cannot be judged fail + the head, in either mode. It passes when: - every lens read the head, or a newer override vouches for it, and every - required lens (`required-lenses`) is among them; + required lens (`required-lenses`, and in `enforce` the path rules' and the + approach lens) is among them, with no open blocker or major and no round past + the cap (`enforce` only); - the pull request has no lens comments and no lens is required: there is nothing to compare (the queue then vouches only for the head it found); - it is a draft and `pass-drafts` is `true`. diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index 4da69ceba5a0..03adf131c770 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -19,7 +19,13 @@ The config's optional `review` block (../pr-queue/README.md, "Review rules") names lenses by changed path: the paths are `git diff --name-status -M` from the merge base to the head (changed_paths), judged by review_judge.judge. In `enforce` its lenses are required too; in `shadow` the summary only says what they would have been. A change it cannot -judge (no merge base, a failed diff) is refused, never passed. Without the block nothing of this runs. +judge (no merge base, a failed diff) is refused, never passed. The same block judges the review itself +(review_judge.judge_rounds, review_rounds): in `enforce` a lens's newest report with a blocker or major finding that +no later round or person cleared fails the head (`findings: true`), and so does a report with no findings section; +the approach lens is required from round `rounds.approach_at`; a round past `rounds.max` fails the head until a +person's override comment for it (a login in `override_authors`; none listed: no override is a person's, issue #78) +lets it through. A round is a read of a different own change (own_change), so a queue update adds none. In `shadow` +the summary only says so. Without the block nothing of this runs. Pass: every lens read the head (or a newer override vouches for it), and every required lens is there; a pull request with no lens comments and no required lens (nothing to compare); a draft, with --pass-drafts only. Fail, @@ -38,7 +44,7 @@ head come from differs (Checkout). `--required-lenses` takes one lens name per line (a name may hold a comma: `data-integrity (DI-1, DI-2)`). """ -import argparse, json, os, re, subprocess, sys, tempfile, time, unicodedata +import argparse, hashlib, json, os, re, subprocess, sys, tempfile, time, unicodedata sys.dont_write_bytecode = True # no __pycache__ next to the scripts sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pr-queue')) @@ -49,6 +55,7 @@ import review_judge # noqa: E402 ATTEMPTS, RETRY_SECONDS = 3, 10 +FETCH_SECONDS = 120 # a fetch of a read commit that takes longer counts as failed (the job has 15 minutes in all) FIELDS = 'isDraft,headRefOid,comments,reviews' SHA = pr_queue.SHA REFUSED = pr_queue.REFUSED @@ -119,10 +126,14 @@ def has(self, sha): def fetch(self, sha): """A read commit no ref of the repository reaches (a force-push dropped it; GitHub keeps it), from origin with the - credentials the checkout stored or the gate's environment gives (none: a private repository then fails).""" - return self.git('fetch', '-q', '--no-tags', '--no-recurse-submodules', self.remote, sha, - fetch=True).returncode == 0 \ - and self.has(sha) + credentials the checkout stored or the gate's environment gives (none: a private repository then fails). A fetch + that takes over FETCH_SECONDS failed.""" + try: + done = self.git('fetch', '-q', '--no-tags', '--no-recurse-submodules', self.remote, sha, fetch=True, + timeout=FETCH_SECONDS).returncode == 0 + except subprocess.TimeoutExpired: + return False + return done and self.has(sha) def required_names(text): @@ -344,12 +355,115 @@ def decide(repo, n, config_path, checkout, required, pass_drafts, gh, sleep, out return 1, False, f'could not read the pull request from GitHub after {ATTEMPTS} tries: {why}', [] review = lens.Review(cfg.bot_logins, cfg.override_prefix, cfg.override_association, cfg.lens_author_associations, log=lambda text: out(escape(text))) + rounds = None + if paths is not None and not (pr['isDraft'] and pass_drafts): + rounds, why = review_rounds(cfg.review, review, pr, checkout, cfg.override_prefix) + if rounds is None: # fail closed, in shadow as in enforce, as for the path rules + return 1, False, f'the review rounds and findings cannot be judged, so the head is not passed: {why}', [row] + if enforced and rounds.approach: + required.setdefault(lens.lens_key(cfg.review.rounds.approach_lens), cfg.review.rounds.approach_lens) ok, reason, rows = verdict(pr, review, checkout, required, pass_drafts, enforced) + if rounds is not None: + mode = cfg.review.mode + rows = [(f'findings and rounds ({mode})', not (enforced and rounds.problems), + ('' if enforced else 'they would have said: ') + rounds.text(cfg.review))] + rows + out(f'findings and rounds ({mode}): {escape(rows[0][2])}') + if enforced and rounds.problems: + reason = '; '.join(rounds.problems) + ('' if ok else f'; and: {reason}') + ok = False if paths is not None: rows = [row] + rows return (0 if ok else 1), ok, reason, rows +def own_change(checkout, base): + """change_of for review_judge.review_round: what the pull request itself changes at a read commit, so two reads of + the same change are one round. The key is a hash of `git diff -U0` from the commit's merge base with `base` to it, + hunk line numbers dropped: a head that only merges the base in (a queue or branch update) has the same own change + as the head it updates, and is no new round. The `index ..` lines and the whole hunk header (line + numbers, and the function name git shows) are dropped too: they move when the base edits other lines of a file the + pull request changes; --binary keeps a binary file's content in the key. A short sha is resolved when it names one + commit. A commit that is not in the repository and cannot be fetched, more than one merge base or a failed diff: a + round of its own (unknown counts more rounds, never fewer).""" + cache = {} + + def of(sha): + if sha not in cache: + cache[sha] = diff_key(checkout, base, sha) or ('unknown', sha) + return cache[sha] + return of + + +def diff_key(checkout, base, sha): + if lens.FULL_SHA.fullmatch(sha): + if not (checkout.has(sha) or checkout.fetch(sha)): + return None + full = sha + else: + r = checkout.git('rev-parse', '-q', '--verify', '--end-of-options', sha + '^{commit}') + full = r.stdout.strip() if r.returncode == 0 else '' + if not SHA.fullmatch(full): + return None + r = checkout.git('merge-base', '--all', base, full) + bases = r.stdout.split() if r.returncode == 0 else [] + if len(bases) != 1 or not SHA.fullmatch(bases[0]): + return None + d = checkout.git('diff', '--no-ext-diff', '--no-textconv', '--no-relative', '--ignore-submodules=none', + '--no-color', '--no-renames', '--binary', '-U0', bases[0], full, '--', + errors='surrogateescape') + if d.returncode != 0: + return None + text = re.sub(r'^index .*\n', '', re.sub(r'^@@ .*', '@@', d.stdout, flags=re.M), flags=re.M) + return ('change', hashlib.sha256(text.encode('utf-8', 'surrogateescape')).hexdigest()) + + +def review_rounds(rules, review, pr, checkout, prefix): + """(review_judge.Rounds, None) of the pull request's lens comments under the base's `review` rules, or (None, why). + The round counts every lens header the check reads now (lens_scan's history: old, edited and hidden ones too; + a deleted comment is gone from GitHub, so only the round marker of a later header keeps it counted), each read + change once (own_change). Each lens's newest entry is judged as review_judge says (REPORT ...): a person's override + (an override comment by a login in review.override_authors that vouches for this head: its sha or that change + merged with the base) newer than the entry clears it; any other override clears no finding. Any error is a why, + never a pass.""" + try: + head = str(pr.get('headRefOid') or '').lower() + parents = checkout.merge_parents() + if not parents: + return None, "HEAD is not GitHub's two-parent merge commit, so the rounds are unknown" + entries, history = review.lens_scan(pr) + people = {a.lower() for a in rules.override_authors} + for _, _, post in review.lens_posts(pr): # listed, but GitHub records another association: it never counts + login = (post.get('author') or {}).get('login') or '' + if login.lower() in people and review.override_re.match(post.get('body') or '') \ + and post.get('authorAssociation') != review.override_association: + review.log(f'warning: the override comment by {login} does not count: GitHub records its author ' + f'association as {post.get("authorAssociation") or "none"}, and an override counts only ' + f'with {review.override_association} (override.author_association); being listed in ' + f'review.override_authors does not change that') + overrides = [((when, i), sha, post) for when, i, post in review.lens_posts(pr) + for sha in [review.override_sha(post)] if sha] + newest = max((at for at, _, _ in overrides), default=None) + person = max((at for at, sha, post in overrides + if ((post.get('author') or {}).get('login') or '').lower() in people + and (sha == head or same_change(checkout, head, sha)[0])), default=None) + reports = {} + for key, (at, sha, _, post) in entries.items(): + if person is not None and at < person: + how = review_judge.CLEARED + elif key.startswith(lens.NO_HEADER): + how = review_judge.LOOSE if newest is not None and at < newest else review_judge.NOT_JUDGED + elif lens.FULL_SHA.fullmatch(sha or ''): + how = review_judge.REPORT + else: + how = review_judge.UNREAD if newest is not None and at < newest else review_judge.NOT_JUDGED + reports[key] = (how, post.get('body') or '') + decided = person is not None and all(at < person for at, _, _, _ in entries.values()) + return review_judge.judge_rounds(rules, history, reports, decided, own_change(checkout, parents[0]), + prefix), None + except Exception as e: # noqa: BLE001 - unknown never passes + return None, f'{type(e).__name__}: {review_judge.shown(str(e))[:300]}' + + def argument_problems(repo, n, required_text): """(required, problems) for the arguments both modes take.""" problems = [] diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index 413a07037836..a55c12804d0c 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -1273,6 +1273,283 @@ def test_a_merge_tree_that_takes_too_long_is_refused(self): # #413 item 5: a t self.assertEqual(self.check(base, head, EVERY_LENS)[0], 0) # the same merge, given its time +# --- findings and rounds (#99, #100): the lens comments' blocker and major lines and how many rounds they read ----- +OLD = ['d4' * 20, 'e5' * 20, 'f6' * 20, '07' * 20] # heads earlier rounds read (gone from the branch now) + + +class FindingsAndRounds(unittest.TestCase): + """The base's `review` block (the shared vectors' rules: findings true, approach lens from round 3, cap 4) judges + the lens comments' findings and rounds. Each test changes one normal path, so the path rules need the lens code.""" + BARE = False + STORE = PathRules.STORE + setUp, base, head, check = PathRules.setUp, PathRules.base, PathRules.head, PathRules.check + + def said(self, name, sha, *lines, **kw): + """A lens report; with no lines a clean one (`blocker: none`, `major: none`).""" + lines = lines or ('blocker: none', 'major: none') + return self.p.post(f'{LENS}: {name} · reviewer: x · read at: {sha}\n' + '\n'.join(lines), **kw) + + def pair(self, mode='enforce', **change): + """A base with the review block (changed by `change`) and a head on it that changes one normal path.""" + cfg = review_cfg(mode, **change) + b = self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE}), [self.m0]) + return b, self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': 'a\n'}), [b]) + + def test_a_major_in_the_newest_report_fails_the_head(self): + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', head, 'major: two writers race')]) + self.assertEqual(code, 1, out) + self.assertIn("lens code reported a major that no later round cleared (issue #99): 'major: two writers race'", + out) + self.assertIn('findings and rounds (enforce): round 1 of 4; lens code reported a major', out) + + def test_shadow_only_says_what_it_would_have_said(self): + base, head = self.pair('shadow') + code, out = self.check(base, head, comments=[self.said('code', head, '**Blocker:** data lost on restart')]) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (shadow): they would have said: round 1 of 4; lens code reported a blocker', + out) + + def test_a_later_round_with_no_finding_clears_it(self): # runs again on the same pull request after a fix + base, head = self.pair() + old = self.said('code', OLD[0], 'major: two writers race') + code, out = self.check(base, head, comments=[old, self.said('code r2', head)]) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (enforce): round 2 of 4; no blocker or major open', out) + + def test_a_round_stopped_half_way_keeps_the_old_finding(self): # the fix was pushed, the lens never read it + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', OLD[0], 'blocker: x')]) + self.assertEqual(code, 1, out) + self.assertIn('lens code reported a blocker', out) + + def test_a_deleted_newer_report_brings_the_old_finding_back(self): # deleted later: GitHub no longer lists it + base, head = self.pair() + first, second = self.said('code', head, 'blocker: x'), self.said('code r2', head) + self.assertEqual(self.check(base, head, comments=[first, second])[0], 0) + code, out = self.check(base, head, comments=[first]) + self.assertEqual(code, 1, out) + self.assertIn("lens code reported a blocker that no later round cleared (issue #99): 'blocker: x'", out) + + def test_an_edited_report_reads_nothing_and_still_counts_as_a_round(self): # edited later + base, head = self.pair() + comments = [self.said('code', OLD[0], edited=True), self.said('code', OLD[1], hidden=True), + self.said('code', head, 'major: fixed later', edited=True)] + code, out = self.check(base, head, comments=comments) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4', out) # edited and hidden headers still count as rounds + self.assertNotIn('reported a major', out) # it reads nothing: the edit itself fails it + self.assertIn('edited', out) + + def test_the_approach_lens_is_required_from_round_three(self): + base, head = self.pair() + rounds = [self.said('code', OLD[0]), self.said('code', OLD[1])] + self.assertEqual(self.check(base, head, comments=rounds[:1] + [self.said('code', head)])[0], 0) + code, out = self.check(base, head, comments=rounds + [self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: approach', out) + code, out = self.check(base, head, comments=rounds + [self.said('code', head), self.said('approach', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_past_the_cap_it_stops_and_only_a_persons_override_passes(self): # #100 + base, head = self.pair() + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head), self.said('approach', head)] + code, out = self.check(base, head, comments=five) + self.assertEqual(code, 1, out) + self.assertIn('review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides', out) + code, out = self.check(base, head, comments=five + [self.p.override(head)]) + self.assertEqual(code, 1, out) # no override_authors (the default, #78): the owner's login proves no person + self.assertIn('no one is listed in review.override_authors yet, so no override is accepted', out) + base, head = self.pair(override_authors=['owner']) + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head)] # no approach lens at all + code, out = self.check(base, head, comments=five) + self.assertEqual(code, 1, out) + self.assertIn('an override comment `tool ready --head : ` for this head by owner', out) + code, out = self.check(base, head, comments=five + [self.p.override(head)]) + self.assertEqual(code, 0, out) # the override alone is enough: the approach lens is waived too + stranger = self.p.override(head, assoc='CONTRIBUTOR', login='someone') # not the owner's: counts for nothing + self.assertEqual(self.check(base, head, comments=five + [stranger])[0], 1) + other = self.p.override(head, login='agent-login') # the right association, a login not listed + self.assertEqual(self.check(base, head, comments=five + [other])[0], 1) + override = self.p.override(head) + later = self.said('code', head) # a lens comment newer than the override: the person decided before it + self.assertEqual(self.check(base, head, comments=five + [override, later])[0], 1) + + def test_only_a_listed_persons_override_clears_a_finding(self): # #99, #78 + base, head = self.pair() + report = self.said('code', head, 'blocker: wrong, the lock is there') + self.assertEqual(self.check(base, head, comments=[report])[0], 1) + code, out = self.check(base, head, comments=[report, self.p.override(head)]) + self.assertEqual(code, 1, out) # an override from a login agents still post as clears nothing + self.assertIn("lens code reported a blocker that no later round cleared (issue #99): 'blocker: wrong", out) + base, head = self.pair(override_authors=['Owner']) # logins compare case-insensitively + report = self.said('code', head, 'blocker: wrong, the lock is there') + self.assertEqual(self.check(base, head, comments=[report, self.p.override(head)])[0], 0) + override = self.p.override(head) + later = self.said('code r2', head, 'blocker: wrong again') # a report newer than the override counts again + self.assertEqual(self.check(base, head, comments=[report, override, later])[0], 1) + + def test_an_unreadable_report_under_an_agents_override_fails(self): # hidden after the override covered it + base, head = self.pair() + hidden = self.said('code', head, 'blocker: x', hidden=True) + code, out = self.check(base, head, comments=[hidden, self.p.override(head)]) + self.assertEqual(code, 1, out) + self.assertIn('lens code: its newest report reads nothing and only an override that is no person', out) + base, head = self.pair(override_authors=['owner']) + hidden = self.said('code', head, 'blocker: x', hidden=True) + self.assertEqual(self.check(base, head, comments=[hidden, self.p.override(head)])[0], 0) + + def test_a_report_with_no_findings_section_fails_the_head(self): # #99: never passes as clean + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', head, 'Looks good to me.')]) + self.assertEqual(code, 1, out) + self.assertIn('lens code: its newest report has no findings section, so it cannot be judged (issue #99)', out) + self.assertEqual(self.check(base, head, comments=[self.said('code', head, '### Major', 'None.')])[0], 1) + code, out = self.check(base, head, comments=[self.said('code', head, 'blocker: none', 'F1 (major): x')]) + self.assertEqual(code, 1, out) # #740 round 2: one label kind and a major in a form not read: no section + self.assertIn('lens code: its newest report has no findings section', out) + clean = ('### Blocking', 'None.', '### Major', 'None.') + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *clean)])[0], 0) + + def test_a_finding_in_a_heading_or_table_or_lookalike_fails_the_head(self): # #99, both ways + base, head = self.pair() + for lines in (('### Blocking', '1. **The lock is gone.**'), ('| major | two writers race |',), + ('maj\u043er: x',), ('Blocker: none, major: x',)): + with self.subTest(lines): + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *lines)])[0], 1) + clean = ('blocker: none', '```', 'major: quoted from a test vector', '```', 'Major: none found') + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *clean)])[0], 0) + + def test_a_queue_update_is_no_new_round(self): # #100: a round is a read after a fix, not a new sha + base, h1 = self.pair() + cfg = review_cfg('enforce') + tree = dict(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': 'a\nb\n'})) + h2 = self.w.commit(tree, [h1]) # the fix: round 2 + base2 = self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'docs/d.md': 'd\n'}), [base]) + h3 = self.w.merge(h2, base2) # the queue merges the base in: the same change, no round + reads = [self.said('code', h1), self.said('code r2', h2), self.said('code', h3)] + code, out = self.check(base2, h3, comments=reads) + self.assertEqual(code, 0, out) # round 3 would need the approach lens + self.assertIn('round 2 of 4; no blocker or major open', out) + again = self.w.commit({**tree, 'docs/d.md': 'd\n'}, [base2]) # the fix rebased onto the new base: the same + code, out = self.check(base2, again, comments=reads[:2] + [self.said('code', again)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + fixed = self.w.merge(h2, base2, files={**tree, 'docs/d.md': 'd\n', 'src/app.py': 'a\nc\n'}) # a fix in it + code, out = self.check(base2, fixed, comments=reads[:2] + [self.said('code', fixed)]) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_a_base_edit_to_a_file_the_pull_request_changes_is_no_new_round(self): # #740 round 2, B major 1 + cfg = review_cfg('enforce') + app = ['top', 'mid', '.', '.', ' 1', ' 2', '.', 'end'] + + def tree(*lines): + return base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': '\n'.join(lines) + '\n'}) + base = self.w.commit(tree(*app), [self.m0]) + h1 = self.w.commit(tree(*app[:7], 'END'), [base]) + h2 = self.w.commit(tree(*app[:4], ' 1x', *app[5:7], 'END'), [h1]) # the fix: round 2 + base2 = self.w.commit(tree('new', 'top', 'MID', *app[2:]), [base]) # other lines of the same file + h3 = self.w.merge(h2, base2) # line numbers, `index` blobs and the hunk's function name (MID) all move + reads = [self.said('code', h1), self.said('code r2', h2), self.said('code', h3)] + code, out = self.check(base2, h3, comments=reads) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4; no blocker or major open', out) + fixed = self.w.commit(tree('new', 'top', 'MID', *app[2:4], ' 1y', *app[5:7], 'END'), [h3]) # its own change + code, out = self.check(base2, fixed, comments=reads + [self.said('code', fixed)]) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_a_binary_file_changed_again_is_a_new_round(self): # --binary: its content is in the key + base, _ = self.pair() + cfg = review_cfg('enforce') + h1, h2 = (self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.bin': blob}), [base]) + for blob in ('\0one\n', '\0two\n')) + code, out = self.check(base, h2, comments=[self.said('code', h1), self.said('code', h2)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + + def test_a_listed_login_without_the_override_association_is_warned_about(self): # #740 round 2, B minor 3 + base, head = self.pair(override_authors=['a-collaborator', 'owner']) + report = self.said('code', head, 'blocker: x', assoc='COLLABORATOR', login='a-collaborator') # no override + listed = self.p.override(head, assoc='COLLABORATOR', login='a-collaborator') + code, out = self.check(base, head, comments=[report, listed, self.p.override(OLD[0])]) # owner's: counts + self.assertEqual(code, 1, out) + self.assertIn('warning: the override comment by a-collaborator does not count: GitHub records its author ' + 'association as COLLABORATOR, and an override counts only with OWNER', out) + self.assertEqual(out.count('warning: the override comment'), 1, out) + other = self.p.override(head, assoc='COLLABORATOR', login='someone') # not listed: no warning + self.assertNotIn('warning: the override comment', self.check(base, head, comments=[report, other])[1]) + + def test_a_short_sha_read_still_counts_a_round(self): # old state: a lens that wrote 12 characters + base, head = self.pair() + short = self.said('code', OLD[0][:12]) + code, out = self.check(base, head, comments=[short, self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + + def test_a_strangers_finding_neither_blocks_nor_counts_a_round(self): # anyone can comment on a public repo + base, head = self.pair() + noise = [self.said('code', sha, 'blocker: x', assoc='NONE', login='someone') for sha in OLD] + code, out = self.check(base, head, comments=noise + [self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 1 of 4', out) + + def test_no_lens_comment_yet_is_round_zero(self): # empty state: the path rules' lens is still required + base, head = self.pair() + code, out = self.check(base, head, comments=[]) + self.assertEqual(code, 1, out) + self.assertIn('round 0 of 4; no blocker or major open', out) + self.assertIn('required lens without a read: code', out) + + def test_the_same_comments_give_the_same_verdict(self): # runs again + base, head = self.pair() + comments = [self.said('code', OLD[0], 'major: x'), self.said('code', head, 'major: y')] + self.assertEqual(self.check(base, head, comments=comments), self.check(base, head, comments=comments)) + + def test_no_review_block_judges_no_finding(self): # old state: a base config from before the block + base = self.base(mode=None) + head = self.head(base, mode=None, **{'src/app.py': 'a\n'}) + code, out = self.check(base, head, comments=[self.said('code', head, 'blocker: x')]) + self.assertEqual(code, 0, out) + self.assertNotIn('findings and rounds', out) + + def test_findings_false_judges_no_finding_but_still_counts_rounds(self): + b = self.w.commit(base_files(cfg_raw=review_cfg(findings=False), **{**PROMPTS, **self.STORE}), [self.m0]) + head = self.w.commit(base_files(cfg_raw=review_cfg(findings=False), + **{**PROMPTS, **self.STORE, 'src/app.py': 'a\n'}), [b]) + code, out = self.check(b, head, comments=[self.said('code', head, 'blocker: x')]) + self.assertEqual(code, 0, out) + self.assertIn('findings are not judged', out) + + def test_a_judge_that_fails_is_a_failed_head_in_either_mode(self): # fail closed: never a silent pass + for mode in ('enforce', 'shadow'): + with self.subTest(mode): + base, head = self.pair(mode) + with unittest.mock.patch.object(H.review_judge, 'judge_rounds', side_effect=ValueError('boom')): + code, out = self.check(base, head, comments=[self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('the review rounds and findings cannot be judged, so the head is not passed: ' + 'ValueError: boom', out) + + +class FindingsAndRoundsBare(FindingsAndRounds): + BARE = True + + +class FetchTimeout(unittest.TestCase): # #740 round 2, B minor 4: a hung fetch of a read commit never hangs the check + def test_a_fetch_that_takes_too_long_failed(self): + timeouts = [] + + def run(cmd, **kw): + timeouts.append(kw.get('timeout')) + raise subprocess.TimeoutExpired(cmd, kw.get('timeout')) + with tempfile.TemporaryDirectory() as d, unittest.mock.patch.object(H.subprocess, 'run', run): + self.assertFalse(H.Checkout(d, bare=(A, A)).fetch(A)) + self.assertEqual(timeouts, [H.FETCH_SECONDS]) + + class GitEnvironment(unittest.TestCase): # #442 item 2: pr_queue.GIT_RUN """The gate's git calls get no program or transport of the job's environment where none can be needed: its local calls reach no remote, and its fetch from GitHub (https) needs git's own helpers only. The checkout mode fetches diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index d105dfa056d4..7ff83d13675d 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -61,26 +61,28 @@ does not match itself as a pattern, and the queue then refuses to start. The optional `review` block of the config says which changed paths need which lens reads and caps the review rounds. The head gate (`../head-check`) judges -the required lenses by changed path with it (below, "How the gate judges"); the -round cap, the approach lens and the `findings` / `state_line` lines are not -enforced yet. `review.recommended.json` (next to this file) is a complete -example with made-up paths; a test parses it. Real paths, lens names and prompt -files belong in the project's own config. - -Every key is required and an unknown key is an error, as in the rest of the -config. A wrong block makes the whole config invalid, naming every problem at +the required lenses by changed path with it (below, "How the gate judges"), and +the findings, the approach lens and the round cap from the lens comments +(below, "Findings and rounds"). The `state_line` key is parsed but not enforced +yet. `review.recommended.json` (next to this file) is a complete example with +made-up paths; a test parses it. Real paths, lens names and prompt files belong +in the project's own config (below, "Turning it on"). + +Every key is required except `override_authors`, and an unknown key is an +error, as in the rest of the config. A wrong block makes the whole config invalid, naming every problem at once. -| Key | Meaning | -| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `version` | `1`. Any other version is refused by name (a newer one: update the pinned release first). | -| `mode` | `shadow`: the gate keeps judging by its `required-lenses` input and only reports what the path rules would have said. `enforce`: the path rules decide. | -| `rules` | An ordered list of `{"name", "weight", "paths", "lenses"}`. A changed file belongs to the first rule with a matching glob. `weight` is `critical`, `normal` or `light`; `lenses` are names from `lenses`. | -| `lenses` | `{"": {"prompt": ""}}`: every lens a rule or `rounds` names, with its prompt file (repo-relative, no `..`, no leading `/`, no glob characters). | -| `min_lenses` | `{"critical", "normal", "light"}`: the fewest different lenses a rule of that weight must list, 0-10, never falling with weight. | -| `rounds` | `{"approach_at", "max", "approach_lens"}`: from round `approach_at` the approach lens is required too; after round `max` a person must write an override comment or close the pull request. `max` is not below `approach_at`. | -| `findings` | `true` or `false`: lens comments carry `blocker:`, `major:` and `minor:` lines, and a blocker or major at the head fails the gate. | -| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. | +| Key | Meaning | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `version` | `1`. Any other version is refused by name (a newer one: update the pinned release first). | +| `mode` | `shadow`: the gate keeps judging by its `required-lenses` input and only reports what the path rules would have said. `enforce`: the path rules decide. | +| `rules` | An ordered list of `{"name", "weight", "paths", "lenses"}`. A changed file belongs to the first rule with a matching glob. `weight` is `critical`, `normal` or `light`; `lenses` are names from `lenses`. | +| `lenses` | `{"": {"prompt": ""}}`: every lens a rule or `rounds` names, with its prompt file (repo-relative, no `..`, no leading `/`, no glob characters). | +| `min_lenses` | `{"critical", "normal", "light"}`: the fewest different lenses a rule of that weight must list, 0-10, never falling with weight. | +| `rounds` | `{"approach_at", "max", "approach_lens"}`: from round `approach_at` the approach lens is required too; after round `max` a person must write an override comment or close the pull request. `max` is not below `approach_at`. | +| `findings` | `true` or `false`: lens comments carry a findings section (`blocker:` and `major:` lines, or headings), and a blocker or major at the head fails the gate. | +| `override_authors` | Optional, default `[]`: the GitHub logins whose override comment is a person's decision (clears a finding, passes the round cap). `[]`: no override is a person's, so none does that (issue #78). | +| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. Not enforced yet. | The config is refused when: @@ -188,6 +190,98 @@ It judges the merge with the base of its run: two pull requests that each pass alone can still merge into a bad config unless the ruleset requires branches to be up to date, which the queue needs anyway. +**Findings and rounds.** The gate also judges the review itself, from the lens +comments it reads anyway (section "Lens comments"; `review_judge.judge_rounds`). +Owner decisions: issue #99 (findings), issue #100 (rounds). + +- **Findings** (`findings: true`): a lens report has a findings section in its + body: a `blocker:` and a `major:` line (`blocker: none`, `major: none` when it + found nothing), or headings `### Blocking` / `### Major` with the findings, + or `None.`, under them. What counts as a finding (`review_judge.read_findings`, + table in `review_judge.test.py`): a label `blocker`, `blocking` or `major` + (plural, `Major findings`, `major 2`, `major (state)` too; bold, list, emoji + and heading marks do not matter) followed by text on the same line + (`major: x`, `**Blocking:** x`, `major — x`, `[major] x`, a table row + `| major | x |` or a table column headed `major`), or, for a bare label, the + first line under it (a heading or nothing under it is a finding too). Every + label of a line counts (`blocker: none, major: x`). Only an explicit nothing + is no finding (`none`, `None found`, `none (only minors)`, `(none)`, + `none — two minors below`, `nothing`, `no`, `0`, `n/a`, `-`). A recap of an + earlier round with a label (`Major 1: fixed`) is read as an open finding: + write it without one (`round 1, fixed: x`). The format of one line per + finding, `finding: ...`, is read too (with + `findings: none` when there is none); only its minors pass. Before reading, + the text is folded: NFKC, zero-width and other format characters removed, + Cyrillic and Greek look-alike letters read as Latin, `0` for `o` in a label, + HTML entities decoded (`major`) and backslash escapes dropped + (`major\:`). Lines inside a fenced code block are never labels, so quote + code and example findings in one; quotes and indented blocks are read (fail + closed). A title line or `verdict:` line that says + NOT SAFE is a blocker. `minor:` and `nit:` never block. + A lens whose newest report has a blocker or major fails the head. A report + without a findings section cannot be judged and fails that lens: it needs a + blocker label and a major label (`blocker: none` and `major: none` too), or + `findings: none`, or `finding:` lines. One of the two labels is not enough: + `blocker: none` with the major written another way (`F1 (major): x`) would + read as clean. A report that writes `major: none` and then a major in a form + not read here contradicts itself, and is read by its `major: none`. Findings + go in the report body: findings in inline review comments (on a line of the + diff) are not read, so a lens posted as a PR review whose body has no + findings section fails. The ways out: fix it, and the lens + posts a new report on the fixed head without a blocker or major; or a person + posts an override comment for the head (config `override`) newer than the + report. A person is a login listed in `override_authors` whose override + comment GitHub also records with the override association + (`override.author_association`); an override by a listed login with another + association does not count, and the check output warns about it. An override by any + other author of the override association covers a lens's read as before, but + clears no finding and passes no cap. While agents still post as the owner's + login (issue #78) an override from it proves no person, so keep the list + empty until they no longer do: then no override clears a finding, and a + finding is fixed or the pull request closed. +- **Rounds:** a round is a lens read of a fixed change, not of a new sha. The + gate counts the different own changes the counted lens headers read: for each + read sha, `git diff -U0 --binary` from its merge base with the base to it, + the hunk headers (line numbers, function names) and `index` lines dropped + (`head_check.own_change`), so a base that edits other lines of a file the + pull request changes moves nothing. A head that only merges the base in (a + queue update, `gh pr update-branch`) or the same change rebased onto a newer + base is the same change, so the same round; a merge that also changes the pull + request's own diff is a new one. A short sha counts as well, and a sha the + gate cannot fetch, or one with several merge bases, counts as a round of its + own (unknown counts more, never fewer). A round marker in a lens name + (`code r3`, also `**r3**`) counts as that round when it is higher. The count + comes from the lens comments the gate reads now: hidden and edited ones count + too, so hiding or editing old rounds does not lower it. A deleted comment is + gone from GitHub, and only lens authors can delete one, so the round marker of + a later header is what keeps it counted; counting from commits instead would + count every push, the opposite of a round. From round `rounds.approach_at` + the lens `rounds.approach_lens` is required as well. Round `rounds.max` is the + last: a round past it fails the head with "no more rounds; a person decides": + the pull request is closed, or a person's override comment for the head, + newer than every lens comment, lets it through on its own (it also waives the + approach lens). With `override_authors` empty no override does: the reason + says so. At round `rounds.max` a blocker or major says the same. +- In `shadow` the summary adds a row "findings and rounds (shadow): they would + have said: ..." and nothing fails. Lens comments that cannot be judged fail + the head in both modes. Without the `review` key nothing of this runs. + +**Workflow, queue and lens files are critical** (owner decision, issue #82). +The built-in protected paths (above) make every change to `.github/workflows/`, +to the config itself and to a lens prompt need the critical lenses: the extra +review. A project whose gate or queue code lives in its own repository (the +fork's `fork-ci/`) adds a critical rule for those paths. + +**Turning it on.** A project opts in through its own config, never through +the fork: add the `review` block to the queue config its head gate reads +(`config` input), with its real paths, lens names and prompt files, in +`shadow` first. Read a few gate summaries, fix every config warning, then set +`mode` to `enforce` and dispatch the backfill (head-check README). The fork's +own config is [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), +in `shadow`, without `override_authors` (no person yet) until issue #78 is done; its lens +prompts (`.github/fork-lens-*.md`) tell each lens the exact header and findings +lines, so a switch to `enforce` does not fail every pull request. + **The empty input.** The gate refuses an empty `required-lenses` input (a head with no lens comment would pass), except when the config at the base tip has a `review` block in `enforce`: then the path rules name the lenses. A name that @@ -203,7 +297,8 @@ posts a failure that says why, in `shadow` as in `enforce`. Never a pass, never **Adding the block.** A release that does not know the `review` key refuses the whole config (an unknown key), so: release first, then pin that release in every caller of the queue and the gate, then add the key. Roll back in the reverse order: -remove the key, then the pin. +remove the key, then the pin. The same holds for `override_authors`: a +release from before it refuses a config that has it. ## Lens comments diff --git a/fork-ci/actions/pr-queue/lens.py b/fork-ci/actions/pr-queue/lens.py index f4c8671fea07..1a1fd5200b69 100644 --- a/fork-ci/actions/pr-queue/lens.py +++ b/fork-ci/actions/pr-queue/lens.py @@ -437,16 +437,23 @@ def lens_posts(self, pr): return [(when, i, post) for i, (when, post) in enumerate(posts) if not self.is_bot(post)] def lens_entries(self, pr): - """{lens key: ((when, index), sha or None, why or None)}: each lens's NEWEST header, from the posts that count - (counts). A comment that looks like a lens report but is not one valid read (lens_problem) is an entry of its - own, keyed by the comment, with no sha: a correct new post of that lens does not replace it, and hiding it - does not clear it. It reads nothing until it is deleted, its first line is edited to `not a lens: ...` with no - lens header line left in it, or a newer override covers it (NO_HEADER_WAY_OUT). A lens comment edited after - posting (includesCreatedEdit not false: GitHub records that it changed, not what) or hidden (minimized) reads - nothing until that lens posts a new comment; so does a review whose flags were not read (FLAGS_UNREAD). A - lens-looking post that does not count is logged (log only, never a comment), so a wrong lens_author_associations - shows in the run.""" - found = {} + """{lens key: ((when, index), sha or None, why or None)}: lens_scan's entries without their posts.""" + return {k: v[:3] for k, v in self.lens_scan(pr)[0].items()} + + def lens_scan(self, pr): + """(entries, history). entries: {lens key: ((when, index), sha or None, why or None, post)}: each lens's NEWEST + header, from the posts that count (counts). A comment that looks like a lens report but is not one valid read + (lens_problem) is an entry of its own, keyed by the comment, with no sha: a correct new post of that lens does + not replace it, and hiding it does not clear it. It reads nothing until it is deleted, its first line is edited + to `not a lens: ...` with no lens header line left in it, or a newer override covers it (NO_HEADER_WAY_OUT). A + lens comment edited after posting (includesCreatedEdit not false: GitHub records that it changed, not what) or + hidden (minimized) reads nothing until that lens posts a new comment; so does a review whose flags were not read + (FLAGS_UNREAD). A lens-looking post that does not count is logged (log only, never a comment), so a wrong + lens_author_associations shows in the run. review_judge.judge_rounds reads the newest report's findings from the + entry's post. history: [(lens name, sha or None)] of EVERY counted post with a valid lens header, in list order, + older rounds and edited or hidden ones too (review_judge.judge_rounds counts the rounds by it: hiding an old + round must not lower the count).""" + found, history = {}, [] for when, i, post in self.lens_posts(pr): where = post.get('url') or f'by {(post.get("author") or {}).get("login") or "?"} at {when or "?"}' if not self.counts(post): # the author first: a stranger's text is only ever read in part, for the log @@ -462,16 +469,17 @@ def lens_entries(self, pr): header = None if problem else lens_header(post.get('body') or '') if problem: hidden = ', hidden' if self.hidden(post) else '' - found[f'{NO_HEADER} ({where}, {problem}{hidden})'] = ((when, i), None, None) + found[f'{NO_HEADER} ({where}, {problem}{hidden})'] = ((when, i), None, None, post) continue if not header: continue + history.append(header) key = lens_key(header[0]) if key not in found or (when, i) >= found[key][0]: why = FLAGS_UNREAD if post.get(UNREAD) else HIDDEN if self.hidden(post) \ else EDITED if post.get('includesCreatedEdit') is not False else None - found[key] = ((when, i), None, why) if why else ((when, i), header[1], None) - return found + found[key] = ((when, i), None, why, post) if why else ((when, i), header[1], None, post) + return found, history def lens_reads(self, pr): """{lens key: (sha, gap, override)}: what each lens of lens_entries counts as reading, the one rule reviewed_head diff --git a/fork-ci/actions/pr-queue/review_config.py b/fork-ci/actions/pr-queue/review_config.py index 24612706f48a..801f3bec5ecc 100644 --- a/fork-ci/actions/pr-queue/review_config.py +++ b/fork-ci/actions/pr-queue/review_config.py @@ -28,6 +28,12 @@ WEIGHTS = ('critical', 'normal', 'light') # heaviest first CATCH_ALL = '**' KEYS = ('version', 'mode', 'rules', 'lenses', 'min_lenses', 'rounds', 'findings', 'state_line') +# Optional keys, with what leaving them out means. override_authors: the GitHub logins whose override comment counts as a +# person's decision (a finding cleared, the round cap passed). Left out or empty: no override is a person's (issue #78: +# while agents post as the owner's login, an override from it proves no person). +OPTIONAL = ('override_authors',) +LOGIN_RE = re.compile(r'[A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38}') +MAX_OVERRIDE_AUTHORS = 20 RULE_KEYS = ('name', 'weight', 'paths', 'lenses') ROUND_KEYS = ('approach_at', 'max', 'approach_lens') MAX_LENSES = 10 @@ -83,6 +89,7 @@ class ReviewRules: warnings: tuple = () # The rule-order problems themselves, in either mode (enforce_order_problems): empty in a parsed enforce block. order: tuple = () + override_authors: tuple = () # GitHub logins as written, each once (case-insensitive); empty: no person's override def lens(self, name): return next((x for x in self.lenses if x.name == name), None) @@ -447,7 +454,8 @@ def parse_review(obj, where='review'): newer = _int(obj['version'], VERSION + 1, 2 ** 31) raise ConfigError(f'{where}: version {_show(obj["version"])} is not {VERSION}, the only version this release ' f'reads' + (': update the pinned release first' if newer else '')) - problems.extend(f'unknown key {k!r} (known: {", ".join(KEYS)})' for k in obj if k not in KEYS) + problems.extend(f'unknown key {k!r} (known: {", ".join(KEYS + OPTIONAL)})' for k in obj + if k not in KEYS + OPTIONAL) problems.extend(f'{k} is missing' for k in KEYS if k not in obj) if 'mode' in obj and obj['mode'] not in MODES: problems.append(f'mode is {_show(obj["mode"])}: must be one of {", ".join(MODES)}') @@ -461,6 +469,7 @@ def parse_review(obj, where='review'): mins = _min_lenses(obj['min_lenses'], problems) if 'min_lenses' in obj else None rules = _rules(obj['rules'], declared, mins, problems) if 'rules' in obj else () rounds = _rounds(obj['rounds'], declared, problems) if 'rounds' in obj else None + authors = _override_authors(obj.get('override_authors', []), problems) # Rule order: refused in enforce (the rules decide), a warning in shadow (they only say what they would have said). order = _shadowed(rules) warnings = tuple(f'{w}; {SWITCH}' for w in order) if obj.get('mode') == 'shadow' else () @@ -469,4 +478,24 @@ def parse_review(obj, where='review'): if problems: raise ConfigError(f'{where}: ' + '; '.join(problems)) return ReviewRules(obj['version'], obj['mode'], rules, tuple(Lens(n, p) for n, p in lenses.items()), mins, - rounds, obj['findings'], obj['state_line'], warnings, tuple(order)) + rounds, obj['findings'], obj['state_line'], warnings, tuple(order), authors) + + +def _override_authors(raw, problems): + """The override_authors list: GitHub logins, each once (case-insensitive), at most MAX_OVERRIDE_AUTHORS.""" + if not isinstance(raw, list): + problems.append(f'override_authors is {_show(raw)}: must be a list of GitHub logins ([] = no override is a ' + f'person\'s)') + return () + if len(raw) > MAX_OVERRIDE_AUTHORS: + problems.append(f'override_authors has {len(raw)} logins: at most {MAX_OVERRIDE_AUTHORS}') + seen, out = set(), [] + for x in raw: + if not isinstance(x, str) or not LOGIN_RE.fullmatch(x): + problems.append(f'override_authors: {_show(x)} is not a GitHub login') + elif x.lower() in seen: + problems.append(f'override_authors: {x!r} is listed twice') + else: + seen.add(x.lower()) + out.append(x) + return tuple(out) diff --git a/fork-ci/actions/pr-queue/review_judge.py b/fork-ci/actions/pr-queue/review_judge.py index 6b2600bed1e5..10e8e7927329 100644 --- a/fork-ci/actions/pr-queue/review_judge.py +++ b/fork-ci/actions/pr-queue/review_judge.py @@ -16,9 +16,29 @@ Unknown never judges: a path no rule matches, a protected path with no critical rule (the parser refuses both, so only a hand-built rule set gets here) or a diff line this module cannot read raises JudgeError, which the gate turns into a failure. + +judge_rounds judges the review itself, from the lens comments (README.md, "Review rules", Findings and rounds): + - Findings (`findings: true`): a lens whose newest entry has a blocker or major finding (read_findings: labels, + headings, bold, tables, folded Unicode; never inside a fenced code block) fails the head, until that lens posts a + new report on a fixed head without one (a later round clears it) or a person's override newer than the report + covers it. A person is a login listed in `override_authors`; none listed (the default) means no override clears + anything. A report with no findings section (a blocker label and a major label, `none` allowed) cannot be judged + and fails that lens. + Owner decision: issue #99. + - Rounds: a round is a lens read of a different change, not of a different sha: review_round counts the different + own changes (head_check.own_change: the diff from the merge base, so a head that only merges the base in is the + same round) the lens headers name, old, edited and hidden ones too, or the highest round marker (`r3`), whichever + is higher. From round `rounds.approach_at` the approach lens is required; a round past `rounds.max` fails the head + until a person's override for the head, newer than every lens comment, lets it through (it alone is enough: it + also waives the approach lens). Owner decision: issue #100. +Pure as judge is: the lens comments come in (lens.Review.lens_scan), the verdict goes out. """ +import html +import re +import unicodedata from dataclasses import dataclass +import lens import review_config import review_glob @@ -152,3 +172,293 @@ def judge(review, changes, config_path): lenses = tuple(dict.fromkeys(n for r in review.rules if r.weight == weight and any(h.rule == r.name for h in hits) for n in r.lenses)) return Judgement(weight, lenses, hits, tuple(protected), len(paths)) + + +# --- findings (issue #99) ----------------------------------------------------------------------------------------- +# What a lens report writes, as the fork's reports are written (README.md, "Findings and rounds"; the table in +# review_judge.test.py, Findings): a label `blocker`, `blocking`, `major` (also plural, `Major findings`, `major 2`, +# `major (state)`), then +# - a separator and the finding on the same line: `major: x`, `**Blocking:** x`, `major — x`, `major - x`, `[major] x`, +# `🔴 Major: x`; or a table row `| major | x |` (or a table whose header row names the label: its column); +# - nothing on the line (`### Major`, `**Blocking**`, `Major:`): the first line under it is the finding. +# Text that says nothing (NONE_RE: `none`, `None found.`, `none (only minors)`, `n/a`, `-` ...) is no finding, and every +# label of a line is judged (`Blocker: none, major: none`). `minor` and `nit` labels never block; they only end the +# text of the label before them. Lines inside a fenced code block hold no label (a report quotes code and test vectors +# there); the line under a bare label is its content even when it opens a fence. A report has a findings section only +# when it has a blocker label AND a major label (`blocker: none` and `major: none` too), or `findings: none`, or +# `finding: ...` lines: with one of the two labels only, a major written in a form not read here +# (`F1 (major): x`) would pass as clean, so such a report cannot be judged (judge_rounds fails it). Each finding is one +# line `finding: ...` in that format (FINDING_LINE_RE). A recap of an earlier round with a label +# (`Major 1: fixed`) is read as a finding like any other; write it without one (`round 1, fixed: x`). +# Before any of this the text is folded (fold): NFKC (`Major` -> Major), format characters removed (zero-width space and +# joiner, soft hyphen, bidi marks: category Cf), and Cyrillic and Greek letters that look like Latin ones read as Latin +# (`majоr` with a Cyrillic о); a 0 for an o in a label is read too (`maj0r`); HTML entities decoded (`major`) and +# Markdown backslash escapes dropped (`major\: x`), as GitHub renders them. +CONFUSABLES = str.maketrans({ + 'А': 'A', 'В': 'B', 'Е': 'E', 'К': 'K', 'М': 'M', 'Н': 'H', 'О': 'O', 'Р': 'P', 'С': 'C', 'Т': 'T', 'Х': 'X', + 'І': 'I', 'Ј': 'J', 'Ѕ': 'S', 'а': 'a', 'е': 'e', 'о': 'o', 'р': 'p', 'с': 'c', 'х': 'x', 'у': 'y', 'і': 'i', + 'ј': 'j', 'ѕ': 's', 'к': 'k', 'ԁ': 'd', 'ӏ': 'l', 'һ': 'h', 'ԛ': 'q', 'ԝ': 'w', + 'Α': 'A', 'Β': 'B', 'Ε': 'E', 'Ζ': 'Z', 'Η': 'H', 'Ι': 'I', 'Κ': 'K', 'Μ': 'M', 'Ν': 'N', 'Ο': 'O', 'Ρ': 'P', + 'Τ': 'T', 'Υ': 'Y', 'Χ': 'X', 'ο': 'o', 'ι': 'i', 'κ': 'k', 'ν': 'v', 'α': 'a', 'ρ': 'p', + 'ı': 'i', 'ℓ': 'l', 'ǀ': 'l', '∶': ':', +}) +_KW = r'(?Pbl[o0]ck(?:ers?|ing)|ma[j][o0]rs?|m[i1]n[o0]rs?|nits?)' +_LABEL = (_KW + r'(?![^\W_])(?:\s+(?:findings?|issues?|problems?|items?|points?|bugs?|ones?)(?![^\W_]))?' + r'(?:\s*#?\d{1,4}(?![^\W_]))?(?:\s*\([^()\n]{0,80}\))?') +_SEP = r'\s*(?::|[—–=]|-(?=\s|$))' +# A label that starts a part of a line, with a separator after it (`Major version: 2` is no label) or in brackets +# (`[major] x`): at the line start, or after punctuation, `and` or `but` (`blocker: none, major: x`). +LABEL_RE = re.compile(r'(?:^|(?<=[,;.!?|/])\s*|\s+(?:and|but)\s+)(?:\[' + _LABEL + r'\]|\(' + _LABEL.replace('kw', 'kw2') + + r'\)|' + _LABEL.replace('kw', 'kw3') + _SEP + ')', re.I) +BARE_RE = re.compile(r'[\[(]?' + _LABEL + r'[\])]?' + _SEP + r'?\s*', re.I) +CELL_RE = re.compile(_LABEL + r'\s*:?\s*', re.I) +NONE_RE = re.compile( + r'[(\[]?\s*(?:none|nothing|nil|no|zero|0|n/?a|-+|—|–)' + r'(?:\s+(?:found|open|left|remaining|seen|here|known|so\s+far|at\s+all|this\s+round|blocking|blockers?|majors?|' + r'findings?|issues?|problems?))*\s*(?:$|[.,;:!?)(\[]|[^\w\s\x00-\x7f]|-(?=\s|$))', re.I) +FINDING_LINE_RE = re.compile(r'findings?\s*:\s*[\[(]?' + _KW + r'(?![^\W_])', re.I) # `finding: major x` +FINDINGS_NONE_RE = re.compile(r'(?:findings|issues|problems)\s*:\s*(?:none|nothing|no|0|n/?a)\s*\.?\s*$', re.I) +NOT_SAFE_RE = re.compile(r'\bnot\s+safe\b', re.I) +VERDICT_NOT_SAFE_RE = re.compile(r'verdict\s*[:—–-]?\s*not\s+safe\b', re.I) +RULE_LINE_RE = re.compile(r'\s*(?:[-*_]\s*){3,}') # a thematic break: no content +HEADING_RE = re.compile(r' {0,3}#{1,6}(?:\s|$)') +TABLE_RE = re.compile(r'\s*\|') +TABLE_RULE_RE = re.compile(r'\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?') # on a stripped line: linear time +ROUND_RE = re.compile(r'r(\d{1,9})') +SHOW_FINDING = 120 # characters of a finding line named in the reason +BLOCKING = ('blocker', 'major') + + +def fold(text): + """The text as the findings rules read it: NFKC, format characters (category Cf) gone, look-alike letters Latin.""" + text = re.sub(r'\\([!-/:-@\[-`{-~])', r'\1', html.unescape(unicodedata.normalize('NFKC', text or ''))) + return ''.join(ch for ch in text if unicodedata.category(ch) != 'Cf').translate(CONFUSABLES) + + +def _kind(kw): + kw = kw.lower().replace('0', 'o').replace('1', 'i') + return 'blocker' if kw.startswith('block') else 'major' if kw.startswith('major') else 'minor' + + +def _plain(line, narrow): + """One line normalised (lens.norm_line), folded, with leading symbols (emoji, arrows, bullets) gone and each run of + whitespace one space (the label patterns then take linear time on a long run of spaces).""" + return re.sub(r'\s+', ' ', re.sub(r'^[^\w\[(]+', '', lens.norm_line(fold(line), narrow))).strip() + + +def _says_nothing(text): + return bool(NONE_RE.match(re.sub(r'^[^\w\-—–(\[]+', '', text.strip()))) + + +def _cells(raw): + return [c.strip() for c in fold(raw).strip().strip('|').split('|')] + + +def _read(body, narrow): + """(findings [(line index, kind, text)], the set of label kinds seen), one way of reading.""" + lines = (body or '').splitlines() + out, seen, fence, columns = [], set(), None, None + for i, raw in enumerate(lines): + fence, fenced = lens.fence_step(fence, raw) + if fenced: + columns = None + continue + if TABLE_RE.match(raw): + cells = [_plain(c, narrow) for c in _cells(raw)] + if i + 1 < len(lines) and TABLE_RULE_RE.fullmatch(lines[i + 1].strip()): # a header row: its label columns + columns = {j: _kind(m.group('kw')) for j, c in enumerate(cells) for m in [CELL_RE.fullmatch(c)] if m} + seen |= set(columns.values()) + continue + if TABLE_RULE_RE.fullmatch(raw.strip()): + continue + for j, c in enumerate(cells): # a label cell: the next cell says what it found + m = CELL_RE.fullmatch(c) + if m and _kind(m.group('kw')) in BLOCKING: + seen.add(_kind(m.group('kw'))) + said = next((x for x in cells[j + 1:] if x), '') + if not _says_nothing(said): + out.append((i, _kind(m.group('kw')), _plain(raw, narrow))) + for j, kind in (columns or {}).items(): + if kind in BLOCKING and j < len(cells): + if not _says_nothing(cells[j]): + out.append((i, kind, _plain(raw, narrow))) + continue + columns = None + line = _plain(raw, narrow) + if FINDINGS_NONE_RE.match(line): + seen |= set(BLOCKING) + continue + m = FINDING_LINE_RE.match(line) + if m: # `finding: major x`: that format's section, and a finding of its kind + seen |= set(BLOCKING) + if _kind(m.group('kw')) in BLOCKING: + out.append((i, _kind(m.group('kw')), line)) + continue + if BARE_RE.fullmatch(line): # `### Major`: the first line under it says what it found + kind = _kind(BARE_RE.fullmatch(line).group('kw')) + if kind not in BLOCKING: + continue + seen.add(kind) + under = _under(lines, i + 1, narrow) + if under is None or not _says_nothing(under): + out.append((i, kind, f'{line} (nothing under it)' if under is None else f'{line} {under}')) + continue + marks = list(LABEL_RE.finditer(line)) + for n, m in enumerate(marks): + kind = _kind(m.group('kw') or m.group('kw2') or m.group('kw3')) + if kind not in BLOCKING: + continue + seen.add(kind) + said = line[m.end():marks[n + 1].start() if n + 1 < len(marks) else len(line)] + if not _says_nothing(said): + out.append((i, kind, line)) + break + return out, seen + + +def _under(lines, start, narrow): + """The first content line after a bare label, normalised, or None when the next one is a heading, another label or + the end (nothing under it). Blank lines and thematic breaks are skipped; a fence line is content.""" + for raw in lines[start:]: + if not raw.strip() or RULE_LINE_RE.fullmatch(raw): + continue + if HEADING_RE.match(raw): + return None + line = _plain(raw, narrow) + if BARE_RE.fullmatch(line) or LABEL_RE.match(line): + return None + return line or raw.strip() + return None + + +def read_findings(body): + """(findings, has_section). findings: [(kind, line)] of the report's blocker and major findings in line order, kind + 'blocker' or 'major', line as normalised (cut to SHOW_FINDING). Read both ways lens reads a line (clean_inline + narrow or not): a line either way reads as a finding is one. has_section: the report has a blocker label and a major + label (`blocker: none` and `major: none` too), or `findings: none`, or `finding: ` lines. A first line (the + lens header, or the title line under it) or a `verdict:` line that says NOT SAFE is a blocker whatever the labels + say.""" + found = {} + seen = set() + for narrow in (False, True): + out, saw = _read(body, narrow) + seen |= saw + for i, kind, text in out: + found.setdefault(i, (kind, text[:SHOW_FINDING])) + lines = (body or '').splitlines() + top = [i for i, line in enumerate(lines) if line.strip()][:2] + first = set(top if lens.lens_header(body) else top[:1]) # the line after a lens header is a title too + fence = None + for i, raw in enumerate(lines): + fence, fenced = lens.fence_step(fence, raw) + if fenced or i in found: + continue + line = _plain(raw, False) + if (i in first and NOT_SAFE_RE.search(line)) or VERDICT_NOT_SAFE_RE.match(line): + found[i] = ('blocker', f'the report says NOT SAFE: {line}'[:SHOW_FINDING]) + return [found[i] for i in sorted(found)], seen >= set(BLOCKING) + + +def findings(body): + """[(kind, line)] of the report's blocker and major findings (read_findings).""" + return read_findings(body)[0] + + +def round_marker(name): + """The highest round marker (`r3`) among the words of a lens name after its first, or 0. Markdown emphasis or + backticks around the marker are stripped first (`data **r3**` is round 3), and the name is folded (`r3`).""" + words = [w.strip('*_`') for w in fold(name).lower().split()[1:]] + return max((int(m.group(1)) for w in words for m in [ROUND_RE.fullmatch(w)] if m), default=0) + + +def review_round(history, change_of=None): + """The fix-and-review round the lens headers say: how many different changes they read, or the highest round + marker, whichever is higher. history: [(lens name, sha or None)] of every lens header that counts + (lens.Review.lens_scan), old, edited and hidden ones too, a short sha too. change_of(sha): what the pull request + changes at that commit (head_check.own_change: the diff from its merge base with the base, so a commit that only + merges the base in, or another lens's read of the same change, is the same round); without it each different sha + is one. 0: no lens header yet.""" + change_of = change_of or (lambda sha: sha) + changes = {change_of(sha.lower()) for _, sha in history if sha} + return max([len(changes)] + [round_marker(name) for name, _ in history]) + + +# How judge_rounds reads each lens's newest entry (head_check.review_rounds says which one it is): +REPORT = 'report' # a read that counts: its findings, and it needs a findings section +LOOSE = 'loose' # a comment that looks like a report but reads nothing, covered by an override that is no +# person's: its finding lines block, it needs no section (it may be a fix-round note) +UNREAD = 'unread' # a lens report that reads nothing (edited, hidden, a short sha), covered by such an override: +# its findings cannot be read, so it blocks +CLEARED = 'cleared' # a person's override is newer: nothing of it is judged +NOT_JUDGED = 'skip' # it reads nothing and nothing covers it: the head fails for it already + + +@dataclass(frozen=True) +class Rounds: + round: int # review_round of the lens headers + approach: bool # the approach lens is required (round >= rounds.approach_at and no person decided) + over_cap: bool # round > rounds.max and no person decided + findings: tuple # (lens key, kind, line) of every blocker or major no later round or person cleared + problems: tuple # why the head fails, plain text; empty: findings and rounds pass + + def text(self, review): + """One line for the check summary.""" + cap = review.rounds.max + said = f'round {self.round} of {cap}' + if self.approach: + said += f', approach lens {review.rounds.approach_lens} required from round {review.rounds.approach_at}' + if not review.findings: + said += '; findings are not judged (findings: false)' + return said + ('; ' + '; '.join(self.problems) if self.problems else '; no blocker or major open') + + +def person_way(review, prefix='override for head'): + """How a person lets a head through, as the reasons say it: who may, and what they write.""" + if not review.override_authors: + return ('a person\'s override comment, but no one is listed in review.override_authors yet, so no override ' + 'is accepted (issue #78: until agents no longer post as the person\'s login, an override is not proof ' + 'of a person)') + return (f'an override comment `{prefix} : ` for this head by ' + f'{" or ".join(review.override_authors)}, newer than every lens comment') + + +def judge_rounds(review, history, reports, decided=False, change_of=None, prefix='override for head'): + """Rounds of a pull request's lens comments under the parsed `review` rules. history: [(lens name, sha)] of every + lens header that counts (lens.Review.lens_scan); reports: {lens key: (how, body)} of each lens's newest entry, how + one of REPORT, LOOSE, UNREAD, CLEARED, NOT_JUDGED (head_check.review_rounds); decided: a person's override (one of + review.override_authors) for this head is newer than every lens comment; change_of: review_round's. prefix: the + config's override prefix, for the reasons.""" + n, cap = review_round(history, change_of), review.rounds.max + over = n > cap and not decided + way = person_way(review, prefix) + problems = [] + if over: + problems.append(f'review round {n} is past the cap of {cap} rounds (issue #100): no more rounds; a person ' + f'decides: close the pull request, or {way}. A person\'s override alone is enough, also for a ' + f'required approach lens that never posted') + found = [] + if review.findings: + for key in sorted(reports): + how, body = reports[key] + if how in (CLEARED, NOT_JUDGED): + continue + if how == UNREAD: + problems.append(f'lens {key}: its newest report reads nothing and only an override that is no ' + f'person\'s covers it, so its findings cannot be read (issue #99): that lens posts a ' + f'new report, or {way}') + continue + said, section = read_findings(body) + for kind, line in said: + found.append((key, kind, line)) + if how == REPORT and not said and not section: + problems.append(f'lens {key}: its newest report has no findings section, so it cannot be judged ' + f'(issue #99): write `blocker: none` and `major: none`, or the findings, as lines of ' + f'the report body (findings in inline review comments are not read)') + for key, kind, line in found: + again = (f'this is round {n} of {cap}: no more rounds; a person decides: close the pull request, or {way}' + if n >= cap else + f'fix it, and lens {key} posts a new report on the fixed head with no blocker or major ' + f'(`major: none`); or {way}') + problems.append(f'lens {key} reported a {kind} that no later round cleared (issue #99): {line!r}; {again}') + approach = n >= review.rounds.approach_at and not decided + return Rounds(n, approach, over, tuple(found), tuple(problems)) diff --git a/fork-ci/actions/pr-queue/review_judge.test.py b/fork-ci/actions/pr-queue/review_judge.test.py index d154b1a5d433..3ac1b9212d6b 100644 --- a/fork-ci/actions/pr-queue/review_judge.test.py +++ b/fork-ci/actions/pr-queue/review_judge.test.py @@ -4,7 +4,7 @@ are the contract both sides test; the rest here: reading git's name-status output, and that a rule set the parser would refuse, or a diff line nobody knows, never judges. """ -import dataclasses, json, os, sys, unittest +import dataclasses, json, os, sys, time, unittest sys.dont_write_bytecode = True # no __pycache__ next to the scripts @@ -163,5 +163,344 @@ def test_the_parser_refuses_those_rule_sets_first(self): rc.parse_review({**block, 'rules': rules}) +# --- findings and rounds (issues #99, #100): judge_rounds on lens comments --------------------------------------- +H1, H2, H3, H4, H5 = (c * 40 for c in 'abcde') +BASIC = CONFIGS['basic'] # findings true, rounds approach_at 3, max 4, approach lens `approach`, no override_authors +PEOPLE = dataclasses.replace(BASIC, override_authors=('a-person',)) +HEAD = 'lens: code · reviewer: x · read at: ' + H1 + '\n' +NONE = 'blocker: none\nmajor: none' # the clean findings section the lens prompts ask for + + +def rounds(history, reports, decided=False, review=BASIC, change_of=None): + """judge_rounds with each lens's newest entry a report that counts, unless given as (how, body).""" + reports = {k: v if isinstance(v, tuple) else (rj.REPORT, v) for k, v in reports.items()} + return rj.judge_rounds(review, list(history), reports, decided, change_of) + + +# Findings as the fork's lens reports write them (pull requests #720-#739) and as reviews A and B of #740 showed they +# can be written: each must count as a blocker or major finding. +FINDING = [ + ('heading, numbered item', '### Blocking\n\n1. **The lock is gone.** Two writers race on the row.'), + ('heading, bold item', '### Major\n**B1. The retry loops forever.**'), + ('heading with a note', '### Blocking (merge, not safety)\n1. The queue merges an unread head.'), + ('heading with a long note', '### Major (older than this PR; a follow-up issue is fine)\n- the cache never expires'), + ('bold inline', '**Blocking:** the lock is gone.'), + ('bold bare, item under it', '**Major**\n- two writers race'), + ('plain', 'Blocking: x'), + ('findings word', 'Major findings: x'), + ('findings word, list under it', 'Major findings:\n1. x'), + ('dash list', '- major — x'), + ('en dash', 'major – x'), + ('hyphen', 'major - x'), + ('emoji', '🔴 Major: x'), + ('emoji, bold', '⚠️ **Blocker**: x'), + ('numbered label', 'major 2: a retry loops'), + ('plural with a note', 'Majors (state): stopped half-way leaves a row'), + ('brackets', '[major] x'), + ('second label of a line', 'Blocker: none, major: two writers race'), + ('after a semicolon', 'blocker: none; major: x'), + ('after and', 'blocker: none and major: x'), + ('table row', '| severity | finding |\n|---|---|\n| major | x |'), + ('table row, bold', '| **Blocker** | the lock is gone |'), + ('table column', '| blocker | major |\n| --- | --- |\n| none | two writers race |'), + ('fullwidth M (NFKC)', '\uff2dajor: x'), + ('zero-width space', 'maj\u200bor: x'), + ('zero-width joiner in heading', '### Bl\u200docking\n1. x'), + ('soft hyphen', 'ma\u00adjor: x'), + ('Cyrillic o', 'maj\u043er: x'), + ('Cyrillic M', '\u041cajor: x'), + ('Greek omicron', 'bl\u03bfcker: x'), + ('digit zero for o', 'maj0r: x'), + ('heading, code under it', '### Major\n```\nthe race is in x\n```'), + ('heading, nothing under it', '### Major'), + ('heading, next heading under it', '### Major\n### Minor\n- a typo'), + ('bare label, nothing under it', 'major:'), + ('a none that is not one', 'major: none of the retries stop'), + ('a no that is not one', 'major: no lock on the write path'), + ('a dash that is not one', 'major: - x'), + ('see below', 'blocker: see below'), + ('quoted (read: fail closed)', '> major: quoted from round 1'), + ('indented (read: fail closed)', ' major: in an indented block'), + ('HTML tag', 'major: x'), + ('link', '[major](http://e/x): y'), + ('summary tag', '
major: x'), + ('narrow read only', 'major (see '), # the wide tag eats the colon + ('NOT SAFE header', '## Review lens A, head 0123abc: **NOT SAFE**\nblocker: none\nmajor: none'), + ('NOT SAFE verdict', 'verdict: NOT SAFE\nblocker: none\nmajor: none'), + ('finding line', 'finding: major src/a.py:3 loses data'), # another project's lens format + ('finding line, blocker', '- finding: blocker the lock is gone'), + ('finding line after blocker: none', 'blocker: none\nfinding: major src/a.py:3 loses data'), + ('finding line after findings: none', 'findings: none\nfinding: major x'), + ('recap of a fixed major', NONE + '\n- Major 1 (cap): fixed at the root.'), # labelled: read as open + ('recap of a fixed blocker', NONE + '\n**Blocking 1:** fixed.'), + ('HTML entity', NONE + '\nmajor: x'), + ('Markdown escape', NONE + '\nmajor\\: x'), + ('ratio sign for a colon', NONE + '\nmajor\u2236 x'), + ('a dash glued to none', 'major: none-critical race'), +] +# Lines a clean report writes: no finding, and a findings section. +CLEAN = [ + ('both on one line', 'Blocker: none, major: none'), + ('none found', 'Major: none found'), + ('none with a note', 'major: none (only minors)'), + ('none!', 'major: none!'), + ('heading, none under it', '### Major\nnone'), + ('headings, None.', '### Blocking\nNone.\n\n### Major\nNone.'), + ('bold inline', '**Blocking:** none.\n**Major:** none.'), + ('a rule between', '### Major\n\n---\n\nNone.'), + ('table columns', '| blocker | major |\n|---|---|\n| none | none |'), + ('table columns, blanks round the rule', '| blocker | major |\n |---|---| \n| none | none |'), + ('table row', '| severity | finding |\n|---|---|\n| major | none |'), + ('n/a', 'major: n/a'), ('zero', 'major: 0'), ('em dash', 'major: —'), ('list', '- majors: none'), + ('bold no', '**blockers:** no'), ('nothing', 'blocker: nothing'), ('NA', 'major: NA'), + ('minors do not block', 'major: none\n### Minor\n- a typo\nminor: x\nnit: y'), + ('code block', 'major: none\n```\nmajor: in code\n### Blocking\n1. x\n```'), + ('tilde fence', 'blocker: none\n~~~\nblocker: x\n~~~'), + ('longer fence holds a shorter one', 'major: none\n````\n```\nmajor: x\n```\n````'), + ('findings: none', 'findings: none'), + ('Unicode none', '\u041cajor: n\u043ene'), + ('both, full stops', 'Blocker: none. Major: none.'), + ('nothing found', 'major: nothing found'), + ('a minor after it', 'major: none; minor: x'), + ('prose that says not safe', 'major: none\nThe old code was not safe to run twice; this fixes it.'), + ('none, a dash, a note', 'major: none — two minors below'), + ('none, a hyphen, a note', 'major: none - two minors below'), + ('none in parentheses', 'Major: (none)'), + ('none and an emoji', 'major: None \u2705'), + ('only minor finding lines', 'finding: minor src/a.py:3 a typo\nfinding: nit y'), + ('a recap without a label', NONE + '\nround 1, fixed: the race'), +] +# Each makes a findings section on its own: a blocker and a major label, `findings: none` or `finding:` lines. +SECTION = ['Blocker: none, major: none', '### Blocking\nNone.\n\n### Major\nNone.', + '**Blocking:** none.\n**Major:** none.', '| blocker | major |\n|---|---|\n| none | none |', 'findings: none', + 'Blocker: none. Major: none.', + 'finding: minor a typo', NONE, '| severity | finding |\n|---|---|\n| blocker | none |\n| major | none |'] +# Review A and B of #740 round 2: `blocker: none` kept from the prompt, the major written in a form not read as a label. +# With one label kind only there is no findings section, so each fails closed (no section), never clean. +ONE_KIND = ['F1 (major): x', '1. **Retry drops rows** (major): x', '- F1. Severity: major. x', + '- F1 — **Severity:** Major — x', '- F1 — major: x', 'F1: major: x', 'One major: x', 'Major 1. x', + 'minor: typo. Also major: x'] +ONE_LABEL = ['major: none', 'blocker: none', '### Major\nNone.', '| severity | finding |\n|---|---|\n| major | none |'] +# Lines that only mention a label: no finding, and no findings section either. +MENTION = ['fixed: the major of round 1', 'no major found', 'majority: x', 'Major version: 2', 'blockerx: y', + 'major 12345: too long a number', 'the blocking call in x'] + + +class Findings(unittest.TestCase): + def test_each_real_finding_format_counts(self): + for what, body in FINDING: + with self.subTest(what): + said, section = rj.read_findings(HEAD + body) + self.assertTrue(said, body) + self.assertTrue(said[0][0] in ('blocker', 'major')) + + def test_each_clean_format_is_no_finding(self): + for what, body in CLEAN: + with self.subTest(what): + self.assertEqual(rj.read_findings(HEAD + body)[0], [], body) + self.assertEqual(rj.read_findings(HEAD + NONE + '\n\n' + body), ([], True), body) + + def test_a_section_needs_a_blocker_and_a_major_label(self): # #740 round 2, A major 1 + for body in SECTION: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + body), ([], True)) + for body in ONE_LABEL: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + body), ([], False)) + for body in ONE_KIND: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + 'blocker: none\n' + body), ([], False)) + r = rounds([('code', H1)], {'code': HEAD + 'blocker: none\n' + body}) + self.assertEqual(len(r.problems), 1) + self.assertIn('lens code: its newest report has no findings section', r.problems[0]) + + def test_a_long_run_of_spaces_takes_linear_time(self): # #740 round 2: 81-91 s for one 65,000-character line + n = 200_000 + for line in ('major' + ' ' * n + 'x', 'blocker: none' + ' ' * n + 'x', '| a | b |\n' + ' ' * n + 'x', + '| a | b |\n|' + ' ' * n + 'x', '| a | b |\n|-' + ' ' * n + 'x', + 'not' + ' ' * n + 'safe', 'finding:' + ' ' * n + 'x', 'major: none' + ' \t' * (n // 2) + 'x'): + with self.subTest(line[:12]): + start = time.monotonic() + rj.read_findings(HEAD + line) + self.assertLess(time.monotonic() - start, 2) + + def test_a_table_does_not_go_on_past_a_fence(self): # a fence ends the table, as GitHub renders it + table = '| blocker | major |\n|---|---|\n| none | none |\n' + self.assertEqual(rj.read_findings(table + '```\ncode\n```\n| a | b |'), ([], True)) + self.assertEqual(len(rj.findings(table + '| a | b |')), 1) # without the fence: a row of the table + + def test_a_line_that_only_mentions_a_label_is_neither(self): + for line in MENTION: + with self.subTest(line): + self.assertEqual(rj.read_findings(HEAD + line), ([], False)) + + def test_the_kind_and_line_are_named(self): + self.assertEqual(rj.findings('blocker: the lock is gone\n- **Major:** two writers race\nminor: a typo'), + [('blocker', 'blocker: the lock is gone'), ('major', 'Major: two writers race')]) + self.assertEqual(rj.findings('### Major\n- race')[0], ('major', 'Major race')) + self.assertEqual(rj.findings('### Major')[0], ('major', 'Major (nothing under it)')) + + def test_an_unclosed_fence_runs_to_the_end(self): # GitHub shows the rest as code + self.assertEqual(rj.read_findings(NONE + '\n```\nmajor: x'), ([], True)) + + def test_fold(self): + self.assertEqual(rj.fold('\uff2da\u200bj\u043er'), 'Major') + self.assertEqual(rj.fold(None), '') + + +class Rounds(unittest.TestCase): + def test_each_different_change_read_is_a_round(self): + self.assertEqual(rj.review_round([]), 0) + self.assertEqual(rj.review_round([('code', H1), ('data-1', H1)]), 1) + self.assertEqual(rj.review_round([('code', H1), ('code', H2), ('data-1', H2), ('code', H3)]), 3) + self.assertEqual(rj.review_round([('code', None)]), 0) + + def test_a_short_sha_counts_too(self): # it reads nothing for the head, but it was a read: a round + self.assertEqual(rj.review_round([('code', 'abcdef1'), ('code', H1)]), 2) + + def test_reads_of_one_change_are_one_round(self): # #100: a queue update merges the base in, no fix + change = {H1: 'x', H2: 'x', H3: 'y'}.get + self.assertEqual(rj.review_round([('code', H1), ('code', H2)], change), 1) + self.assertEqual(rj.review_round([('code', H1), ('code', H2), ('code', H3)], change), 2) + self.assertEqual(rj.review_round([('code', H1.upper()), ('code', H1)]), 1) + + def test_a_round_marker_counts_when_it_is_higher(self): # deleted old rounds: the marker still says it + self.assertEqual(rj.review_round([('code r3', H1)]), 3) + self.assertEqual(rj.review_round([('code **r4**', H1), ('data r2', H2)]), 4) # emphasis stripped: counts + self.assertEqual(rj.review_round([('code `r5`', H1)]), 5) + self.assertEqual(rj.review_round([('code r1', H1), ('code r1', H2), ('code', H3)]), 3) + self.assertEqual(rj.round_marker('r2d2 lens'), 0) + self.assertEqual(rj.round_marker('r9'), 0) # a name's first word is its name, never a marker + self.assertEqual(rj.round_marker('code r\uff13'), 3) # folded + + def test_no_lens_comment_yet_is_round_zero_and_passes(self): # empty state + r = rounds([], {}) + self.assertEqual((r.round, r.approach, r.over_cap, r.findings, r.problems), (0, False, False, (), ())) + + def test_a_major_blocks_until_a_later_round_of_that_lens_clears_it(self): + r = rounds([('code', H1)], {'code': 'major: two writers race'}) + self.assertEqual(r.findings, (('code', 'major', 'major: two writers race'),)) + self.assertIn('lens code reported a major that no later round cleared', r.problems[0]) + self.assertIn('lens code posts a new report on the fixed head', r.problems[0]) + r = rounds([('code', H1), ('code r2', H2)], {'code': NONE}) # round 2 cleared it + self.assertEqual((r.round, r.problems), (2, ())) + + def test_a_report_with_no_findings_section_fails_that_lens(self): # #99: never passes as clean + r = rounds([('code', H1)], {'code': HEAD + 'Looks good to me.'}) + self.assertEqual(len(r.problems), 1) + self.assertIn('lens code: its newest report has no findings section', r.problems[0]) + self.assertIn('findings in inline review comments are not read', r.problems[0]) + self.assertEqual(rounds([('code', H1)], {'code': HEAD}).problems, r.problems) # empty body: the same + self.assertEqual(rounds([('code', H1)], {'code': HEAD + NONE}).problems, ()) + self.assertEqual(rounds([('code', H1)], {'code': HEAD + 'major: none'}).problems, r.problems) # one kind + + def test_an_entry_that_reads_nothing_under_an_agents_override_cannot_be_judged(self): + r = rounds([('code', H1)], {'code': (rj.UNREAD, HEAD + 'major: none')}) + self.assertIn('its findings cannot be read', r.problems[0]) + + def test_a_loose_comment_under_an_agents_override_is_judged_by_its_lines(self): # no section needed + self.assertEqual(rounds([], {'x': (rj.LOOSE, 'fixed it')}).problems, ()) + self.assertEqual(len(rounds([], {'x': (rj.LOOSE, 'major: still racy')}).findings), 1) + + def test_a_cleared_or_unjudged_entry_is_not_judged_here(self): + for how in (rj.CLEARED, rj.NOT_JUDGED): # a person's override; or it fails the head already + with self.subTest(how): + self.assertEqual(rounds([('code', H1)], {'code': (how, 'blocker: x')}).problems, ()) + + def test_findings_false_judges_no_finding(self): + r = rounds([('code', H1)], {'code': 'blocker: x'}, review=dataclasses.replace(BASIC, findings=False)) + self.assertEqual(r.problems, ()) + self.assertIn('findings are not judged', r.text(dataclasses.replace(BASIC, findings=False))) + + def test_the_approach_lens_is_required_from_round_approach_at(self): + self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': NONE}).approach) + r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': NONE}) + self.assertTrue(r.approach) + self.assertIn('approach lens approach required from round 3', r.text(BASIC)) + self.assertFalse(rounds([('code', H1), ('code', H2), ('code', H3)], {}, decided=True).approach) + + def test_a_round_past_the_cap_stops_and_says_what_a_person_does(self): # #100 + four = [('code', h) for h in (H1, H2, H3, H4)] + self.assertEqual(rounds(four, {'code': NONE}).problems, ()) # round 4 of 4 may pass + r = rounds(four + [('code', H5)], {'code': NONE}) + self.assertTrue(r.over_cap) + self.assertEqual(r.problems, ( + 'review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides: close the pull ' + 'request, or a person\'s override comment, but no one is listed in review.override_authors yet, so no ' + 'override is accepted (issue #78: until agents no longer post as the person\'s login, an override is not ' + 'proof of a person). A person\'s override alone is enough, also for a required approach lens that never ' + 'posted',)) + r = rounds(four + [('code', H5)], {'code': NONE}, review=PEOPLE) + self.assertIn('an override comment `override for head : ` for this head by a-person, newer ' + 'than every lens comment', r.problems[0]) + self.assertEqual(rounds([('code r5', H1)], {'code': NONE}).round, 5) + + def test_a_persons_decision_passes_the_cap_and_waives_the_approach_lens(self): + five = [('code', h) for h in (H1, H2, H3, H4, H5)] + r = rounds(five, {'code': (rj.CLEARED, '')}, decided=True, review=PEOPLE) + self.assertEqual((r.over_cap, r.approach, r.problems), (False, False, ())) + + def test_a_finding_at_the_last_round_says_no_more_rounds(self): + r = rounds([('code', h) for h in (H1, H2, H3, H4)], {'code': 'major: still racy'}) + self.assertIn('this is round 4 of 4: no more rounds; a person decides', r.problems[0]) + + def test_the_same_comments_give_the_same_verdict(self): # runs again: pure + args = ([('code', H1), ('code r2', H2)], {'code': 'major: x'}) + self.assertEqual(rounds(*args), rounds(*args)) + + +class OverrideAuthors(unittest.TestCase): + def test_left_out_or_empty_is_no_person(self): # #78: agents still post as the owner's login + self.assertEqual(BASIC.override_authors, ()) + block = VECTORS['configs']['basic'] + self.assertEqual(rc.parse_review({**block, 'override_authors': []}).override_authors, ()) + self.assertEqual(rc.parse_review({**block, 'override_authors': ['a-person', 'B2']}).override_authors, + ('a-person', 'B2')) + + def test_a_wrong_list_is_refused(self): + block = VECTORS['configs']['basic'] + for bad in ('a-person', [''], ['-x'], ['a b'], ['x', 'X'], [1], ['x'] * 21, [f'p{i}' for i in range(21)], None): + with self.subTest(bad), self.assertRaises(config.ConfigError) as e: + rc.parse_review({**block, 'override_authors': bad}) + self.assertIn('override_authors', str(e.exception)) + self.assertEqual(len(rc.parse_review({**block, 'override_authors': [f'p{i}' for i in range(20)]}) + .override_authors), 20) + + +# --- the fork's own config (issue #730): in shadow, workflow, queue and lens files critical (#82) ---------------- +ROOT = os.path.join(HERE, '..', '..', '..') +FORK_CONFIG = '.github/fork-pr-queue.json' + + +class ForkConfig(unittest.TestCase): + def setUp(self): + with open(os.path.join(ROOT, FORK_CONFIG), encoding='utf-8') as f: + self.cfg = config.parse(f.read(), FORK_CONFIG) + self.review = self.cfg.review + + def test_it_parses_in_shadow_with_the_owners_numbers_and_no_person_yet(self): + r = self.review # shadow first (README, Turning it on); no override author until #78 + self.assertEqual((r.mode, r.findings, r.rounds.approach_at, r.rounds.max, r.warnings, r.override_authors), + ('shadow', True, 3, 4, (), ())) + self.assertEqual(r.order, ()) # enforce would take it as it is + self.assertGreaterEqual(r.min_lenses.critical, r.min_lenses.normal + 1) # #82: one extra review + for x in r.lenses: + with self.subTest(x.name): + self.assertTrue(os.path.isfile(os.path.join(ROOT, x.prompt)), x.prompt) + + def test_workflow_queue_gate_config_and_lens_files_are_critical(self): # #82 + critical = next(x for x in self.review.rules if x.weight == 'critical') + for path in ('.github/workflows/fork-head-gate.yml', 'fork-ci/actions/pr-queue/review_judge.py', + 'fork-ci/actions/head-check/head_check.py', FORK_CONFIG, '.github/fork-lens-state.md', + '.github/actions/setup-ci-deps/action.yml', '.github/fork-ci-release-rulesets.json'): + with self.subTest(path): + j = rj.judge(self.review, [('M', (path,))], FORK_CONFIG) + self.assertEqual((j.weight, j.lenses), ('critical', critical.lenses)) + self.assertGreaterEqual(len(j.lenses), 2) + j = rj.judge(self.review, [('M', ('apps/server/src/x.ts',))], FORK_CONFIG) + self.assertEqual(j.weight, 'normal') + self.assertTrue(j.lenses) + + if __name__ == '__main__': unittest.main()