diff --git a/.github/fork-ci-rerun.json b/.github/fork-ci-rerun.json index 4487aa09a5bd8..311cdfa1e78ab 100644 --- a/.github/fork-ci-rerun.json +++ b/.github/fork-ci-rerun.json @@ -8,8 +8,15 @@ "list_hours: how far back runs are listed (by creation; every page is read); catch_up_minutes: of those, the", " runs that ended (updated) this recently. window_hours: the re-run count read from the tracking issue.", "hosted_runner_groups: GitHub-hosted runners' group; an OOM kill there is the job's own (no box of ours).", - "probe_end_step: the step fork-ci-pressure.py writes the note in (fork-tests.yml); only a job that completed it", - " has a note of its own, and a job that never reached it lost its runner or was stopped.", + "probe_start_step, probe_end_step: the steps fork-ci-pressure.py takes its samples in (fork-tests.yml); the end", + " step writes the note. Only a job that completed the end step has a note of its own, and a job that never", + " reached it lost its runner or was stopped. note_slack_seconds: the note's seconds from start to end must lie", + " between the two steps' own times, widened by this much (GitHub's step times are whole seconds).", + "annotation_cap: how many annotations GitHub keeps of one job; a job with that many may have lost its note (the", + " end step writes it last), so its note is never read.", + "probe_limit_seconds: the probe steps' own time limit (fork-tests.yml `changes` output probe_seconds, the same).", + " An end step that took this long was stopped (it warns and ends green, with no note of its own), and a job whose", + " probe printed a warning (fork-ci-pressure: ...) may have none either: neither job's note is read.", "limit_slack_seconds: a time-limit message counts only when the job (or the step) ran its minutes less this.", "signs: GitHub's own runner messages, each pattern a whole message (re.fullmatch), only on an annotation in the", " runner's form (failure, no title, .github), and only together with the job's step record (the script's rules).", @@ -18,8 +25,9 @@ " cancelled_by: a person cancelled it; the run stays red.", "stalled: the job's memory pressure note shows the machine or a box stalled: some-or-all-tasks-waiting (full) share", " of the job's time, or the 5-minute full average at its end.", - "memory_full: on a runner that is not GitHub-hosted, a box with a memory limit stalled (full) at least", - " box_full_share_percent of the job, AND in that box OOM kills, memory.max hits or memory.high throttling.", + "memory_full: on a runner that is not GitHub-hosted, the job's own cgroup AND a box with a memory limit (its own", + " or an ancestor, which other jobs may share) stalled (full) at least box_full_share_percent of the job, AND an", + " OOM kill in the job's own cgroup, or memory.max hits or memory.high throttling in that box.", "skip_when: with this environment variable (the workflow sets it from a repository variable) at one of these", " values, a run that qualifies is only reported, once, not re-run (fleet mode: a re-run of failed jobs would wait", " for fleet runners that refuse it, .github/fork-fleet.md).", @@ -34,7 +42,11 @@ "window_hours": 24, "skip_when": { "variable": "CI_RUNNERS", "is": ["fleet"] }, "hosted_runner_groups": ["GitHub Actions"], + "probe_start_step": "Note memory pressure at the start", "probe_end_step": "Note memory pressure at the end", + "note_slack_seconds": 3, + "annotation_cap": 50, + "probe_limit_seconds": 40, "limit_slack_seconds": 60, "signs": { "lost": [ diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md index fe3a1c8a0971d..4a2b531b36264 100644 --- a/.github/fork-ci-rerun.md +++ b/.github/fork-ci-rerun.md @@ -7,13 +7,13 @@ a test failure, never for a person's cancel, never twice: a repeat red is left red, and it goes to an issue and its fix first. Every re-run is also a comment on one tracking issue, so a machine that keeps stalling is never hidden. -| Part | What it does | -| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`), each with a 1-minute timeout. Facts only: it never fails (it catches every error), and a hung one never holds up the job. | -| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, the tracking issue, and the re-run. Uses `fork-fleet.py`'s trust rule. | -| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: the trust config (`fork-fleet.json`), workflow, followers, windows, the hosted runner group, the probe's end step, sign patterns, thresholds, the fleet-mode switch, the tracking issue. | -| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`, `issues: write`. `workflow_dispatch` with `run_id` checks one run, in a concurrency group of its own. | -| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal (each rule line is pinned: removing it turns a test red), the state cases below, the paging, the wiring. | +| Part | What it does | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`), each with a 1-minute timeout. Facts only: it never fails (it catches every error). Each runs under `timeout` with the `changes` job's `probe_seconds` (a stalled machine can hang it), then warns and goes on, so the tests still run. | +| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, the tracking issue, and the re-run. Uses `fork-fleet.py`'s trust rule. | +| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: the trust config (`fork-fleet.json`), workflow, followers, windows, the hosted runner group, the probe's two steps and its time limit, the note's time slack, GitHub's annotation cap, sign patterns, thresholds, the fleet-mode switch, the tracking issue. | +| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`, `issues: write`. `workflow_dispatch` with `run_id` checks one run, in a concurrency group of its own. | +| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal, each threshold at its edge, the state cases below, the paging, the wiring. Each rule line is pinned: a mutation sweep of the rule functions (each condition forced true and false, each comparison, `and`/`or`, `not`, `fullmatch` and sign changed) turns a test red for every mutant that changes what the code does. | ## The numbers in the job log @@ -60,11 +60,16 @@ start sample, a reboot in between, a file the runner cannot see. of its steps failed, no time limit was hit, and the probe's end step never completed (the runner died before it). No note needed. - **memory full**: the job ended `failure` on a runner that is not - GitHub-hosted (`hosted_runner_groups`: an OOM kill there is the job's - own), and its own note shows a box with a memory limit stalled (full) at - least `box_full_share_percent` of the job AND, in that box, OOM kills, - `memory.max` hits or `memory.high` throttling. One event without the - stall, or a machine-wide OOM kill, is no sign. + GitHub-hosted (`hosted_runner_groups`), and its own note shows the job's + own cgroup (the note's first: the runner's unit, one job at a time) + stalled (full) at least `box_full_share_percent` of the job, a box with + a memory limit (that cgroup or an ancestor) stalled as much, AND an OOM + kill in the job's **own** cgroup, or `memory.max` hits or `memory.high` + throttling in that box. An ancestor's OOM count is never read: + `memory.events` counts a whole subtree, so a slice several jobs share + counts the other jobs' kills too. One event without the stall, a + machine-wide OOM kill, or a job whose own cgroup is the root (`/`) is + no sign. - **limit while stalled**: GitHub says the job or a step ran past its time limit; its minutes are that job's (or step's) own `timeout-minutes` in the workflow file at the run's commit (read through the contents API; a step @@ -75,8 +80,20 @@ start sample, a reboot in between, a file the runner cannot see. machine or a box stalled (full) at least `full_share_percent` of the job, or a 5-minute full average at least `full_avg300_percent` at the end. 4. The note counts only when the probe's end step completed (so the job wrote - its own) and it is the job's only one, a notice on `.github` with the - `fork-ci-pressure v1 ` header and the right shape. + its own), the job has fewer annotations than GitHub's cap + (`annotation_cap`: GitHub keeps only the first 50 of a job, and the end + step's note comes last, so at the cap the real one may be gone), the end + step took less than the probe's limit (`probe_limit_seconds`) and no + annotation starts with `fork-ci-pressure:` (the probe's own warnings: a + probe stopped at its limit, or one that raised, ends its step green with + only that warning and no note of its own), it is the job's only one, a + notice on `.github` with the `fork-ci-pressure v1 ` header and the right + shape, and its `seconds` fit the time between the two probe steps by the + runner's own step record (`note_slack_seconds` for whole-second times). + A note printed well before the end step cannot know when the end step + will run, so it does not fit. A note printed by the step just before the + end step can fit (that step knows the start time): only trust stops that + one (rule 1, and rule 2: once). 5. One failed job without a sign leaves the run red: a test failure, a job with no note, a garbled one, two notes. GitHub's "The run was canceled by @..." on a failed job leaves it red too: a person's cancel stands, whatever @@ -93,9 +110,12 @@ start sample, a reboot in between, a file the runner cannot see. issue, by `github-actions`, with that title; every later re-run (and every fleet-mode report) is one comment on it: the run's link, event, branch and commit, each job's sign, GitHub's message and the pressure numbers. Never one - issue per re-run. Only an open issue by `github-actions` with exactly that - title is used; closing it is fine (the next re-run opens a new one). A - comment carries a marker line (``): only an open issue by + `github-actions` whose body starts with it is used, whatever its title (a + person's issue never is). Closing it is fine (the next re-run opens a new + one). A pass looks for it again just before opening one. A comment carries a + marker line (``); the comment is written **before** the request. - Each re-run is also a **warning annotation** on the `Fork CI re-run` run. - Each pass writes a **job summary**: a table of every red run it looked at and @@ -106,20 +126,24 @@ start sample, a reboot in between, a file the runner cannot see. Nothing is kept between passes: each one reads GitHub (the runs and the issue). -| Case | What happens | -| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. No second comment. | -| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | -| Two passes open the issue at once | Each reads the open issues after opening its own; the lowest number wins and the other closes its own as a duplicate. | -| A pass stops before its request reached GitHub | The comment is there, the run unchanged; the next pass (within `catch_up_minutes`) sends the request once and adds no second comment. | -| A pass stops after GitHub took the request (reply lost) | The comment, written first, says why. The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | -| The comment cannot be written | No re-run (it would be hidden): a warning, the pass ends red, the next pass tries both. | -| The tracking issue cannot be read | No re-run (its record cannot be checked) and no count: the pass ends red and says so; the next pass tries again. | -| A dispatch with `run_id` while a scheduled pass waits | Its own concurrency group: it is never replaced by a scheduled pass. | -| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | -| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). Runs are listed by creation over `list_hours`, every page read; a listing past 10 pages ends the pass red, nothing re-run. | -| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | -| No tracking issue, or it was closed | The count reads 0 from a new window; the next re-run opens a new one. | +| Case | What happens | +| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. No second comment. | +| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | +| Two passes open the issue at once | Each reads the open issues after opening its own; the lowest number wins and the other closes its own as a duplicate. | +| A pass stops after opening the issue (reply lost) | No comment, no re-run: the pass ends red. The next pass finds the open issue by its marker and uses it; no second issue. | +| Two tracking issues are open (a pass stopped half-way) | The next pass, whatever it finds (every pass reads the issue for its 24 h count at its end, a pass with a run to record before it), keeps the lowest and closes the others: closed first, then one "Duplicate of #N." comment. If a close fails, the lowest is still the one used, the pass warns and posts no comment, and each later pass tries again. | +| A closed duplicate held re-run comments | Its markers drop out of the 24 h count (only open issues are read), so the count can read one low until the window passes. The count only informs: never-twice comes from the run's own attempts, and the comments stay on the closed issue. | +| A pass stops before its request reached GitHub | The comment is there, the run unchanged; the next pass (within `catch_up_minutes`) sends the request once and adds no second comment. | +| A pass stops after GitHub took the request (reply lost) | The comment, written first, says why. The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | +| The comment cannot be written | No re-run (it would be hidden): a warning, the pass ends red, the next pass tries both. | +| The tracking issue cannot be read | No re-run (its record cannot be checked) and no count: the pass ends red and says so; the next pass tries again. | +| A dispatch with `run_id` while a scheduled pass waits | Its own concurrency group: it is never replaced by a scheduled pass. | +| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | +| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). Runs are listed by creation over `list_hours`, every page read; a listing past 10 pages ends the pass red, nothing re-run. | +| A person re-runs a run created more than `list_hours` ago | Not looked at: GitHub lists runs only by creation time, and a re-run keeps its run's. Reading further back every 10 minutes costs API calls for a rare case; the person who re-ran it is watching it. Dispatch `Fork CI re-run` with its `run_id` to check it. | +| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | +| No tracking issue, or it was closed | The count reads 0 from a new window; the next re-run opens a new one. | ## Overlap with the fleet sweep @@ -139,14 +163,29 @@ them); that is `fork-fleet.py`'s to change. qualifies, fail closed) and (b) for a `continue-on-error` step that failed; and on a real lost runner, its steps' state. - A note is the job's own words: a trusted run's code could still print one - and keep the real end step from writing its own (two notes are refused, and - a note without its end step's completion is not read). Only trusted runs are - read, and only once. + and keep the real end step from writing its own. Two notes are refused; a + note without its end step's completion, after an end step that ran to the + probe's limit, or beside a probe warning is not read; and its seconds must + fit the probe steps' own times, which stops a note printed well before the + end step. A step that runs just before the end step knows the start time + and can print a note that fits, then hide the real one (a garbled start + sample, a fake `python3` first on the path). Only trust stops that: only + trusted runs are read, and each is re-run once at most, on the issue. - An OOM kill by a user-space killer (earlyoom, systemd-oomd) is not in `memory.events` or `oom_kill`: it shows only as a stall, if at all. - GitHub keeps at most 10 notices per step and 50 annotations per job: on a - job with very many annotations the note can be dropped. The job then reads - as "no note" and stays red (fail closed). + job with very many annotations the note can be dropped. A job with 50 + annotations (`annotation_cap`) is never read for a note, and stays red + (fail closed). +- A probe step that hangs (a stalled machine) is stopped after the + `changes` job's `probe_seconds` (`timeout -k 5 "${{ ... }}"`), with a + warning, `fork-ci-pressure: start stopped`, and the job goes on. That + output is the workflow's one copy of the number (a workflow-level `env:` + is not allowed in `fork-tests.yml`), and `probe_limit_seconds` in the + config must equal it (tested). A start step stopped that way leaves no + start sample, so the note has no `seconds` and the job can only qualify as + `lost`. An end step stopped that way leaves no note, and its job's note is + never read (rule 4). The 1-minute `timeout-minutes` stays as a backstop. - Text quoted into the public tracking issue (annotation messages, note paths) is made one line with no `@` mention and no HTML, and only a comment's first line is read as its marker. diff --git a/.github/scripts/fork-ci-pressure.test.py b/.github/scripts/fork-ci-pressure.test.py index b7648e16d802d..aed3a4abbed4f 100644 --- a/.github/scripts/fork-ci-pressure.test.py +++ b/.github/scripts/fork-ci-pressure.test.py @@ -16,6 +16,10 @@ # The jobs that run app builds and tests on a fleet or GitHub-hosted runner (each routable job of fork-fleet.json). with open(os.path.join(ROOT, '.github/fork-fleet.json'), encoding='utf-8') as f: APP_JOBS = sorted({j['job'] for j in json.load(f)['jobs']}) +with open(os.path.join(ROOT, '.github/fork-ci-rerun.json'), encoding='utf-8') as f: + RERUN_CFG = json.load(f) +WORKFLOW = os.path.join(ROOT, '.github/workflows/fork-tests.yml') +LIMIT = '${{ needs.changes.outputs.probe_seconds }}' # the probe's time limit, one value of the changes job GIB = 1024 ** 3 @@ -190,9 +194,98 @@ def test_it_never_fails_the_job(self): self.assertIn('fork-ci-pressure: job failure', r.stdout) +def step_run(verb): + """The probe step's command: a hang (a stalled machine) is stopped after the changes job's probe_seconds with a + warning, and the job goes on.""" + tail = 'no numbers; the job goes on' if verb == 'start' else 'no numbers' + return (f'timeout -k 5 "{LIMIT}" python3 -I .github/scripts/fork-ci-pressure.py {verb} || ' + f'echo "::warning::fork-ci-pressure: {verb} stopped (exit $?), {tail}"') + + +def load_rerun(): + spec = importlib.util.spec_from_file_location('fork_ci_rerun', os.path.join(HERE, 'fork-ci-rerun.py')) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + class Steps(unittest.TestCase): """Each app job of fork-tests.yml takes a sample right after its checkout and prints the numbers last.""" + def run_step(self, verb, python3, limit=str(RERUN_CFG['probe_limit_seconds'])): + """The step's own text in GitHub's shell (bash -e), with a stand-in python3 and its time limit made short.""" + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d) + with open(os.path.join(d, 'python3'), 'w', encoding='utf-8') as f: + f.write('#!/bin/sh\n' + python3 + '\n') + os.chmod(os.path.join(d, 'python3'), 0o755) + text = step_run(verb) + self.assertEqual((text.count(' -k 5 '), text.count(LIMIT)), (1, 1)) + text = text.replace(' -k 5 ', ' -k 1 ').replace(LIMIT, limit) # as GitHub fills in the expression + return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', text], capture_output=True, + text=True, timeout=30, env={'PATH': d + os.pathsep + os.environ.get('PATH', '')}) + + def test_a_hung_probe_is_stopped_and_the_job_goes_on(self): + # A stalled machine can hang the probe itself: without its own limit the step's 1-minute timeout fails the + # step, and every later step (the tests) is skipped. Now the step ends well after the limit, with a warning. + for verb in ('start', 'end'): + with self.subTest(verb): + r = self.run_step(verb, 'sleep 25', limit='1') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn(f'::warning::fork-ci-pressure: {verb} stopped (exit 124)', r.stdout) + + def test_a_probe_that_ignores_the_stop_signal_is_killed(self): + r = self.run_step('start', "trap '' TERM; sleep 25", limit='1') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('::warning::fork-ci-pressure: start stopped (exit 137)', r.stdout) + + def test_a_stopped_end_probe_leaves_the_rerun_no_note_to_read(self): + # The stop warning, as GitHub hands it back, with a note a step printed earlier that fits the window: the + # re-run reads no note (fork-ci-rerun.py PROBE_WARNS), so the job stays red. + rerun = load_rerun() + r = self.run_step('end', 'sleep 25', limit='1') + line = r.stdout.strip().splitlines()[-1] + self.assertTrue(line.startswith('::warning::' + rerun.PROBE_WARNS), line) + end = RERUN_CFG['probe_end_step'] + stamp = lambda s: '2026-01-01T00:%02d:%02dZ' % divmod(s, 60) + steps = [{'name': n, 'status': 'completed', 'conclusion': 'success', 'started_at': stamp(a), + 'completed_at': stamp(b)} for n, a, b in ((RERUN_CFG['probe_start_step'], 0, 1), ('Test', 1, 60), + (end, 60, 62))] + body = {'status': 'failure', 'seconds': 61, 'host': {}, 'cgroups': []} + printed = {'message': rerun.NOTE_HEAD + json.dumps(body), 'annotation_level': 'notice', + 'title': rerun.NOTE_TITLE, 'path': '.github'} + cfg = dict(RERUN_CFG) + self.assertIsNotNone(rerun.note_of(cfg, {'steps': steps}, [printed])[0]) # alone, it would be read + warning = {'message': line[len('::warning::'):], 'annotation_level': 'warning', 'title': '', + 'path': '.github'} + got, why = rerun.note_of(cfg, {'steps': steps}, [printed, warning]) + self.assertIsNone(got) + self.assertIn('the probe warned', why) + + def test_the_probe_limit_is_in_one_place(self): + # One value of the changes job (every app job needs it); the re-run's config holds the same number (it reads + # no note from an end step that took that long). No step or comment keeps a copy of the number. + import yaml + with open(WORKFLOW, encoding='utf-8') as f: + text = f.read() + wf = yaml.safe_load(text) + limit = RERUN_CFG['probe_limit_seconds'] + self.assertEqual(wf['jobs']['changes']['outputs']['probe_seconds'], str(limit)) + probes = [line for line in text.splitlines() if 'fork-ci-pressure.py' in line and 'timeout' in line] + self.assertEqual(len(probes), 2 * len(APP_JOBS)) + for line in probes: + self.assertIn(f'timeout -k 5 "{LIMIT}" python3 ', line) + for name in APP_JOBS: + needs = wf['jobs'][name]['needs'] + self.assertIn('changes', [needs] if isinstance(needs, str) else needs, name) + self.assertEqual(re.findall(rf'timeout +(?:-k +[0-9]+ +)?[0-9]+ +python3|\b{limit} ?s\b', text), []) + # The step's 1-minute timeout-minutes stays a backstop: the limit and the kill's 5 s end well inside it. + self.assertLessEqual(limit + 5, 50) + + def test_a_probe_that_ends_in_time_adds_no_warning(self): + r = self.run_step('start', 'echo "fork-ci-pressure: start sample saved"') + self.assertEqual((r.returncode, r.stdout), (0, 'fork-ci-pressure: start sample saved\n'), r.stderr) + def test_every_app_job_has_the_start_and_end_steps(self): import yaml with open(os.path.join(ROOT, '.github/workflows/fork-tests.yml'), encoding='utf-8') as f: @@ -202,10 +295,10 @@ def test_every_app_job_has_the_start_and_end_steps(self): with self.subTest(name): steps = wf['jobs'][name]['steps'] self.assertEqual(steps[0]['name'], 'Checkout') - self.assertEqual(steps[1]['run'], 'python3 -I .github/scripts/fork-ci-pressure.py start') + self.assertEqual(steps[1]['run'], step_run('start')) self.assertNotIn('if', steps[1]) end = steps[-1] - self.assertEqual(end['run'], 'python3 -I .github/scripts/fork-ci-pressure.py end') + self.assertEqual(end['run'], step_run('end')) self.assertTrue(re.fullmatch(r"\$\{\{ always\(\) && hashFiles\('\.github/scripts/fork-ci-pressure" r"\.py'\) != '' \}\}", end['if']), end['if']) self.assertEqual(end['env'], {'JOB_STATUS': '${{ job.status }}'}) diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py index 02d1e07df4b39..16bd3edf4e2a2 100644 --- a/.github/scripts/fork-ci-rerun.py +++ b/.github/scripts/fork-ci-rerun.py @@ -20,15 +20,19 @@ lost GitHub says the runner lost communication or was shut down (signs.lost), the job ended failure on that runner, none of its steps failed, no time limit was hit, and the probe's end step (probe_end_step) never completed; - memory full the job ended failure on a runner that is not GitHub-hosted (hosted_runner_groups: a kill there is - the job's own), and its own pressure note shows a box with a memory limit stalled (full) at least - memory_full.box_full_share_percent of the job AND OOM kills, memory.max hits or memory.high - throttling in it. A machine-wide OOM kill, or one event without the stall, is no sign; + memory full the job ended failure on a runner that is not GitHub-hosted (hosted_runner_groups), and its own + pressure note shows the job's own cgroup AND a box with a memory limit (its own or an ancestor, which + other jobs may share) stalled (full) at least memory_full.box_full_share_percent of the job, AND an + OOM kill of one of the job's own processes (its own cgroup's count, never an ancestor's: that holds + other jobs' kills), or memory.max hits or memory.high throttling in that box. A machine-wide or + another job's OOM kill, or one event without the job's own stall, is no sign; limit GitHub says the job or a step ran past its time limit (signs.limit, with the minutes), those minutes are that job's (or step's) own timeout-minutes in the workflow file at the run's commit (never the message's word alone: a step can print any number), the job (or that step) really ran that long, no other step failed, and the note shows the machine or the box stalled (CONFIG stalled). - The note counts only when the probe's end step completed (so the job wrote its own) and is the job's only one. + The note counts only when the probe's end step completed (so the job wrote its own), it is the job's only one, + the job has fewer than annotation_cap annotations (GitHub drops the rest, and the note comes last), and its + seconds fit the runner's own times of the probe's start and end steps (a note printed earlier cannot know them). One failed job without a sign (a test failure, no note, a garbled note, two notes) leaves the run red: fail closed. GitHub's "canceled by" (signs.cancelled_by) on any failed job leaves it red: a person's cancel stands. A run whose branch has moved on, or is gone, is not re-run. @@ -51,9 +55,11 @@ API = os.environ.get('GITHUB_API_URL') or 'https://api.github.com' NOTE_TITLE = 'fork-ci-pressure' # fork-ci-pressure.py's NOTE_HEAD = 'fork-ci-pressure v1 ' +PROBE_WARNS = 'fork-ci-pressure:' # how each of its warnings (and the probe steps' own stop warning) starts RED = ('failure', 'cancelled', 'timed_out') CHECK_RUN = re.compile(r'/check-runs/([0-9]{1,20})$') MARK = re.compile(r'') +ISSUE_MARK = '' # the first line of the tracking issue's body # Each sign's named groups: `minutes` is how long GitHub says the limit was; `step` (when there) names the step. GROUPS = {'lost': set(), 'limit': {'minutes'}, 'cancelled_by': set()} LONGEST = 60000 # characters of a comment (GitHub takes 65536) @@ -95,7 +101,8 @@ def check_config(cfg): if not isinstance(cfg, dict): return ['not a JSON object'] known = {'_doc', 'trust', 'workflow', 'followers', 'list_hours', 'catch_up_minutes', 'window_hours', 'skip_when', - 'hosted_runner_groups', 'probe_end_step', 'limit_slack_seconds', 'signs', 'stalled', 'memory_full', + 'hosted_runner_groups', 'probe_start_step', 'probe_end_step', 'limit_slack_seconds', + 'note_slack_seconds', 'annotation_cap', 'probe_limit_seconds', 'signs', 'stalled', 'memory_full', 'notice'} out += [f'unknown key {k!r}' for k in sorted(set(cfg) - known)] out += [f'missing key {k!r}' for k in sorted(known - {'_doc'} - set(cfg))] @@ -105,7 +112,8 @@ def check_config(cfg): out.append('workflow must be a workflow file name') if not names(cfg.get('followers'), empty=True): out.append('followers must be a list of job names') - for key in ('list_hours', 'catch_up_minutes', 'window_hours', 'limit_slack_seconds'): + for key in ('list_hours', 'catch_up_minutes', 'window_hours', 'limit_slack_seconds', 'note_slack_seconds', + 'annotation_cap', 'probe_limit_seconds'): if not number(cfg.get(key)) or cfg[key] <= 0: out.append(f'{key} must be a positive number') if number(cfg.get('list_hours')) and number(cfg.get('catch_up_minutes')) \ @@ -117,8 +125,11 @@ def check_config(cfg): out.append('skip_when must be {"variable": NAME, "is": [values]}') if not names(cfg.get('hosted_runner_groups')): out.append('hosted_runner_groups must be a list of runner group names') - if not (isinstance(cfg.get('probe_end_step'), str) and cfg['probe_end_step']): - out.append('probe_end_step must be the name of the probe\'s end step') + for key in ('probe_start_step', 'probe_end_step'): + if not (isinstance(cfg.get(key), str) and cfg[key]): + out.append(f'{key} must be the name of the probe\'s {key.split("_")[1]} step') + if cfg.get('probe_start_step') == cfg.get('probe_end_step'): + out.append('probe_start_step and probe_end_step must be two steps') signs = cfg.get('signs') if not isinstance(signs, dict) or set(signs) != set(GROUPS): out.append(f'signs must have exactly {", ".join(sorted(GROUPS))}') @@ -250,6 +261,22 @@ def note_of(cfg, job, annotations): """The job's own pressure note (fork-ci-pressure.py) as a dict, or (None, why).""" if not end_step_ran(cfg, job): return None, f'its "{cfg["probe_end_step"]}" step did not complete, so it has no note of its own' + if len(annotations) >= cfg['annotation_cap']: + # GitHub keeps only the first annotation_cap of a job: the end step's note, written last, may be the one + # dropped, and a note a step printed earlier would then stand alone. + return None, (f'{len(annotations)} annotations, GitHub\'s cap of {cfg["annotation_cap"]} a job: its own note ' + 'may have been dropped') + # A probe the step's own limit stopped (timeout ... || echo) ends its step green with only a warning, and so + # does one that raised: the job then has no note of its own, and a note a step printed earlier would stand + # alone, in a window the stop made wide. Either sign is enough; a step that prints one only fails closed. + spent = [took(s) for s in job.get('steps') or [] if s.get('name') == cfg['probe_end_step']] + if any(t is not None and t >= cfg['probe_limit_seconds'] for t in spent): + return None, (f'its "{cfg["probe_end_step"]}" step took {max(t for t in spent if t is not None)} s, the ' + f'probe\'s limit of {cfg["probe_limit_seconds"]} s: it was stopped, so it has no note of its own') + warned = [a for a in annotations if str(a.get('message') or '').startswith(PROBE_WARNS)] + if warned: + return None, (f'the probe warned ("{str(warned[0].get("message"))[:120]}"), so it may have left no note of ' + 'its own') notes = [a for a in annotations if a.get('title') == NOTE_TITLE] if not notes: return None, 'no pressure note (the note was dropped)' @@ -266,9 +293,32 @@ def note_of(cfg, job, annotations): if not (isinstance(note, dict) and isinstance(note.get('host'), dict) and isinstance(note.get('cgroups'), list) and all(isinstance(c, dict) for c in note['cgroups']) and isinstance(note.get('status'), str)): return None, 'a garbled pressure note' + window = probe_window(cfg, job) + if window is None: + return None, f'no "{cfg["probe_start_step"]}" and "{cfg["probe_end_step"]}" times to check the note against' + if not (number(note.get('seconds')) and window[0] <= note['seconds'] <= window[1]): + return None, (f'the note says {note.get("seconds")} s from start to end, but the job\'s own steps say ' + f'{window[0]}-{window[1]} s: not its end step\'s') return note, None +def probe_window(cfg, job): + """(fewest, most) seconds between the probe's start and end samples, by the runner's own step times (each sample + is taken inside its step), widened by note_slack_seconds for whole-second times; None without both steps' times. + A note printed well before the end step cannot know when the end step will run, so it does not fit. One printed + by the step just before the end step can (it knows the start time): only trust stops that one (only trusted + runs are read, and each is re-run once at most).""" + def times(name): + got = [(stamp(s.get('started_at')), stamp(s.get('completed_at'))) for s in job.get('steps') or [] + if s.get('name') == name] + return got[0] if len(got) == 1 and None not in got[0] else None + start, end = times(cfg['probe_start_step']), times(cfg['probe_end_step']) + if start is None or end is None: + return None + slack = cfg['note_slack_seconds'] + return end[0] - start[1] - slack, end[1] - start[0] + slack + + def at_least(value, bound): return number(value) and value >= bound @@ -292,20 +342,31 @@ def hosted(cfg, job): def memory_full(cfg, job, note): - """Evidence that the job's own box was full and stalled, or None. Only a failure on a runner that is not - GitHub-hosted, only a box with a memory limit, and only events together with that box's stall.""" + """Evidence that the job itself was held by a full box, or None. Only a failure on a runner that is not + GitHub-hosted. The note's first cgroup is the job's own (fork-ci-pressure.py: the runner's unit, which runs one + job at a time); the others are its ancestors with a memory limit, which several jobs may share. So: + - the job's own tasks stalled (full) at least box_full_share_percent of the job (its own cgroup's pressure); + - a box with a memory limit (its own cgroup or an ancestor) stalled that much too; + - and an OOM kill of one of the job's own processes (its own cgroup's oom_kill: memory.events counts a cgroup's + whole subtree, so an ancestor's count holds other jobs' kills, which never count), or that box hit its + memory.max or memory.high (the box was full, and the stall above says it held this job up).""" if job.get('conclusion') != 'failure' or note.get('status') != 'failure' or hosted(cfg, job): return None mf = cfg['memory_full'] + own = note['cgroups'][0] if note['cgroups'] else {} + if own.get('path') in (None, '', '/'): + return None # no cgroup of its own (the root is the whole machine) + if not at_least(own.get('full_share'), mf['box_full_share_percent']): + return None # the job's own tasks did not wait for memory for c in note['cgroups']: if not (number(c.get('max_mb')) or number(c.get('high_mb'))): continue # no limit: not a box if not at_least(c.get('full_share'), mf['box_full_share_percent']): continue where = (f'box {c.get("path")} (max {c.get("max_mb")} MB, high {c.get("high_mb")} MB), stalled (full) ' - f'{c["full_share"]}% of the job') - if at_least(c.get('oom_kill'), mf['oom_kills']): - return f'{where}: {c["oom_kill"]} OOM kill(s) in it' + f'{c["full_share"]}% of the job, the job\'s own cgroup {own["full_share"]}%') + if at_least(own.get('oom_kill'), mf['oom_kills']): + return f'{where}: {own["oom_kill"]} OOM kill(s) of the job\'s own processes' if at_least(c.get('max'), mf['max_events']): return f'{where}: hit its memory.max {c["max"]} time(s)' if at_least(c.get('high'), mf['high_events']): @@ -537,25 +598,42 @@ def decide(cfg, gh, run, env): class Notice: """The one tracking issue (CONFIG notice): the durable, visible record of every automatic re-run outside this - sweep's own run. Only an open issue github-actions opened with exactly that title is ours, never one a person - opened; with none open, the first re-run opens it. Each re-run (and each fleet-mode report) is one comment with a - marker line (MARK: run, attempt, what), so a pass never writes the same one twice and the count reads them.""" + sweep's own run. Only an open issue CONFIG notice author opened whose body starts with ISSUE_MARK is ours, never + one a person opened (whatever its title); with none open, the first re-run opens it, after looking once more. If + more than one is open (two passes opened one at once, or a pass stopped before closing its own), the lowest is + kept and the others are closed as its duplicates by load(), which every pass runs (its 24 h count reads the + issue at the end of the pass, if not before). Each re-run (and each fleet-mode report) is one comment with a + marker line (MARK: run, attempt, what), so a pass never writes the same one twice and the count reads them. The + markers on a closed duplicate drop out of the count: the count only informs, and never-twice comes from the + run's own attempts.""" def __init__(self, cfg, gh, now): self.cfg, self.gh, self.now, self.hours = cfg['notice'], gh, now, cfg['window_hours'] - self.issue, self.comments = None, None + self.issue, self.comments, self.unclosed = None, None, [] def _ours(self): n = self.cfg listed = self.gh.pages(f'issues?state=open&creator={urllib.parse.quote(n["author"], safe="")}' f'&sort=created&direction=asc') return sorted(int(i['number']) for i in listed if isinstance(i, dict) and 'pull_request' not in i - and i.get('title') == n['title'] and login(i.get('user')) == n['author']) + and str(i.get('body') or '').startswith(ISSUE_MARK) and login(i.get('user')) == n['author']) + + def _close(self, number, keep): + # Closed first, then the comment: a close that keeps failing posts nothing, so a later pass that retries it + # never adds a second "Duplicate of"; and a closed issue is never ours again, so it gets the comment once. + self.gh.call(f'issues/{number}', method='PATCH', data={'state': 'closed', 'state_reason': 'not_planned'}) + self.gh.call(f'issues/{number}/comments', method='POST', data={'body': f'Duplicate of #{keep}.'}) def load(self, since=None): if self.comments is not None: return ours = self._ours() + for extra in ours[1:]: + try: + self._close(extra, ours[0]) + except (OSError, ValueError) as e: + # ours[0] is used either way, and the next pass tries again; the pass says so (a warning). + self.unclosed.append(f'#{extra}: {e.__class__.__name__}: {e}') self.issue = ours[0] if ours else None start = iso(min(self.now() - self.hours * 3600, since or self.now())) self.comments = [] if self.issue is None else [ @@ -581,24 +659,26 @@ def count(self): return sum(1 for (_, _, what), at in self.marks().items() if what == 'rerun' and (stamp(at) or 0) >= since) def where(self): - return f'issue #{self.issue}' if self.issue else 'no tracking issue yet' + return f'issue #{self.issue}' if self.issue else 'no open tracking issue' def _open(self): n = self.cfg got = self.gh.call('issues', method='POST', data={ 'title': n['title'], 'labels': n['labels'], - 'body': ('Each comment below is one automatic re-run of a failed run (or, in fleet mode, a run that ' - 'would have had one), with its jobs, the signs and the memory pressure numbers. Many of them ' - 'mean a machine that stalls or fills up: fix that, the re-run only hides it for one attempt. ' - 'Closing this issue is fine; the next re-run opens a new one. Rules and runbook: ' + 'body': (f'{ISSUE_MARK}\nEach comment below is one automatic re-run of a failed run (or, in fleet ' + 'mode, a run that would have had one), with its jobs, the signs and the memory pressure ' + 'numbers. Many of them mean a machine that stalls or fills up: fix that, the re-run only ' + 'hides it for one attempt. Closing this issue is fine; the next re-run opens a new one. ' + 'Rules and runbook: ' '`.github/fork-ci-rerun.md`.')}) mine = int(got['number']) # A dispatched pass may run beside a scheduled one (fork-ci-rerun.yml); if both opened one, the lowest wins - # and the other is closed, pointing at it, so there is only ever one. + # and the other is closed, pointing at it. A pass that stops before this leaves its own open: the next + # pass's load() closes it. ours = self._ours() if ours and ours[0] < mine: - self.gh.call(f'issues/{mine}/comments', method='POST', data={'body': f'Duplicate of #{ours[0]}.'}) - self.gh.call(f'issues/{mine}', method='PATCH', data={'state': 'closed', 'state_reason': 'not_planned'}) + if mine in ours: # not already closed by another pass's load() + self._close(mine, ours[0]) return ours[0] return mine @@ -616,7 +696,10 @@ def add(self, run, what, text): f'{quoted(run.get("event"))} `{quoted(run.get("head_branch"))}` `{str(run.get("head_sha"))[:12]}`\n\n' + ''.join(f'- {quoted(part.strip())}\n' for part in text.split(' | ')))[:LONGEST] if self.issue is None: - self.issue = self._open() + # Look once more first: an issue this pass opened whose reply was lost, or one another pass opened + # since load(), is the one to use. + ours = self._ours() + self.issue = ours[0] if ours else self._open() got = self.gh.call(f'issues/{self.issue}/comments', method='POST', data={'body': body}) self.comments.append(dict(got if isinstance(got, dict) else {}, body=body, created_at=(got or {}).get('created_at') or iso(self.now()))) @@ -725,6 +808,9 @@ def record(): count = (f'{asked} run(s) re-run by this pass; the tracking issue could not be read for the ' f'{cfg["window_hours"]} h count ({e.__class__.__name__}: {e}): red, the next pass tries again') say(f'fork-ci re-run: {count}', out) + for why in notice.unclosed: + say(f'::warning::fork-ci re-run: a second tracking issue could not be closed as a duplicate of ' + f'{notice.where()} ({why}); the next pass tries again', out) lines = ['### fork-ci re-run', '', count, ''] if rows: lines += ['| Run | Outcome | Why |', '| --- | --- | --- |'] + rows diff --git a/.github/scripts/fork-ci-rerun.test.py b/.github/scripts/fork-ci-rerun.test.py index d0bc7bb47063c..2f6d0aa43d18d 100644 --- a/.github/scripts/fork-ci-rerun.test.py +++ b/.github/scripts/fork-ci-rerun.test.py @@ -87,7 +87,10 @@ def ann(message, level='failure', title='', path='.github'): return {'message': message, 'annotation_level': level, 'title': title, 'path': path} -def note(host=None, cgroups=None, status='failure', seconds=600): +FIT = 'fit' # a note's seconds: the fake GitHub puts in the time between the job's own probe steps + + +def note(host=None, cgroups=None, status='failure', seconds=FIT): """A pressure note as fork-ci-pressure.py writes it, decoded as GitHub hands it back.""" h = {'full_share': 0.0, 'some_share': 0.5, 'full_avg10': 0.0, 'full_avg60': 0.0, 'full_avg300': 0.0, 'some_avg60': 0.1, 'some_avg300': 0.1, 'available_mb': 9000, 'swap_used_mb': 0, 'oom_kill': 0} @@ -97,6 +100,29 @@ def note(host=None, cgroups=None, status='failure', seconds=600): return ann(cr.NOTE_HEAD + body, level='notice', title=cr.NOTE_TITLE) +def fitted(annotations, j): + """The annotations with a FIT note's seconds set to the middle of the job's probe window (600 without one).""" + out = [] + for a in annotations: + if a.get('title') == cr.NOTE_TITLE and a['message'].startswith(cr.NOTE_HEAD): + try: + body = json.loads(a['message'][len(cr.NOTE_HEAD):]) + except ValueError: + body = None # a garbled note stays as it is + if isinstance(body, dict) and body.get('seconds') == FIT: + window = cr.probe_window(CFG, j) if j else None + body['seconds'] = int(sum(window) / 2) if window else 600 + a = dict(a, message=cr.NOTE_HEAD + json.dumps(body, separators=(',', ':'))) + out.append(a) + return out + + +def tracking(number, state='open', user=BOT, body=None, **kw): + """A tracking issue as GitHub lists it: the bot's, its body starting with the script's marker.""" + return dict({'number': number, 'title': TITLE, 'state': state, 'user': {'login': user}, 'comments': [], + 'body': cr.ISSUE_MARK + '\nEach comment below ...' if body is None else body}, **kw) + + def box(**kw): b = {'path': '/fleet.slice/job-1.service', 'high_mb': 6144, 'max_mb': 8192, 'current_mb': 3000, 'high': 0, 'max': 0, 'oom': 0, 'oom_kill': 0, 'full_share': 0.0} @@ -245,7 +271,8 @@ def paged(items): return copy.deepcopy(got) m = re.fullmatch(r'check-runs/(\d+)/annotations', base) if m: - return paged(self.ann.get(int(m[1]), [])) + j = next((j for js in self.jobs.values() for j in js if j['id'] == int(m[1])), None) + return paged(fitted(self.ann.get(int(m[1]), []), j)) m = re.fullmatch(r'branches/(.+)', base) if m: if m[1] not in self.heads: @@ -338,7 +365,7 @@ def test_a_full_box_is_rerun_with_its_numbers_in_the_log(self): code, out, text = sweep(gh) self.assertReran(gh, out) self.assertIn('memory full', out) - self.assertRegex(out, r'machine: full [0-9.]+% some [0-9.]+% of 600 s') + self.assertRegex(out, r'machine: full [0-9.]+% some [0-9.]+% of 65 s') self.assertIn('box /fleet.slice/job-1.service: max', out) self.assertIn('memory full', text) @@ -359,6 +386,65 @@ def test_one_event_or_a_stall_alone_is_no_memory_sign(self): gh = one([ann(EXIT_137), n]) self.assertLeft(gh, *sweep(gh)[:2], 'no infrastructure sign') + def test_only_the_jobs_own_oom_kill_counts(self): + """memory.events counts a cgroup's whole subtree: a shared slice's OOM kill can be another job's. The note's + first cgroup is the job's own (one job a runner at a time): only its OOM kills are the job's, and only its + own stall says the job waited for memory.""" + own = dict(path='/fleet.slice/job-1.service', high_mb=None, max_mb=None) + shared = dict(path='/fleet.slice', high_mb=None, max_mb=8192) + rerun = { + 'its own OOM kill and stall in a stalled slice': [box(**own, oom_kill=1, full_share=30.0), + box(**shared, oom_kill=1, full_share=30.0)], + 'its own stall while the slice hit memory.max': [box(**own, full_share=30.0), + box(**shared, max=2, full_share=30.0)], + 'its own stall while the slice was throttled': [box(**own, full_share=30.0), + box(**dict(shared, high_mb=6144), high=5, full_share=30.0)], + } + left = { + "another job's OOM kill, its own cgroup calm": [box(**own), box(**shared, oom_kill=2, full_share=40.0)], + "another job's OOM kill while it stalled too": [box(**own, full_share=30.0), + box(**shared, oom_kill=2, full_share=40.0)], + 'its own OOM kill, its own tasks not stalled': [box(**own, oom_kill=1, full_share=2.0), + box(**shared, oom_kill=1, full_share=40.0)], + 'its own OOM kill and stall, no box stalled': [box(**own, oom_kill=1, full_share=30.0), + box(**shared, oom_kill=1, full_share=2.0)], + 'no cgroup of its own (the root)': [box(path='/', oom_kill=1, full_share=40.0)], + 'no cgroup read at all': [], + } + for label, boxes in list(rerun.items()) + list(left.items()): + with self.subTest(label): + gh = one([ann(EXIT_137), note(cgroups=boxes)]) + code, out, _ = sweep(gh) + if label in rerun: + self.assertReran(gh, out) + self.assertIn("the job's own cgroup 30.0%", out) + else: + self.assertLeft(gh, code, out, 'no infrastructure sign') + + def test_each_memory_threshold_is_where_the_config_puts_it(self): + mf = CFG['memory_full'] + at, under = mf['box_full_share_percent'], mf['box_full_share_percent'] - 0.1 + own = dict(path='/fleet.slice/job-1.service', high_mb=None, max_mb=None) + shared = dict(path='/fleet.slice', max_mb=8192) + cases = { + 'the box stalled just enough': ([box(oom_kill=mf['oom_kills'], full_share=at)], True), + 'the box stalled just too little': ([box(oom_kill=mf['oom_kills'], full_share=under)], False), + 'its own cgroup stalled just enough': ([box(**own, oom_kill=1, full_share=at), + box(**shared, full_share=50.0)], True), + 'its own cgroup stalled just too little': ([box(**own, oom_kill=1, full_share=under), + box(**shared, full_share=50.0)], False), + 'memory.max hit just enough': ([box(max=mf['max_events'], full_share=50.0)], True), + 'memory.max hit too little': ([box(max=mf['max_events'] - 1, full_share=50.0)], False), + 'memory.high hit just enough': ([box(high=mf['high_events'], full_share=50.0)], True), + 'memory.high hit too little': ([box(high=mf['high_events'] - 1, full_share=50.0)], False), + 'OOM kills just too few': ([box(oom_kill=mf['oom_kills'] - 1, full_share=50.0)], False), + } + for label, (boxes, qualifies) in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), note(cgroups=boxes)]) + code, out, _ = sweep(gh) + (self.assertReran if qualifies else lambda gh, out: self.assertLeft(gh, code, out))(gh, out) + def test_an_oom_kill_on_a_github_hosted_runner_never_qualifies(self): for group in (HOSTED, None, ''): with self.subTest(group=group): @@ -388,6 +474,72 @@ def test_a_timed_out_run_is_a_candidate(self): conclusion='timed_out') self.assertReran(gh, sweep(gh)[1]) + def test_each_stall_threshold_is_where_the_config_puts_it(self): + st = CFG['stalled'] + cases = { + 'the machine stalled just enough': (dict(host={'full_share': st['full_share_percent']}), True), + 'the machine stalled just too little': (dict(host={'full_share': st['full_share_percent'] - 0.1}), False), + 'a box stalled just enough': (dict(cgroups=[box(full_share=st['full_share_percent'])]), True), + 'a box stalled just too little': (dict(cgroups=[box(full_share=st['full_share_percent'] - 0.1)]), False), + 'an ancestor box stalled': (dict(cgroups=[box(), box(path='/fleet.slice', full_share=50.0)]), True), + 'the 5-minute average just enough': (dict(host={'full_avg300': st['full_avg300_percent']}), True), + 'the 5-minute average just too little': (dict(host={'full_avg300': st['full_avg300_percent'] - 0.1}), + False), + 'only the 10-second average high': (dict(host={'full_avg10': 90.0, 'full_avg60': 90.0}), False), + 'only some tasks stalled': (dict(host={'some_share': 90.0, 'some_avg300': 90.0}), False), + } + for label, (kw, qualifies) in cases.items(): + with self.subTest(label): + gh = one([ann(STEP_LIMIT), note(**kw)], dict(steps=ran('failure', took=600))) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, 'not stalled') + + def test_the_time_limit_slack_is_where_the_config_puts_it(self): + slack = CFG['limit_slack_seconds'] + cases = { + 'the job ran its minutes less the slack': (JOB_LIMIT, dict(conclusion='cancelled', took=900 - slack, + steps=ran('cancelled', took=820)), True), + 'the job ran a second less': (JOB_LIMIT, dict(conclusion='cancelled', took=900 - slack - 1, + steps=ran('cancelled', took=820)), False), + 'the step ran its minutes less the slack': (STEP_LIMIT, dict(steps=ran('failure', took=600 - slack)), + True), + 'the step ran a second less': (STEP_LIMIT, dict(steps=ran('failure', took=600 - slack - 1)), False), + } + for label, (message, kw, qualifies) in cases.items(): + with self.subTest(label): + gh = one([ann(message), STALLED], kw) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, "GitHub's time-limit text, but") + + def test_a_step_limit_needs_the_step_itself_ended_red(self): + # GitHub ends a step that ran past its limit cancelled or failed; a step that passed or was skipped did not + # hit it, however long it took. + for conclusion, qualifies in (('failure', True), ('cancelled', True), ('success', False), ('skipped', False)): + with self.subTest(conclusion): + steps = ran('failure', took=600) + steps[2] = step('Test', conclusion, took=600, at=10) + gh = one([ann(STEP_LIMIT), STALLED], dict(steps=steps)) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, "no step 'Test' ran 10 min and ended red") + + def test_a_time_limit_message_counts_only_as_a_whole_runner_message(self): + for label, a in {'inside a longer line': ann('Error: ' + JOB_LIMIT), 'with text after': ann(JOB_LIMIT + ' x'), + 'as a warning': ann(JOB_LIMIT, 'warning'), 'with a title': ann(JOB_LIMIT, title='Error'), + 'on a file': ann(JOB_LIMIT, path='src/x.test.ts')}.items(): + with self.subTest(label): + gh = one([a, ann(CANCELED), STALLED], dict(conclusion='cancelled', steps=ran('cancelled', took=880), + took=900)) + self.assertLeft(gh, *sweep(gh)[:2]) + def test_a_time_limit_message_needs_the_time_really_spent(self): cases = { 'the job ran 3 of its 15 minutes': (JOB_LIMIT, dict(steps=ran('cancelled', took=170), took=180)), @@ -442,6 +594,54 @@ def test_the_workflow_reader_reads_every_limit_of_the_real_workflow(self): self.assertEqual(cr.limit_of(wf, 'Lint'), real['lint']['timeout-minutes']) self.assertIsNone(cr.limit_of(wf, 'Lint', 'Check'), 'a step without timeout-minutes has none of its own') + def test_the_reader_takes_only_values_it_is_sure_of(self): + cases = { + 'plain': ('10', '10'), 'a comment after it': ("10 # the step's own", '10'), + 'single-quoted': ("'Note it'", 'Note it'), 'a quote in single quotes': ("'it''s'", "it's"), + 'empty single quotes': ("''", ''), 'double-quoted': ('"Test"', 'Test'), 'empty double quotes': ('""', ''), + 'an escape in double quotes': ('"a\\tb"', None), 'a block': ('|', None), 'a folded block': ('>-', None), + 'an anchor': ('&x 10', None), 'a list': ('[1]', None), 'a map': ('{a: 1}', None), 'a lone quote': ("'", None), + 'mismatched quotes': ('\'a"', None), 'a quote only at the end': ("x'", "x'"), + 'a double quote only at the end': ('x"', 'x"'), 'a bar inside': ('a|', 'a|'), 'empty': ('', None), + } + for label, (raw, want) in cases.items(): + with self.subTest(label): + self.assertEqual(cr.scalar(raw), want) + text = WORKFLOW.replace(' timeout-minutes: 15', ' timeout-minutes: ${{ inputs.minutes || 15 }}') + ( + 'env:\n A: 1\n B:\n C:\n - name: x\n timeout-minutes: 3\n') + wf = cr.workflow_limits(text) # a key after `jobs:` ends the jobs; a limit that is not a number is none + self.assertEqual((sorted(wf), wf['test']['minutes'], cr.limit_of(wf, 'Test (1)', 'Test')), + (['lint', 'test'], None, 10)) + self.assertEqual([s['name'] for s in wf['test']['steps']], + ['Checkout', 'Note memory pressure at the start', 'Test', END_STEP]) + + def test_a_name_matches_only_as_github_shows_it(self): + wf = cr.workflow_limits(WORKFLOW) + self.assertEqual((cr.limit_of(wf, 'Test (1)'), cr.limit_of(wf, 'Test'), cr.limit_of(wf, 'Test (1)', 'Test')), + (15, 15, 10)) + self.assertIsNone(cr.limit_of(wf, 'Pre Test (1)'), 'a longer name is another job') + self.assertIsNone(cr.limit_of(wf, 'Test (1) x')) + self.assertIsNone(cr.limit_of(wf, 'Test (1)', 'Test (2)'), "a step's name never carries matrix values") + self.assertIsNone(cr.limit_of(wf, 'Test (1)', 'Pre Test')) + + def test_the_workflow_is_read_once_at_the_runs_own_commit(self): + gh = Fake([run()]) + limits = cr.workflow_reader(CFG, gh, run()) + self.assertEqual((limits('Test (1)', 'Test'), limits('Test (2)'), limits('Lint')), (10, 15, 20)) + self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('contents/')), 1) + for sha in ('main', SHA + '&ref=main', 'v' + SHA): # not exactly one commit: could change under the reader + with self.assertRaisesRegex(ValueError, 'no commit to read the workflow at'): + cr.workflow_reader(CFG, gh, dict(run(), head_sha=sha))('Test (1)') + self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('contents/')), 1) + real = gh.call + gh.call = lambda path, method='GET', data=None: dict(real(path, method, data), encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'not readable through the contents API'): + cr.workflow_reader(CFG, gh, run())('Test (1)') + + def test_a_job_with_no_note_at_all_is_left_red(self): + gh = one([ann(EXIT_137)]) + self.assertLeft(gh, *sweep(gh)[:2], 'no pressure note') + def test_a_stalled_note_without_its_status_is_no_limit_sign(self): n = dict(STALLED, message=STALLED['message'].replace('"status":"failure",', '')) self.assertNotIn('status', n['message']) @@ -554,6 +754,85 @@ def test_a_note_counts_only_when_the_job_wrote_it_and_it_is_the_only_one(self): gh = one([ann(EXIT_137), self.FORGED], dict(steps=steps)) self.assertLeft(gh, *sweep(gh)[:2], 'did not complete') + def test_a_note_must_fit_the_runners_own_times_of_the_probe_steps(self): + """A step can print a note in the runner's own form and keep the end step from writing its own (or have the + real one dropped at GitHub's cap). Its seconds must fit the time between the two probe steps by the runner's + own record, which a step that prints a note earlier cannot know.""" + window = cr.probe_window(CFG, job(11)) + self.assertEqual(window, (70 - 10 - CFG['note_slack_seconds'], 75 - 5 + CFG['note_slack_seconds'])) + full = [box(oom_kill=1, full_share=30.0)] + for seconds, qualifies in ((window[0], True), (window[1], True), (window[0] - 1, False), (window[1] + 1, False), + (600, False), (None, False), ('65', False), (True, False)): + with self.subTest(seconds=seconds): + gh = one([ann(EXIT_137), note(cgroups=full, seconds=seconds)]) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, "not its end step's") + start = 'Note memory pressure at the start' + for label, steps in {'no start step': [s for s in ran() if s['name'] != start], + 'two start steps': ran()[:2] + ran()[1:], + 'a start step that never completed': [s if s['name'] != start else + step(start, None, 'in_progress', at=5) + for s in ran()]}.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), note(cgroups=full, seconds=65)], dict(steps=steps)) + self.assertLeft(gh, *sweep(gh)[:2], 'times to check the note against') + + def test_at_githubs_annotation_cap_no_note_is_trusted(self): + """GitHub keeps the first annotation_cap annotations of a job. The end step's note comes last, so at the cap + the real one may be gone and a step's printed note stands alone: refused, whatever it says.""" + cap = CFG['annotation_cap'] + lint = [ann(f'line {i}: unused', 'warning', path='src/x.ts') for i in range(cap)] + for others, qualifies in ((cap - 2, True), (cap - 1, False), (cap, False)): + with self.subTest(annotations=others + 1): + gh = one(lint[:others] + [self.FORGED]) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, f"GitHub's cap of {cap}") + + def test_a_stopped_end_probe_leaves_no_note_to_read(self): + """The end step runs its probe under `timeout -k 5 $FORK_CI_PROBE_SECONDS ... || echo "::warning::..."`. A + probe stopped at that limit (or one that raised) ends the step green, with a warning and no note of its own. + A note a step printed earlier is then the only one, and the stop widens the window enough for it to fit + (the review's record: start step 1 s, end step 45 s, the printed note's seconds in the window): refused, by + the end step's own time or by the probe's warning. Old code: re-run.""" + limit = CFG['probe_limit_seconds'] + + def end_took(seconds): + return ran()[:-1] + [step(END_STEP, at=70, took=seconds)] + self.assertEqual(cr.probe_window(CFG, job(11, steps=end_took(45))), (57, 113)) # wide: a guess lands in it + for seconds, qualifies in ((limit - 1, True), (limit, False), (45, False), (limit + 600, False)): + with self.subTest(end_step_took=seconds): + gh = one([ann(EXIT_137), self.FORGED], dict(steps=end_took(seconds))) + code, out, _ = sweep(gh) + if qualifies: + self.assertReran(gh, out) + else: + self.assertLeft(gh, code, out, f"the probe's limit of {limit} s") + self.assertEqual(gh.comments(), []) + warnings = { + 'the end probe stopped': 'fork-ci-pressure: end stopped (exit 124), no numbers', + 'the end probe killed': 'fork-ci-pressure: end stopped (exit 137), no numbers', + 'the end probe raised': 'fork-ci-pressure: end: ValueError: Expecting value: line 1 column 1 (char 0)', + 'the start probe stopped': 'fork-ci-pressure: start stopped (exit 124), no numbers; the job goes on', + 'printed by a step, at notice level': ('fork-ci-pressure: anything', 'notice'), + } + for label, w in warnings.items(): + text, level = w if isinstance(w, tuple) else (w, 'warning') + for where in ('before', 'after'): + with self.subTest(label, where=where): + anns = [ann(EXIT_137), ann(text, level)] + anns.insert(1 if where == 'before' else 2, self.FORGED) + gh = one(anns) + code, out, _ = sweep(gh) + self.assertLeft(gh, code, out, 'the probe warned') + gh = one([ann(EXIT_137), ann('pressure: fork-ci-pressure: inside a line', 'warning'), self.FORGED]) + self.assertReran(gh, sweep(gh)[1]) # only a message that starts with the probe's words is its warning + class Cancel(Rerun): def test_a_run_a_person_cancelled_never_qualifies(self): @@ -721,19 +1000,124 @@ def test_every_rerun_is_a_comment_on_one_issue_opened_once(self): sweep(gh) self.assertEqual((len(gh.issues), len(issue['comments'])), (1, 3)) - def test_only_an_open_issue_the_bot_opened_with_exactly_the_title_is_ours(self): + def test_only_an_open_issue_the_bot_opened_with_the_marker_is_ours(self): gh = lost() gh.issues = [ - {'number': 3, 'title': TITLE, 'state': 'open', 'user': {'login': OWNER}, 'comments': []}, - {'number': 4, 'title': TITLE + ' (old)', 'state': 'open', 'user': {'login': BOT}, 'comments': []}, - {'number': 5, 'title': TITLE, 'state': 'closed', 'user': {'login': BOT}, 'comments': []}, - {'number': 6, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': [], - 'pull_request': {}}, + tracking(3, user=OWNER), # a person's, marker and all + tracking(4, body=f'Seen it.\n{cr.ISSUE_MARK}'), # the marker not first: quoted, not ours + tracking(5, state='closed'), + tracking(6, pull_request={}), + tracking(7, title='Secret sweep failed', body='The sweep failed.'), # the bot's other issues ] sweep(gh) [ours] = [i for i in gh.issues if i['comments']] - self.assertNotIn(ours['number'], (3, 4, 5, 6)) + self.assertNotIn(ours['number'], (3, 4, 5, 6, 7)) self.assertEqual((ours['title'], ours['user']['login']), (TITLE, BOT)) + self.assertTrue(ours['body'].startswith(cr.ISSUE_MARK + '\n')) + # Renaming the issue keeps it ours: the marker, not the title, finds it. + ours['title'] = 'CI re-runs (renamed)' + gh.runs[3] = run(3, created=T0 - 60) + gh.jobs[3] = [job(31, steps=died())] + gh.ann[31] = [ann(LOST)] + sweep(gh) + self.assertEqual((len(gh.issues), len(ours['comments'])), (6, 2)) + + def test_a_pass_whose_issue_reply_was_lost_opens_no_second_one(self): + # GitHub took the POST, the reply never came back: the pass stops red with the issue open. The next pass + # finds it by its marker and uses it (#707's code found it by its title, which also held here). + gh = lost() + real = gh.call + def call(path, method='GET', data=None): + got = real(path, method, data) + if method == 'POST' and path == 'issues': + raise urllib.error.URLError('issue: reply lost') + return got + gh.call = call + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), len(gh.issues), gh.comments()), (1, [], 1, []), out) + gh.call = real + for _ in range(2): # runs again: the open issue is found by its marker, never a second one + code, out, _ = sweep(gh) + self.assertEqual(code, 0, out) + self.assertEqual((len(gh.posts()), len(gh.issues), len(gh.comments())), (1, 1, 1), out) + + def test_an_issue_another_pass_opened_since_this_one_loaded_is_used(self): + gh = lost() + real = gh.call + def call(path, method='GET', data=None): + got = real(path, method, data) + if method == 'GET' and path.startswith('issues?') and not gh.issues: + gh.issues.append(tracking(41)) # a dispatched pass opened it just after this pass's load() + return got + gh.call = call + code, out, _ = sweep(gh) + self.assertEqual((code, [i['number'] for i in gh.issues], len(gh.issues[0]['comments'])), (0, [41], 1), out) + self.assertNotIn(('POST', 'issues'), gh.calls) + + def test_two_open_issues_are_made_one_by_the_next_pass(self): + # A pass stopped after opening its own and before closing it as a duplicate: both are open. + gh = lost() + gh.issues = [tracking(12), tracking(10)] + code, out, _ = sweep(gh) + self.assertEqual(code, 0, out) + first, second = sorted(gh.issues, key=lambda i: i['number']) + self.assertEqual((first['state'], len(first['comments'])), ('open', 1)) + self.assertEqual((second['state'], second['state_reason'], [c['body'] for c in second['comments']]), + ('closed', 'not_planned', ['Duplicate of #10.'])) + + def test_a_duplicate_that_cannot_be_closed_still_leaves_one_record(self): + # Each pass tries the close again; a close that keeps failing posts no "Duplicate of" (old code: one more + # every pass, 10 minutes apart), the pass warns, and the close that works posts it once. + gh = lost() + gh.issues = [tracking(10), tracking(12)] + real = gh.call + def call(path, method='GET', data=None): + if method == 'PATCH': + raise urllib.error.URLError('patch: reset') + return real(path, method, data) + gh.call = call + for n in range(3): + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.issue(10)['comments'])), (0, 1, 1), out) + self.assertEqual((gh.issue(12)['state'], gh.issue(12)['comments']), ('open', []), f'pass {n + 1}') + self.assertIn('::warning::fork-ci re-run: a second tracking issue could not be closed as a duplicate of ' + 'issue #10 (#12: URLError', out) + gh.call = real + for _ in range(2): # runs again: closes it this time, says so once, and writes nothing twice + code, out, _ = sweep(gh) + self.assertNotIn('could not be closed', out) + self.assertEqual((gh.issue(12)['state'], [c['body'] for c in gh.issue(12)['comments']], + len(gh.issue(10)['comments'])), ('closed', ['Duplicate of #10.'], 1)) + + def test_every_pass_closes_a_duplicate_even_with_nothing_to_record(self): + # The runbook's promise: a pass reads the issue for its 24 h count at its end, so the next pass closes an + # extra one, even with no red run (or only a fleet-mode report) to record. + gh = Fake([]) + gh.issues = [tracking(12), tracking(10)] + code, out, _ = sweep(gh) + self.assertEqual((code, gh.issue(10)['state'], gh.issue(12)['state']), (0, 'open', 'closed'), out) + self.assertIn('(issue #10)', out) + + def test_a_pass_whose_own_duplicate_another_pass_closed_leaves_it_alone(self): + # Two passes open one each at once; the other pass's load() closes this pass's issue (#41) before this + # pass reads the open ones again. This pass uses #10 and neither closes #41 again nor posts a second + # "Duplicate of" (the `mine in ours` guard). + gh = lost() + real = gh.call + def call(path, method='GET', data=None): + got = real(path, method, data) + if method == 'POST' and path == 'issues': + gh.issues.insert(0, tracking(10)) + gh.issue(41).update(state='closed', state_reason='not_planned') # the other pass's load() + gh.issue(41)['comments'].append({'id': 1, 'body': 'Duplicate of #10.', 'user': {'login': BOT}, + 'created_at': iso(gh.now)}) + return got + gh.call = call + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), len(gh.issue(10)['comments'])), (0, ['actions/runs/1/rerun-failed-jobs'], + 1), out) + self.assertEqual([c['body'] for c in gh.issue(41)['comments']], ['Duplicate of #10.']) + self.assertNotIn(('PATCH', 'issues/41'), gh.calls) def test_two_passes_opening_it_at_once_keep_the_lowest(self): gh = lost() @@ -742,8 +1126,7 @@ def call(path, method='GET', data=None): got = real(path, method, data) if method == 'POST' and path == 'issues' and len(gh.issues) == 1: # A dispatched pass opened one too, just before this one. - gh.issues.insert(0, {'number': 10, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, - 'comments': []}) + gh.issues.insert(0, tracking(10)) return got gh.call = call sweep(gh) @@ -753,7 +1136,7 @@ def call(path, method='GET', data=None): def test_the_24_hour_count_reads_every_comment_past_hundreds_of_reruns(self): gh = Fake([]) - gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + gh.issues = [tracking(2)] comments = gh.issues[0]['comments'] for n in range(350): # one every 4 minutes: 350 in the last 24 h comments.append({'id': n, 'user': {'login': BOT}, 'created_at': iso(T0 - 26 * 3600 + n * 240), @@ -774,7 +1157,7 @@ def test_the_24_hour_count_reads_every_comment_past_hundreds_of_reruns(self): def test_an_issue_that_cannot_be_read_ends_the_pass_red_and_asks_nothing(self): for label, gh in (('no run to look at', Fake([])), ('a run that qualifies', lost())): with self.subTest(label): - gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + gh.issues = [tracking(2)] gh.comments_fail = True code, out, text = sweep(gh) self.assertEqual((code, gh.posts(), gh.comments()), (1, [], []), out) @@ -804,8 +1187,8 @@ def test_quoted_text_pings_no_one_and_fakes_no_record(self): def test_no_issue_yet_counts_zero_and_opens_nothing(self): code, out, text = sweep(Fake([])) self.assertEqual(code, 0, out) - self.assertIn('0 run(s) re-run by this pass; 0 automatic re-run(s) asked in the last 24 h (no tracking ' - 'issue yet)', out) + self.assertIn('0 run(s) re-run by this pass; 0 automatic re-run(s) asked in the last 24 h (no open ' + 'tracking issue)', out) self.assertIn('### fork-ci re-run', text) def test_fleet_mode_reports_once_and_does_not_rerun(self): @@ -997,6 +1380,12 @@ def bad(change): 'a listing shorter than the window': lambda c: c.__setitem__('list_hours', 0.5), 'no hosted group': lambda c: c.__setitem__('hosted_runner_groups', []), 'no end step': lambda c: c.__setitem__('probe_end_step', ''), + 'no start step': lambda c: c.pop('probe_start_step'), + 'one step for both': lambda c: c.__setitem__('probe_start_step', c['probe_end_step']), + 'no note slack': lambda c: c.__setitem__('note_slack_seconds', 0), + 'no annotation cap': lambda c: c.__setitem__('annotation_cap', None), + 'no probe limit': lambda c: c.pop('probe_limit_seconds'), + 'a zero probe limit': lambda c: c.__setitem__('probe_limit_seconds', 0), 'a notice by a person': lambda c: c['notice'].__setitem__('author', OWNER), 'a notice without a title': lambda c: c['notice'].__setitem__('title', ''), } @@ -1047,10 +1436,12 @@ def test_a_probe_notice_round_trips(self): self.assertNotIn('\n', data) message = data.replace('%0A', '\n').replace('%0D', '\r').replace('%25', '%') # what the runner does anns = [ann(EXIT_137), ann(message, 'notice', cr.NOTE_TITLE)] - got, why = cr.note_of(CFG, job(1), anns) + j = job(1, steps=ran(took=300)) # its probe steps 297 to 313 s apart: the note's 300 s fit + got, why = cr.note_of(CFG, j, anns) self.assertIsNone(why) self.assertEqual(got['cgroups'][0]['path'], '/a%b\nc') - self.assertEqual(cr.job_sign(CFG, job(1), anns)[0], 'memory full') + self.assertEqual(cr.job_sign(CFG, j, anns)[0], 'memory full') + self.assertIn('not its end step', cr.note_of(CFG, job(1), anns)[1]) # 300 s on a job of 57 to 73 s self.assertEqual((probe.NOTE_TITLE, probe.NOTE_HEAD), (cr.NOTE_TITLE, cr.NOTE_HEAD)) diff --git a/.github/scripts/fork-fleet.py b/.github/scripts/fork-fleet.py index cf44c2bdadbd3..abab1768e0c35 100644 --- a/.github/scripts/fork-fleet.py +++ b/.github/scripts/fork-fleet.py @@ -369,8 +369,9 @@ def made_by_the_sync(cfg, sha, run=subprocess.run): in this repository and moves main only when that push worked. A tag here can only be made by a holder of push rights (the owner, the sync's token): nothing an upstream commit holds (its message, author, files, a branch or pull request upstream) makes or removes one. And a tag only ever sends a run to - GitHub-hosted runners, so the worst a stray tag does is that. Read anonymously (the job hook has no token); any - doubt (no full sha, git failing, a timeout) refuses.""" + GitHub-hosted runners, so the worst a stray tag does is that. Read anonymously (the job hook has no token), from + / so git never reads a checkout's .git/config (actions/checkout keeps the job's token there); any doubt (no full + sha, git failing, a timeout) refuses.""" if 'upstream_sync' not in cfg: return None # this repository has no upstream sync if not (isinstance(sha, str) and SHA.fullmatch(sha)): @@ -380,8 +381,8 @@ def made_by_the_sync(cfg, sha, run=subprocess.run): 'GIT_CONFIG_GLOBAL': '/dev/null'} # no git config of whoever runs this (the job hook: a runner user) try: got = run(['git', '-c', 'protocol.version=2', 'ls-remote', '--exit-code', SYNC_REMOTE.format(cfg['repository']), - ref], env=plain, stdin=subprocess.DEVNULL, capture_output=True, text=True, - timeout=SYNC_CHECK_SECONDS, check=False) + ref], env=plain, cwd='/', stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=SYNC_CHECK_SECONDS, check=False) # cwd: never a checkout, whose .git/config holds a token except (OSError, subprocess.SubprocessError) as e: return f'whether the upstream sync made {sha[:12]} cannot be told ({e.__class__.__name__})' if got.returncode == 2: # --exit-code: the remote has no such ref diff --git a/.github/scripts/fork-fleet.test.py b/.github/scripts/fork-fleet.test.py index 0f4c68d2bb553..5af4c76a4c757 100644 --- a/.github/scripts/fork-fleet.test.py +++ b/.github/scripts/fork-fleet.test.py @@ -554,6 +554,8 @@ def test_it_asks_this_repository_anonymously_for_the_one_tag(self): self.assertEqual(kw['env']['GIT_CONFIG_NOSYSTEM'], '1') self.assertEqual(kw['env']['GIT_TERMINAL_PROMPT'], '0') self.assertFalse({k for k in kw['env'] if 'TOKEN' in k or k == 'HOME'}) + # Not inside a checkout: git would read its .git/config, where actions/checkout keeps the job's token. + self.assertEqual(kw['cwd'], '/') self.assertTrue(0 < kw['timeout'] <= 60) def test_a_tagged_commit_and_any_doubt_are_refused(self): diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index edebaefc1fd79..92af67bffb347 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -120,6 +120,10 @@ jobs: fleet_mode: ${{ steps.route.outputs.mode }} fleet_attempt: ${{ steps.route.outputs.attempt }} fleet_watch: ${{ steps.route.outputs.watch || 'false' }} + # The memory pressure probe's time limit in seconds (fork-ci-pressure.py, the first and last step of each + # app job), here only. fork-ci-rerun.py reads no note from a job whose end step took this long + # (.github/fork-ci-rerun.json probe_limit_seconds is the same number: fork-ci-pressure.test.py). + probe_seconds: "40" steps: - name: Checkout if: inputs.ref == '' @@ -747,10 +751,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' - name: Reject repository-owned PR assets run: | @@ -812,7 +818,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' build: name: Build and Typecheck @@ -833,10 +839,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' # Runs in the background while Vite+ installs; the step before the build waits for it. # Xvfb is for the desktop launch check below. @@ -982,7 +990,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' # Everything except `t3` (apps/server) and the web app, which have their own jobs. # Upstream runs this on one large runner; one 4-CPU runner took 6 minutes. @@ -1011,10 +1019,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' # A shard with nothing to run says so in the summary and stops here. - name: Pick this shard's packages @@ -1126,7 +1136,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' # Upstream runs this on one large runner; on a 4-CPU runner the 548 files # took 7.5 minutes, so each of the shards below takes a vitest slice. @@ -1156,10 +1166,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' - name: Setup Vite+ uses: ./.github/actions/setup-ci-deps @@ -1217,7 +1229,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' # apps/server runs its test files one at a time; shards spread them over # separate runners. As upstream, the shards run the change detector @@ -1249,10 +1261,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' - name: Detect affected jobs id: changes @@ -1379,7 +1393,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' rust: name: Rust @@ -1400,10 +1414,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' - name: Setup Rust uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master @@ -1429,7 +1445,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' # Upstream runs this on macOS 26. GitHub-hosted macOS minutes are free on a # public repository; it runs only for native mobile sources or a workflow change. @@ -1483,10 +1499,12 @@ jobs: sparse-checkout-cone-mode: false # Memory pressure from here on, for the one automatic re-run of an infrastructure failure - # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job, + # and one that hangs on a stalled machine is stopped after the `changes` job's probe_seconds with a warning: + # the tests still run). - name: Note memory pressure at the start timeout-minutes: 1 - run: python3 -I .github/scripts/fork-ci-pressure.py start + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py start || echo "::warning::fork-ci-pressure: start stopped (exit $?), no numbers; the job goes on"' - name: Setup Vite+ uses: ./.github/actions/setup-ci-deps @@ -1503,7 +1521,7 @@ jobs: if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} - run: python3 -I .github/scripts/fork-ci-pressure.py end + run: 'timeout -k 5 "${{ needs.changes.outputs.probe_seconds }}" python3 -I .github/scripts/fork-ci-pressure.py end || echo "::warning::fork-ci-pressure: end stopped (exit $?), no numbers"' # The one check a ruleset requires. Fails when any job fails or is cancelled, # or is skipped without a reason: `changes` said its gates were not needed,