diff --git a/.github/fork-ci-rerun.json b/.github/fork-ci-rerun.json new file mode 100644 index 0000000000000..4487aa09a5bd8 --- /dev/null +++ b/.github/fork-ci-rerun.json @@ -0,0 +1,62 @@ +{ + "_doc": [ + "Fork only: when a failed Fork tests run gets its one automatic re-run. Read by .github/scripts/fork-ci-rerun.py", + "(.github/workflows/fork-ci-rerun.yml every 10 minutes); the script has no fact about this repository, these are", + "all of them. Runbook: .github/fork-ci-rerun.md.", + "trust: the fleet config beside this one; its repository, trusted workflows, actors and branches, and its rule", + " (fork-fleet.py run_refused), decide which runs are trusted. workflow must be one of its trusted workflows.", + "list_hours: how far back runs are listed (by creation; every page is read); catch_up_minutes: of those, the", + " runs that ended (updated) this recently. window_hours: the re-run count read from the tracking issue.", + "hosted_runner_groups: GitHub-hosted runners' group; an OOM kill there is the job's own (no box of ours).", + "probe_end_step: the step fork-ci-pressure.py writes the note in (fork-tests.yml); only a job that completed it", + " has a note of its own, and a job that never reached it lost its runner or was stopped.", + "limit_slack_seconds: a time-limit message counts only when the job (or the step) ran its minutes less this.", + "signs: GitHub's own runner messages, each pattern a whole message (re.fullmatch), only on an annotation in the", + " runner's form (failure, no title, .github), and only together with the job's step record (the script's rules).", + " lost: the runner went away. limit: the job or a step ran past its time limit; only with `stalled`, and only", + " when its minutes are that job's or step's own timeout-minutes in `workflow` at the run's commit.", + " cancelled_by: a person cancelled it; the run stays red.", + "stalled: the job's memory pressure note shows the machine or a box stalled: some-or-all-tasks-waiting (full) share", + " of the job's time, or the 5-minute full average at its end.", + "memory_full: on a runner that is not GitHub-hosted, a box with a memory limit stalled (full) at least", + " box_full_share_percent of the job, AND in that box OOM kills, memory.max hits or memory.high throttling.", + "skip_when: with this environment variable (the workflow sets it from a repository variable) at one of these", + " values, a run that qualifies is only reported, once, not re-run (fleet mode: a re-run of failed jobs would wait", + " for fleet runners that refuse it, .github/fork-fleet.md).", + "notice: the one tracking issue every re-run (and report) is a comment on; opened by `author` when none is open.", + " labels: given when it is opened (each must exist in the repository)." + ], + "trust": "fork-fleet.json", + "workflow": "fork-tests.yml", + "followers": ["fork-ci-result"], + "list_hours": 6, + "catch_up_minutes": 60, + "window_hours": 24, + "skip_when": { "variable": "CI_RUNNERS", "is": ["fleet"] }, + "hosted_runner_groups": ["GitHub Actions"], + "probe_end_step": "Note memory pressure at the end", + "limit_slack_seconds": 60, + "signs": { + "lost": [ + "The (?:self-hosted|hosted) runner: (?P.+?) lost communication with the server\\..*", + "The runner has received a shutdown signal\\..*" + ], + "limit": [ + "The job running on runner (?P.+) has exceeded the maximum execution time of (?P[0-9]+) minutes\\.", + "The action '(?P[^']+)' has timed out after (?P[0-9]+) minutes\\." + ], + "cancelled_by": ["The run was canceled by @[^ ]+\\."] + }, + "stalled": { "full_share_percent": 10, "full_avg300_percent": 10 }, + "memory_full": { + "box_full_share_percent": 10, + "oom_kills": 1, + "max_events": 1, + "high_events": 1 + }, + "notice": { + "title": "Fork CI: automatic re-runs (infrastructure)", + "labels": [], + "author": "github-actions[bot]" + } +} diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md new file mode 100644 index 0000000000000..fe3a1c8a0971d --- /dev/null +++ b/.github/fork-ci-rerun.md @@ -0,0 +1,154 @@ +# Fork CI re-run: one automatic re-run for infrastructure failures only + +A failed `Fork tests` run gets **one** automatic "re-run failed jobs" (GitHub's +own), only when every failed job shows a known infrastructure sign that +GitHub's own runner message and the job's own step record agree on. Never for +a test failure, never for a person's cancel, never twice: a repeat red is left +red, and it goes to an issue and its fix first. Every re-run is also a comment +on one tracking issue, so a machine that keeps stalling is never hidden. + +| Part | What it does | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`), each with a 1-minute timeout. Facts only: it never fails (it catches every error), and a hung one never holds up the job. | +| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, the tracking issue, and the re-run. Uses `fork-fleet.py`'s trust rule. | +| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: the trust config (`fork-fleet.json`), workflow, followers, windows, the hosted runner group, the probe's end step, sign patterns, thresholds, the fleet-mode switch, the tracking issue. | +| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`, `issues: write`. `workflow_dispatch` with `run_id` checks one run, in a concurrency group of its own. | +| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal (each rule line is pinned: removing it turns a test red), the state cases below, the paging, the wiring. | + +## The numbers in the job log + +The `end` step prints one line in every app job, passed or not: + +``` +fork-ci-pressure: job failure; 612 s since start; memory stall full 14.2% some 30.1% of the job; +full avg10/60/300 22.0/18.4/12.9%; available 310 MB, swap used 1024 MB, OOM kills 1; +cgroup /fleet.slice/fork-fleet-x.service: high None MB max None MB use 2900 MB, events high None max None oom_kill None, full 15.0%; ... +``` + +- **stall full / some**: the share of the job's time in which all (full) or + some tasks of the machine waited for memory (`/proc/pressure/memory`, from + its `total` counters between `start` and `end`). +- **avg10/60/300**: the kernel's own full-stall averages at the end. +- **OOM kills**: `/proc/vmstat` `oom_kill` since `start`. +- **cgroup**: the job's own cgroup and each ancestor with a memory limit (the + box: a slice or unit with `MemoryHigh` or `MemoryMax`), with limit, use, + `memory.events` (high, max, oom_kill) since `start` and its own stall share. + +A job that did not succeed also leaves the same numbers as one notice +annotation (`fork-ci-pressure`), which the decision reads from GitHub after the +run. `null` (or `None` in the line) is a number that could not be read: no +start sample, a reboot in between, a file the runner cannot see. + +## The rules (fork-ci-rerun.py) + +1. Only a trusted run, by the fleet's own rule (`fork-fleet.py` `run_refused` + with `fork-fleet.json`): this repository, a trusted workflow, started and + triggered by a trusted actor, a push to a trusted branch not made by the + upstream sync, or a pull request a trusted actor opened from a branch of + this repository. Never looser than what may run on the fleet. +2. Never twice: if any attempt of the run was triggered by someone else (the + re-run is triggered by `github-actions`), the run is left red. +3. Each failed job (failure, cancelled, timed out) except the followers + (`fork-ci-result`, which needs every job) must show a sign. **Text is never + enough**: a step can print any annotation (`::error::` with GitHub's own + words, `::notice title=fork-ci-pressure::`), with exactly the fields the + runner's own have. So a message counts only in the runner's form (failure + level, no title, on `.github`), only as a whole message (`re.fullmatch` of a + `signs` pattern), and only when the job's own record agrees: + - **lost**: GitHub says the runner lost communication with the server, or + received a shutdown signal; the job ended `failure` on that runner, none + of its steps failed, no time limit was hit, and the probe's end step never + completed (the runner died before it). No note needed. + - **memory full**: the job ended `failure` on a runner that is not + GitHub-hosted (`hosted_runner_groups`: an OOM kill there is the job's + own), and its own note shows a box with a memory limit stalled (full) at + least `box_full_share_percent` of the job AND, in that box, OOM kills, + `memory.max` hits or `memory.high` throttling. One event without the + stall, or a machine-wide OOM kill, is no sign. + - **limit while stalled**: GitHub says the job or a step ran past its time + limit; its minutes are that job's (or step's) own `timeout-minutes` in the + workflow file at the run's commit (read through the contents API; a step + can print GitHub's words with any number, so the message's number alone + never counts, and a job or step the reader cannot match stays red); the + job (or that step) really ran that long (less + `limit_slack_seconds`); no other step failed; and the note shows the + machine or a box stalled (full) at least `full_share_percent` of the job, + or a 5-minute full average at least `full_avg300_percent` at the end. +4. The note counts only when the probe's end step completed (so the job wrote + its own) and it is the job's only one, a notice on `.github` with the + `fork-ci-pressure v1 ` header and the right shape. +5. One failed job without a sign leaves the run red: a test failure, a job + with no note, a garbled one, two notes. GitHub's "The run was canceled by + @..." on a failed job leaves it red too: a person's cancel stands, whatever + the memory was doing. Fail closed. +6. The branch must still be at the run's commit (a newer push tests it anyway). +7. With `CI_RUNNERS=fleet` (`skip_when`), a run that qualifies is reported, once + (one comment, one error annotation), and not re-run: a re-run of failed jobs + keeps its fleet labels and would wait for fleet runners that refuse the + bot's attempt (`fork-fleet.md`). Re-run it by hand. + +## Visible and counted + +- **The tracking issue** (`notice` in the config): the first re-run opens one + issue, by `github-actions`, with that title; every later re-run (and every + fleet-mode report) is one comment on it: the run's link, event, branch and + commit, each job's sign, GitHub's message and the pressure numbers. Never one + issue per re-run. Only an open issue by `github-actions` with exactly that + title is used; closing it is fine (the next re-run opens a new one). A + comment carries a marker line (``); the comment is written **before** the request. +- Each re-run is also a **warning annotation** on the `Fork CI re-run` run. +- Each pass writes a **job summary**: a table of every red run it looked at and + what was done, and the count of automatic re-runs of the last 24 hours, read + from the issue's markers (two to a few API calls a pass, however many runs). + +## State + +Nothing is kept between passes: each one reads GitHub (the runs and the issue). + +| Case | What happens | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. No second comment. | +| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | +| Two passes open the issue at once | Each reads the open issues after opening its own; the lowest number wins and the other closes its own as a duplicate. | +| A pass stops before its request reached GitHub | The comment is there, the run unchanged; the next pass (within `catch_up_minutes`) sends the request once and adds no second comment. | +| A pass stops after GitHub took the request (reply lost) | The comment, written first, says why. The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | +| The comment cannot be written | No re-run (it would be hidden): a warning, the pass ends red, the next pass tries both. | +| The tracking issue cannot be read | No re-run (its record cannot be checked) and no count: the pass ends red and says so; the next pass tries again. | +| A dispatch with `run_id` while a scheduled pass waits | Its own concurrency group: it is never replaced by a scheduled pass. | +| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | +| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). Runs are listed by creation over `list_hours`, every page read; a listing past 10 pages ends the pass red, nothing re-run. | +| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | +| No tracking issue, or it was closed | The count reads 0 from a new window; the next re-run opens a new one. | + +## Overlap with the fleet sweep + +`fork-fleet.yml`'s sweep also re-runs (the whole run, GitHub-hosted) a run +whose fleet runner was lost, in `auto` mode. Both stay at one re-run per run: +each leaves alone a run with an attempt by `github-actions`, and GitHub refuses +a re-run while one is going. They are offset by 5 minutes. The fleet sweep's +own re-runs are not comments on the tracking issue (its own job summary shows +them); that is `fork-fleet.py`'s to change. + +## Limits + +- No live proof before merge: scheduled workflows run from the default branch. + After merge, dispatch `Fork CI re-run` once and read its summary. Then check, + on a real red run, what the jobs API says (a) for the end step after a job + time limit (does `if: always()` still run it? if not, `limit` never + qualifies, fail closed) and (b) for a `continue-on-error` step that failed; + and on a real lost runner, its steps' state. +- A note is the job's own words: a trusted run's code could still print one + and keep the real end step from writing its own (two notes are refused, and + a note without its end step's completion is not read). Only trusted runs are + read, and only once. +- An OOM kill by a user-space killer (earlyoom, systemd-oomd) is not in + `memory.events` or `oom_kill`: it shows only as a stall, if at all. +- GitHub keeps at most 10 notices per step and 50 annotations per job: on a + job with very many annotations the note can be dropped. The job then reads + as "no note" and stays red (fail closed). +- Text quoted into the public tracking issue (annotation messages, note + paths) is made one line with no `@` mention and no HTML, and only a + comment's first line is read as its marker. +- A job a step's own time limit failed counts as `limit` only with a stall; a + slow test on a calm machine stays red. diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 2303a0eabbf70..845549929c882 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -33,6 +33,8 @@ # is a note; one that was present and then deleted fails. .github/scripts/fork-ci-tests.test.cjs ~11s .github/scripts/fork-checks-ran.test.py +.github/scripts/fork-ci-pressure.test.py +.github/scripts/fork-ci-rerun.test.py .github/scripts/fork-contracts.test.py ~45s .github/scripts/fork-fleet-desk-agent.test.py ~23s .github/scripts/fork-fleet-desk-key.test.py ~10s diff --git a/.github/fork-owned-paths.txt b/.github/fork-owned-paths.txt index 0a70eacb284d8..0802773458c93 100644 --- a/.github/fork-owned-paths.txt +++ b/.github/fork-owned-paths.txt @@ -24,6 +24,7 @@ fork-kit/ .github/fork-* # Read by app jobs or the runner routing every app job uses. !.github/scripts/fork-fleet.py +!.github/scripts/fork-ci-pressure.py !.github/scripts/fork-test-shards.cjs !.github/scripts/fork-vitest-gc-check.cjs !.github/fork-fleet.json diff --git a/.github/scripts/fork-ci-pressure.py b/.github/scripts/fork-ci-pressure.py new file mode 100644 index 0000000000000..9b12d4f62a810 --- /dev/null +++ b/.github/scripts/fork-ci-pressure.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Fork only: a job's memory pressure numbers, for the log and for .github/scripts/fork-ci-rerun.py. + + python3 -I fork-ci-pressure.py start right after a job's checkout: saves a sample to $RUNNER_TEMP. + python3 -I fork-ci-pressure.py end the job's last step (if: always(), JOB_STATUS=${{ job.status }}): prints + one log line with the numbers since `start`, and, when the job did not + succeed, the same numbers as one notice annotation (NOTE_TITLE, NOTE_HEAD + + compact JSON) that fork-ci-rerun.py reads from GitHub after the run. + +Facts only, no decision: what counts as a stalled or full machine is fork-ci-rerun.py's, from its config. The numbers: +- the machine's memory pressure (/proc/pressure/memory, PSI): the share of the job's time that some or all tasks + waited for memory (from the `total` counters), and the 10 s, 60 s and 300 s averages at the end; +- /proc/meminfo's available memory and swap in use, and /proc/vmstat's OOM kill count since `start`; +- the job's own cgroup (v2) and each ancestor with a memory limit (the box the job runs in: a systemd slice or + unit with MemoryHigh or MemoryMax): limit, use, and memory.events' high, max and oom_kill since `start`, and its + own pressure share. +Never fails the job: a file it cannot read is null, a missing start sample makes every "since start" number null, +and any other error prints a warning and exits 0. Standard library only. +""" +import json, os, sys, time + +NOTE_TITLE = 'fork-ci-pressure' +NOTE_HEAD = 'fork-ci-pressure v1 ' +SAMPLE = 'fork-ci-pressure-start.json' +EVENTS = ('high', 'max', 'oom', 'oom_kill') + + +def read(path): + try: + with open(path, encoding='utf-8') as f: + return f.read() + except (OSError, UnicodeDecodeError): + return None + + +def psi(text): + """/proc/pressure/memory or a cgroup's memory.pressure: {"some": {...}, "full": {...}} (avg in %, total in µs).""" + if text is None: + return None + out = {} + for line in text.splitlines(): + kind, _, rest = line.partition(' ') + if kind not in ('some', 'full'): + continue + fields = {} + for part in rest.split(): + key, _, value = part.partition('=') + try: + fields[key] = float(value) if key.startswith('avg') else int(value) + except ValueError: + pass + out[kind] = fields + return out or None + + +def keyed(text, scale=1): + """'Key: 123 kB' or 'key 123' lines as {key: number}.""" + if text is None: + return None + out = {} + for line in text.splitlines(): + parts = line.replace(':', ' ').split() + if len(parts) >= 2: + try: + out[parts[0]] = int(parts[1]) // scale + except ValueError: + pass + return out + + +def limit(text): + """memory.max / memory.high: bytes, or None for 'max' (no limit) or unreadable.""" + text = (text or '').strip() + return int(text) if text.isdigit() else None + + +def own_cgroup(proc): + """The cgroup v2 path of this process ('0::/a/b' in /proc/self/cgroup), or None.""" + for line in (read(os.path.join(proc, 'self/cgroup')) or '').splitlines(): + if line.startswith('0::'): + return line[3:].strip() or '/' + return None + + +def cgroups(proc, root): + """The own cgroup and every ancestor with a memory limit, innermost first.""" + path = own_cgroup(proc) + out = [] + while path: + where = os.path.join(root, path.lstrip('/')) + high, top = limit(read(os.path.join(where, 'memory.high'))), limit(read(os.path.join(where, 'memory.max'))) + if not out or high is not None or top is not None: + events = keyed(read(os.path.join(where, 'memory.events'))) + current = read(os.path.join(where, 'memory.current')) + out.append({'path': path, 'high_bytes': high, 'max_bytes': top, + 'current_bytes': int(current) if (current or '').strip().isdigit() else None, + 'events': {k: events[k] for k in EVENTS if k in events} if events else None, + 'psi': psi(read(os.path.join(where, 'memory.pressure')))}) + if path == '/': + break + path = os.path.dirname(path.rstrip('/')) or '/' + return out + + +def sample(proc='/proc', root='/sys/fs/cgroup', now=time.time): + vm = keyed(read(os.path.join(proc, 'vmstat'))) or {} + return {'t': now(), 'boot': (read(os.path.join(proc, 'sys/kernel/random/boot_id')) or '').strip() or None, + 'psi': psi(read(os.path.join(proc, 'pressure/memory'))), + 'mem_kb': keyed(read(os.path.join(proc, 'meminfo'))), 'oom_kill': vm.get('oom_kill'), + 'cgroups': cgroups(proc, root)} + + +def share(a, b, kind, seconds): + """% of `seconds` that `kind` (some/full) pressure counted between two PSI readings, or None.""" + try: + return round((b[kind]['total'] - a[kind]['total']) / 1e6 / seconds * 100, 1) + except (KeyError, TypeError, ZeroDivisionError): + return None + + +def delta(a, b): + return b - a if isinstance(a, int) and isinstance(b, int) and b >= a else None + + +def mb(n): + return None if n is None else n // (1024 * 1024) + + +def numbers(start, end): + """The note: what fork-ci-rerun.py reads. Every 'since start' number is null without a usable start sample.""" + t = start.get('t') if isinstance(start, dict) else None + same = isinstance(t, (int, float)) and start.get('boot') == end.get('boot') and end['t'] > t + start = start if same else {} + seconds = end['t'] - t if same else None + psi_end, mem = end.get('psi') or {}, end.get('mem_kb') or {} + full, some = psi_end.get('full') or {}, psi_end.get('some') or {} + swap = (mem['SwapTotal'] - mem['SwapFree']) // 1024 if 'SwapTotal' in mem and 'SwapFree' in mem else None + host = {'full_share': share(start.get('psi'), psi_end, 'full', seconds) if same else None, + 'some_share': share(start.get('psi'), psi_end, 'some', seconds) if same else None, + 'full_avg10': full.get('avg10'), 'full_avg60': full.get('avg60'), 'full_avg300': full.get('avg300'), + 'some_avg60': some.get('avg60'), 'some_avg300': some.get('avg300'), + 'available_mb': mem['MemAvailable'] // 1024 if 'MemAvailable' in mem else None, 'swap_used_mb': swap, + 'oom_kill': delta(start.get('oom_kill'), end.get('oom_kill')) if same else None} + before = {c['path']: c for c in (start.get('cgroups') or [])} if same else {} + boxes = [] + for c in end.get('cgroups') or []: + b = before.get(c['path']) or {} + ev_a, ev_b = b.get('events') or {}, c.get('events') or {} + boxes.append({'path': c['path'], 'high_mb': mb(c.get('high_bytes')), 'max_mb': mb(c.get('max_bytes')), + 'current_mb': mb(c.get('current_bytes')), + **{k: delta(ev_a.get(k), ev_b.get(k)) for k in EVENTS}, + 'full_share': share(b.get('psi'), c.get('psi'), 'full', seconds) if b else None}) + return {'seconds': None if seconds is None else int(seconds), 'host': host, 'cgroups': boxes} + + +def line(status, n): + h = n['host'] + out = [f'fork-ci-pressure: job {status}', f'{n["seconds"]} s since start' if n['seconds'] is not None + else 'no start sample (numbers since start unknown)', + f'memory stall full {h["full_share"]}% some {h["some_share"]}% of the job', + f'full avg10/60/300 {h["full_avg10"]}/{h["full_avg60"]}/{h["full_avg300"]}%', + f'available {h["available_mb"]} MB, swap used {h["swap_used_mb"]} MB, OOM kills {h["oom_kill"]}'] + for c in n['cgroups']: + out.append(f'cgroup {c["path"]}: high {c["high_mb"]} MB max {c["max_mb"]} MB use {c["current_mb"]} MB, ' + f'events high {c["high"]} max {c["max"]} oom_kill {c["oom_kill"]}, full {c["full_share"]}%') + return '; '.join(out) + + +def command(text): + """A workflow command's data, escaped as the runner wants (%, CR, LF).""" + return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + + +def main(argv, env=os.environ, out=sys.stdout, proc='/proc', root='/sys/fs/cgroup', now=time.time): + verb = argv[1] if len(argv) == 2 else '' + temp = env.get('RUNNER_TEMP') or '' + if verb not in ('start', 'end') or not temp: + print('::warning::fork-ci-pressure: usage: fork-ci-pressure.py start|end, with RUNNER_TEMP set', file=out) + return 0 + path = os.path.join(temp, SAMPLE) + try: + now_sample = sample(proc, root, now) + if verb == 'start': + with open(path, 'w', encoding='utf-8') as f: + json.dump(now_sample, f) + print(f'fork-ci-pressure: start sample saved ({len(now_sample["cgroups"])} cgroup(s))', file=out) + return 0 + try: + start = json.loads(read(path) or 'null') + except ValueError: + start = None + n = numbers(start if isinstance(start, dict) else {}, now_sample) + status = (env.get('JOB_STATUS') or 'unknown').strip() + print(line(status, n), file=out) + if status != 'success': + note = json.dumps({'status': status, **n}, separators=(',', ':'), sort_keys=True) + print(f'::notice title={NOTE_TITLE}::{command(NOTE_HEAD + note)}', file=out) + except Exception as e: # never fail the job + print(f'::warning::fork-ci-pressure: {verb}: {e.__class__.__name__}: {e}', file=out) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/.github/scripts/fork-ci-pressure.test.py b/.github/scripts/fork-ci-pressure.test.py new file mode 100644 index 0000000000000..b7648e16d802d --- /dev/null +++ b/.github/scripts/fork-ci-pressure.test.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Tests of fork-ci-pressure.py (a job's memory pressure numbers), with a fake /proc and cgroup tree, and of its +steps in fork-tests.yml. Run: python3 .""" +import io, json, os, re, shutil, subprocess, sys, tempfile, unittest +import importlib.util + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SCRIPT = os.path.join(HERE, 'fork-ci-pressure.py') +_spec = importlib.util.spec_from_file_location('fork_ci_pressure', SCRIPT) +probe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(probe) + +# The jobs that run app builds and tests on a fleet or GitHub-hosted runner (each routable job of fork-fleet.json). +with open(os.path.join(ROOT, '.github/fork-fleet.json'), encoding='utf-8') as f: + APP_JOBS = sorted({j['job'] for j in json.load(f)['jobs']}) +GIB = 1024 ** 3 + + +def psi_text(some_total, full_total, full_avg10=0.0, full_avg300=0.0): + return (f'some avg10=1.00 avg60=0.50 avg300=0.20 total={some_total}\n' + f'full avg10={full_avg10:.2f} avg60=0.00 avg300={full_avg300:.2f} total={full_total}\n') + + +class Machine: + """A fake /proc and /sys/fs/cgroup: the job in /fleet.slice/job.service, the slice limited to 8 GiB.""" + + def __init__(self, d): + self.d, self.proc, self.root = d, os.path.join(d, 'proc'), os.path.join(d, 'cgroup') + self.write('proc/self/cgroup', '0::/fleet.slice/job.service\n') + self.write('proc/sys/kernel/random/boot_id', 'boot-1\n') + self.write('proc/meminfo', 'MemTotal: 16000000 kB\nMemAvailable: 8192000 kB\n' + 'SwapTotal: 2097152 kB\nSwapFree: 1048576 kB\n') + self.write('proc/vmstat', 'nr_free_pages 1\noom_kill 3\n') + self.write('proc/pressure/memory', psi_text(0, 0)) + self.box('fleet.slice', high='max', top=str(8 * GIB), events=(5, 0, 0, 0), psi=psi_text(0, 0)) + self.box('fleet.slice/job.service', high='max', top='max', events=(0, 0, 0, 0), psi=psi_text(0, 0)) + self.write('cgroup/fleet.slice/job.service/memory.current', str(3 * GIB)) + + def write(self, rel, text): + path = os.path.join(self.d, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + + def box(self, rel, high=None, top=None, events=None, psi=None): + base = f'cgroup/{rel}' + if high is not None: + self.write(f'{base}/memory.high', high + '\n') + if top is not None: + self.write(f'{base}/memory.max', top + '\n') + if events is not None: + self.write(f'{base}/memory.events', ''.join(f'{k} {v}\n' for k, v in zip(probe.EVENTS, events))) + if psi is not None: + self.write(f'{base}/memory.pressure', psi) + + +def run(m, verb, temp, status=None, t=1000.0): + out = io.StringIO() + env = {'RUNNER_TEMP': temp} + if status: + env['JOB_STATUS'] = status + code = probe.main(['p', verb], env=env, out=out, proc=m.proc, root=m.root, now=lambda: t) + return code, out.getvalue() + + +def the_note(out): + lines = [l for l in out.splitlines() if l.startswith('::notice ')] + if not lines: + return None + assert len(lines) == 1, lines + data = lines[0].split('::', 2)[2].replace('%0A', '\n').replace('%0D', '\r').replace('%25', '%') + assert data.startswith(probe.NOTE_HEAD), data + return json.loads(data[len(probe.NOTE_HEAD):]) + + +class Probe(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d) + self.m = Machine(self.d) + self.temp = os.path.join(self.d, 'temp') + os.makedirs(self.temp) + + def stall(self): + """600 s later: the machine stalled 60 s (10%), the slice OOM-killed twice and hit memory.max.""" + self.m.write('proc/pressure/memory', psi_text(120_000_000, 60_000_000, full_avg10=30.0, full_avg300=12.5)) + self.m.write('proc/vmstat', 'nr_free_pages 1\noom_kill 5\n') + self.m.box('fleet.slice', events=(9, 4, 2, 2), psi=psi_text(0, 90_000_000)) + + def test_a_failed_job_prints_its_numbers_and_one_note(self): + self.assertEqual(run(self.m, 'start', self.temp)[0], 0) + self.stall() + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + self.assertIn('fork-ci-pressure: job failure; 600 s since start; memory stall full 10.0% some 20.0%', out) + self.assertIn('full avg10/60/300 30.0/0.0/12.5%', out) + self.assertIn('available 8000 MB, swap used 1024 MB, OOM kills 2', out) + self.assertIn('cgroup /fleet.slice: high None MB max 8192 MB', out) + n = the_note(out) + self.assertEqual((n['status'], n['seconds']), ('failure', 600)) + self.assertEqual((n['host']['full_share'], n['host']['full_avg300'], n['host']['oom_kill']), (10.0, 12.5, 2)) + own, slice_ = n['cgroups'] + self.assertEqual(own['path'], '/fleet.slice/job.service') + self.assertEqual(own['current_mb'], 3072) + self.assertEqual((slice_['path'], slice_['max_mb'], slice_['high'], slice_['max'], slice_['oom_kill'], + slice_['full_share']), ('/fleet.slice', 8192, 4, 4, 2, 15.0)) + self.assertNotIn('/', [c['path'] for c in n['cgroups']], 'the root has no limit: not listed') + + def test_a_passing_job_prints_its_numbers_but_no_note(self): + run(self.m, 'start', self.temp) + code, out = run(self.m, 'end', self.temp, 'success', t=1600.0) + self.assertEqual(code, 0) + self.assertIn('fork-ci-pressure: job success', out) + self.assertIsNone(the_note(out)) + + def test_a_cancelled_job_and_an_unknown_status_leave_a_note(self): + run(self.m, 'start', self.temp) + for status in ('cancelled', None): + n = the_note(run(self.m, 'end', self.temp, status, t=1600.0)[1]) + self.assertEqual(n['status'], status or 'unknown') + + def test_no_start_sample_makes_every_since_start_number_null(self): + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) # the start step never ran + self.assertEqual(code, 0) + self.assertIn('no start sample', out) + n = the_note(out) + self.assertIsNone(n['seconds']) + self.assertEqual([n['host'][k] for k in ('full_share', 'some_share', 'oom_kill')], [None, None, None]) + self.assertTrue(all(c['oom_kill'] is None and c['full_share'] is None for c in n['cgroups'])) + self.assertEqual(n['cgroups'][1]['max_mb'], 8192, 'what is read at the end is still there') + + def test_a_garbled_or_foreign_start_sample_counts_as_none(self): + for text in ('{"t": ', '[1]', '{"t": "soon"}', json.dumps({'t': 500.0, 'boot': 'boot-0'})): + with self.subTest(text): + with open(os.path.join(self.temp, probe.SAMPLE), 'w') as f: + f.write(text) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + self.assertIsNone(the_note(out)['host']['full_share'], out) + + def test_a_reboot_between_start_and_end_counts_nothing_since_start(self): + run(self.m, 'start', self.temp) + self.stall() + self.m.write('proc/sys/kernel/random/boot_id', 'boot-2\n') + n = the_note(run(self.m, 'end', self.temp, 'failure', t=1600.0)[1]) + self.assertIsNone(n['host']['oom_kill']) + + def test_counters_that_went_back_are_null_not_negative(self): + run(self.m, 'start', self.temp) + self.m.write('proc/vmstat', 'oom_kill 1\n') + n = the_note(run(self.m, 'end', self.temp, 'failure', t=1600.0)[1]) + self.assertIsNone(n['host']['oom_kill']) + + def test_nothing_readable_still_ends_well(self): + shutil.rmtree(self.m.root) + for rel in ('pressure', 'meminfo', 'vmstat', 'self'): + path = os.path.join(self.m.proc, rel) + (shutil.rmtree if os.path.isdir(path) else os.unlink)(path) + self.assertEqual(run(self.m, 'start', self.temp)[0], 0) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + n = the_note(out) + self.assertEqual(n['cgroups'], []) + self.assertIsNone(n['host']['full_avg10']) + + def test_a_cgroup_path_with_odd_characters_stays_one_annotation(self): + self.m.write('proc/self/cgroup', '0::/odd%25\\x2dslice\n') + self.m.box('odd%25\\x2dslice', high=str(GIB), events=(1, 0, 0, 0)) + run(self.m, 'start', self.temp) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(the_note(out)['cgroups'][0]['path'], '/odd%25\\x2dslice') + + def test_it_never_fails_the_job(self): + out = io.StringIO() + self.assertEqual(probe.main(['p', 'start'], env={}, out=out), 0) + self.assertIn('::warning::', out.getvalue()) + self.assertEqual(probe.main(['p', 'nonsense'], env={'RUNNER_TEMP': self.temp}, out=io.StringIO()), 0) + os.chmod(self.temp, 0o500) # the sample cannot be written + self.addCleanup(os.chmod, self.temp, 0o700) + if os.geteuid() != 0: + code, text = run(self.m, 'start', self.temp) + self.assertEqual(code, 0) + self.assertIn('::warning::fork-ci-pressure: start', text) + r = subprocess.run([sys.executable, '-I', SCRIPT, 'end'], capture_output=True, text=True, + env={'PATH': os.environ.get('PATH', ''), 'RUNNER_TEMP': self.temp, 'JOB_STATUS': 'failure'}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('fork-ci-pressure: job failure', r.stdout) + + +class Steps(unittest.TestCase): + """Each app job of fork-tests.yml takes a sample right after its checkout and prints the numbers last.""" + + def test_every_app_job_has_the_start_and_end_steps(self): + import yaml + with open(os.path.join(ROOT, '.github/workflows/fork-tests.yml'), encoding='utf-8') as f: + wf = yaml.safe_load(f) + self.assertTrue(APP_JOBS) + for name in APP_JOBS: + with self.subTest(name): + steps = wf['jobs'][name]['steps'] + self.assertEqual(steps[0]['name'], 'Checkout') + self.assertEqual(steps[1]['run'], 'python3 -I .github/scripts/fork-ci-pressure.py start') + self.assertNotIn('if', steps[1]) + end = steps[-1] + self.assertEqual(end['run'], 'python3 -I .github/scripts/fork-ci-pressure.py end') + self.assertTrue(re.fullmatch(r"\$\{\{ always\(\) && hashFiles\('\.github/scripts/fork-ci-pressure" + r"\.py'\) != '' \}\}", end['if']), end['if']) + self.assertEqual(end['env'], {'JOB_STATUS': '${{ job.status }}'}) + self.assertEqual(sum('fork-ci-pressure' in str(s.get('run')) for s in steps), 2) + for probe in (steps[1], end): + # A hung probe never holds the job up to its own limit. No continue-on-error: nothing in + # fork-tests.yml may fail quietly (fork-contracts.test.py); the probe itself never fails. + self.assertIn(probe.get('timeout-minutes'), (1, 2), probe['name']) + self.assertNotIn('continue-on-error', probe) + with open(os.path.join(ROOT, '.github/fork-ci-rerun.json'), encoding='utf-8') as f: + self.assertEqual(end['name'], json.load(f)['probe_end_step']) # the re-run reads its record + + def test_the_probe_is_not_fork_owned_app_jobs_read_it(self): + with open(os.path.join(ROOT, '.github/fork-owned-paths.txt'), encoding='utf-8') as f: + self.assertIn('!.github/scripts/fork-ci-pressure.py', f.read().splitlines()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py new file mode 100644 index 0000000000000..02d1e07df4b39 --- /dev/null +++ b/.github/scripts/fork-ci-rerun.py @@ -0,0 +1,749 @@ +#!/usr/bin/env python3 +"""Fork only: one automatic re-run of a failed Fork tests run, only for a known infrastructure sign. + + python3 -I fork-ci-rerun.py sweep [--run-id N] .github/workflows/fork-ci-rerun.yml (every 10 minutes, GitHub-hosted, + actions: write, checks: read, issues: write): each failed trusted run + that ended in the last catch_up_minutes (or only run N) whose every + failed job shows a sign gets GitHub's own "re-run failed jobs", once. + python3 -I fork-ci-rerun.py check checks CONFIG (fork-ci-rerun.test.py runs it). + +THE rules, in one place (the facts, patterns and numbers are CONFIG's, .github/fork-ci-rerun.json; none is here): +- Only a trusted run, by the fleet's own trust rule (fork-fleet.py run_refused, with the config CONFIG `trust` + names): never looser than the rule that decides what may run on the fleet. +- Never twice: a run any of whose attempts was triggered by someone else (GitHub's re-run from this script, or the + fleet sweep's, is triggered by github-actions) is left red. A repeat red goes to an issue and its fix first. +- Each failed job (failure, cancelled, timed out) but CONFIG's followers (jobs that need all the others and fail + only because one of them did) must show a sign. Text alone is never a sign: a step can print any annotation + (::error::, ::notice title=...::), so a message counts only in the form the runner itself writes (failure level, + no title, on .github, the whole message one of CONFIG's patterns), and only when the job's own step record + agrees: + lost GitHub says the runner lost communication or was shut down (signs.lost), the job ended failure + on that runner, none of its steps failed, no time limit was hit, and the probe's end step + (probe_end_step) never completed; + memory full the job ended failure on a runner that is not GitHub-hosted (hosted_runner_groups: a kill there is + the job's own), and its own pressure note shows a box with a memory limit stalled (full) at least + memory_full.box_full_share_percent of the job AND OOM kills, memory.max hits or memory.high + throttling in it. A machine-wide OOM kill, or one event without the stall, is no sign; + limit GitHub says the job or a step ran past its time limit (signs.limit, with the minutes), those minutes + are that job's (or step's) own timeout-minutes in the workflow file at the run's commit (never the + message's word alone: a step can print any number), the job (or that step) really ran that long, no + other step failed, and the note shows the machine or the box stalled (CONFIG stalled). + The note counts only when the probe's end step completed (so the job wrote its own) and is the job's only one. + One failed job without a sign (a test failure, no note, a garbled note, two notes) leaves the run red: fail + closed. GitHub's "canceled by" (signs.cancelled_by) on any failed job leaves it red: a person's cancel stands. + A run whose branch has moved on, or is gone, is not re-run. +- With CONFIG skip_when (fleet mode), a run that qualifies is only reported, once. +Visible and durable: every re-run (and every fleet-mode report) is first a comment on one tracking issue (CONFIG +notice: opened once, by github-actions, then a comment per re-run with the run, the signs and the numbers), then a +warning annotation and a row in this job's summary, which also counts the issue's re-runs of the last window_hours. +The comment is written before the request, so a reply lost after GitHub took it still leaves the reason. Nothing +else is kept between passes: each reads GitHub, so a pass that stopped half-way is finished by the next, and one that +runs again finds the bot's attempt (and the issue's marker) and does nothing. +Runbook: .github/fork-ci-rerun.md. Standard library only. +""" +import argparse, base64, calendar, http.client, importlib.util, json, os, re, sys, time +import urllib.error, urllib.parse, urllib.request + +sys.dont_write_bytecode = True # loading fork-fleet.py must not leave __pycache__ in the tree + +HERE = os.path.dirname(os.path.abspath(__file__)) +CONFIG = os.path.join(os.path.dirname(HERE), 'fork-ci-rerun.json') +API = os.environ.get('GITHUB_API_URL') or 'https://api.github.com' +NOTE_TITLE = 'fork-ci-pressure' # fork-ci-pressure.py's +NOTE_HEAD = 'fork-ci-pressure v1 ' +RED = ('failure', 'cancelled', 'timed_out') +CHECK_RUN = re.compile(r'/check-runs/([0-9]{1,20})$') +MARK = re.compile(r'') +# Each sign's named groups: `minutes` is how long GitHub says the limit was; `step` (when there) names the step. +GROUPS = {'lost': set(), 'limit': {'minutes'}, 'cancelled_by': set()} +LONGEST = 60000 # characters of a comment (GitHub takes 65536) + +_spec = importlib.util.spec_from_file_location('fork_fleet', os.path.join(HERE, 'fork-fleet.py')) +FLEET = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(FLEET) + + +def say(msg, out=None): + print(msg, file=out or sys.stdout, flush=True) + + +# ---------------------------------------------------------------- config + +def load(path=CONFIG): + with open(path, encoding='utf-8') as f: + cfg = json.load(f) + problems = check_config(cfg) + if problems: + raise ValueError(f'{path}: ' + '; '.join(problems)) + fleet = FLEET.load(os.path.join(os.path.dirname(os.path.abspath(path)), cfg['trust'])) + workflows = [os.path.basename(w) for w in fleet['trusted']['workflows']] + if cfg['workflow'] not in workflows: + raise ValueError(f'{path}: workflow {cfg["workflow"]!r} is not one the trust config trusts ({workflows})') + return dict(cfg, fleet=fleet) + + +def number(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + +def names(x, empty=False): + return isinstance(x, list) and (empty or bool(x)) and all(isinstance(s, str) and s for s in x) + + +def check_config(cfg): + out = [] + if not isinstance(cfg, dict): + return ['not a JSON object'] + known = {'_doc', 'trust', 'workflow', 'followers', 'list_hours', 'catch_up_minutes', 'window_hours', 'skip_when', + 'hosted_runner_groups', 'probe_end_step', 'limit_slack_seconds', 'signs', 'stalled', 'memory_full', + 'notice'} + out += [f'unknown key {k!r}' for k in sorted(set(cfg) - known)] + out += [f'missing key {k!r}' for k in sorted(known - {'_doc'} - set(cfg))] + if not re.fullmatch(r'[A-Za-z0-9_.-]+\.json', str(cfg.get('trust', ''))): + out.append('trust must be the file name of the fleet config beside this one') + if not re.fullmatch(r'[A-Za-z0-9_.-]+\.ya?ml', str(cfg.get('workflow', ''))): + out.append('workflow must be a workflow file name') + if not names(cfg.get('followers'), empty=True): + out.append('followers must be a list of job names') + for key in ('list_hours', 'catch_up_minutes', 'window_hours', 'limit_slack_seconds'): + if not number(cfg.get(key)) or cfg[key] <= 0: + out.append(f'{key} must be a positive number') + if number(cfg.get('list_hours')) and number(cfg.get('catch_up_minutes')) \ + and cfg['list_hours'] * 60 <= cfg['catch_up_minutes']: + out.append('list_hours must reach further back than catch_up_minutes') + skip = cfg.get('skip_when') + if not (isinstance(skip, dict) and re.fullmatch(r'[A-Z][A-Z0-9_]*', str(skip.get('variable', ''))) + and names(skip.get('is'))): + out.append('skip_when must be {"variable": NAME, "is": [values]}') + if not names(cfg.get('hosted_runner_groups')): + out.append('hosted_runner_groups must be a list of runner group names') + if not (isinstance(cfg.get('probe_end_step'), str) and cfg['probe_end_step']): + out.append('probe_end_step must be the name of the probe\'s end step') + signs = cfg.get('signs') + if not isinstance(signs, dict) or set(signs) != set(GROUPS): + out.append(f'signs must have exactly {", ".join(sorted(GROUPS))}') + else: + for kind, patterns in signs.items(): + if not names(patterns): + out.append(f'signs.{kind} must be a list of patterns') + continue + for p in patterns: + try: + groups = set(re.compile(p).groupindex) + except re.error as e: + out.append(f'signs.{kind}: {p!r}: {e}') + continue + if not GROUPS[kind] <= groups: + out.append(f'signs.{kind}: {p!r} lacks the group(s) {sorted(GROUPS[kind] - groups)}') + if not p.startswith('The '): + out.append(f'signs.{kind}: {p!r}: a whole GitHub message, from its first word') + for key, fields in (('stalled', ('full_share_percent', 'full_avg300_percent')), + ('memory_full', ('box_full_share_percent', 'oom_kills', 'max_events', 'high_events'))): + got = cfg.get(key) + if not isinstance(got, dict) or set(got) != set(fields) or not all(number(got[f]) and got[f] > 0 + for f in fields): + out.append(f'{key} must have exactly {", ".join(fields)}, each a positive number') + notice = cfg.get('notice') + if not (isinstance(notice, dict) and set(notice) == {'title', 'labels', 'author'} + and isinstance(notice.get('title'), str) and 0 < len(notice['title']) <= 200 + and names(notice.get('labels'), empty=True) and re.fullmatch(r'[a-z0-9-]+\[bot\]', str(notice.get('author')))): + out.append('notice must be {"title": TEXT, "labels": [names], "author": "[bot]"}') + return out + + +# ---------------------------------------------------------------- GitHub + +class GitHub: + def __init__(self, token, repo): + self.token, self.repo = token, repo + + def call(self, path, method='GET', data=None): + """path: below /repos//.""" + if not self.token: + raise ValueError(f'no token for {path}') + body = None if data is None else json.dumps(data).encode() + req = urllib.request.Request(f'{API}/repos/{self.repo}/{path}', data=body, method=method, + headers={'Accept': 'application/vnd.github+json', 'User-Agent': 'fork-ci-rerun', + 'Authorization': f'Bearer {self.token}'}) + with urllib.request.urlopen(req, timeout=30) as r: + got = r.read() + return json.loads(got) if got else {} + + def pages(self, path, key=None, most=10): + """Every item of a listing (got[key], or the reply itself), page after page until a short one; each id once + (an item that moved down a page while new ones arrived is listed twice). More than `most` full pages is a + ValueError: a listing cut short would be read as complete.""" + out, seen = [], set() + for page in range(1, most + 1): + got = self.call(f'{path}{"&" if "?" in path else "?"}per_page=100&page={page}') + items = got[key] if key else got + if not isinstance(items, list): + raise ValueError(f'{path}: page {page} is not a list') + for item in items: + ident = item.get('id') if isinstance(item, dict) else None + if ident is None or ident not in seen: + seen.add(ident) + out.append(item) + if len(items) < 100: + return out + raise ValueError(f'{path}: more than {most} pages of 100; not read as complete') + + +# A truncated reply is http.client.IncompleteRead (an HTTPException), a garbled one a ValueError. +TROUBLE = (urllib.error.URLError, http.client.HTTPException, OSError, ValueError, KeyError, TypeError, + AttributeError) + + +def stamp(text): + try: + return calendar.timegm(time.strptime(text, '%Y-%m-%dT%H:%M:%SZ')) + except (TypeError, ValueError): + return None + + +def iso(t): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t)) + + +def login(x): + return (x or {}).get('login') + + +def took(x): + """Seconds from a job's or step's started_at to its completed_at, or None.""" + a, b = stamp(x.get('started_at')), stamp(x.get('completed_at')) + return b - a if a is not None and b is not None and b >= a else None + + +# ---------------------------------------------------------------- the rules + +def from_runner(a): + """An annotation in the form the runner writes its own job messages: failure level, no title, on .github. A step + can print one that looks the same (::error::), so a sign read from one needs the job's step record too.""" + return a.get('annotation_level') == 'failure' and not a.get('title') and a.get('path') == '.github' + + +def github_says(cfg, kind, annotations): + """(message, match) of the first runner-form annotation whose whole message is one of signs[kind], or None.""" + for a in annotations: + if not from_runner(a): + continue + msg = str(a.get('message') or '').strip() + for p in cfg['signs'][kind]: + m = re.fullmatch(p, msg, re.S) + if m: + return msg, m + return None + + +def end_step_ran(cfg, job): + """Did the probe's end step complete? Then the runner lived to the job's end, and the note (if any) is its.""" + return any(s.get('name') == cfg['probe_end_step'] and s.get('status') == 'completed' + and s.get('conclusion') == 'success' for s in job.get('steps') or []) + + +def failed_steps(job): + return [s.get('name') for s in job.get('steps') or [] if s.get('conclusion') == 'failure'] + + +def note_of(cfg, job, annotations): + """The job's own pressure note (fork-ci-pressure.py) as a dict, or (None, why).""" + if not end_step_ran(cfg, job): + return None, f'its "{cfg["probe_end_step"]}" step did not complete, so it has no note of its own' + notes = [a for a in annotations if a.get('title') == NOTE_TITLE] + if not notes: + return None, 'no pressure note (the note was dropped)' + if len(notes) > 1: + return None, f'{len(notes)} pressure notes, not one (a step printed one too)' + a = notes[0] + msg = str(a.get('message') or '') + if a.get('annotation_level') != 'notice' or a.get('path') != '.github' or not msg.startswith(NOTE_HEAD): + return None, 'a garbled pressure note' + try: + note = json.loads(msg[len(NOTE_HEAD):]) + except ValueError: + return None, 'a garbled pressure note' + if not (isinstance(note, dict) and isinstance(note.get('host'), dict) and isinstance(note.get('cgroups'), list) + and all(isinstance(c, dict) for c in note['cgroups']) and isinstance(note.get('status'), str)): + return None, 'a garbled pressure note' + return note, None + + +def at_least(value, bound): + return number(value) and value >= bound + + +def stalled(cfg, note): + """Evidence that the machine or the job's box was stalled for memory, or None.""" + st, host = cfg['stalled'], note['host'] + if at_least(host.get('full_share'), st['full_share_percent']): + return f'the machine stalled (full) {host["full_share"]}% of the job' + for c in note['cgroups']: + if at_least(c.get('full_share'), st['full_share_percent']): + return f'box {c.get("path")} stalled (full) {c["full_share"]}% of the job' + if at_least(host.get('full_avg300'), st['full_avg300_percent']): + return f'the machine\'s 5-minute full stall average was {host["full_avg300"]}% at the end' + return None + + +def hosted(cfg, job): + group = job.get('runner_group_name') + return not (isinstance(group, str) and group) or group in cfg['hosted_runner_groups'] + + +def memory_full(cfg, job, note): + """Evidence that the job's own box was full and stalled, or None. Only a failure on a runner that is not + GitHub-hosted, only a box with a memory limit, and only events together with that box's stall.""" + if job.get('conclusion') != 'failure' or note.get('status') != 'failure' or hosted(cfg, job): + return None + mf = cfg['memory_full'] + for c in note['cgroups']: + if not (number(c.get('max_mb')) or number(c.get('high_mb'))): + continue # no limit: not a box + if not at_least(c.get('full_share'), mf['box_full_share_percent']): + continue + where = (f'box {c.get("path")} (max {c.get("max_mb")} MB, high {c.get("high_mb")} MB), stalled (full) ' + f'{c["full_share"]}% of the job') + if at_least(c.get('oom_kill'), mf['oom_kills']): + return f'{where}: {c["oom_kill"]} OOM kill(s) in it' + if at_least(c.get('max'), mf['max_events']): + return f'{where}: hit its memory.max {c["max"]} time(s)' + if at_least(c.get('high'), mf['high_events']): + return f'{where}: throttled at memory.high {c["high"]} time(s)' + return None + + +def scalar(v): + """A one-line YAML scalar as the workflow writes it (plain, '...' or "..."), or None for anything else.""" + v = v.strip() + if len(v) >= 2 and v[0] == v[-1] == "'": + return v[1:-1].replace("''", "'") + if len(v) >= 2 and v[0] == v[-1] == '"': + return None if '\\' in v else v[1:-1] + v = re.sub(r'\s+#.*$', '', v) + return v if v and v[0] not in '|>&*!{["\'' else None + + +def workflow_limits(text): + """{job key: {'name', 'minutes', 'steps': [{'name', 'minutes'}]}} of a workflow file's timeout-minutes, read + strictly in the layout CONFIG workflow is written in (jobs at 2 spaces, their keys at 4, a step's `- ` at 6 and + its keys at 8). Whatever it cannot read gives no minutes, and a limit it cannot confirm leaves the run red.""" + jobs, job, section, cur, inside = {}, None, None, None, False + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith('#'): + continue + if not line.startswith(' '): + inside, job = line.rstrip() == 'jobs:', None + continue + m = re.fullmatch(r' ([A-Za-z_][A-Za-z0-9_-]*):\s*', line) if inside else None + if m: + job = jobs[m[1]] = {'name': None, 'minutes': None, 'steps': []} + section = cur = None + continue + m = re.fullmatch(r' ([A-Za-z_-]+):(?:\s+(.*))?', line) if job else None + if m: + section, cur = m[1], None + if section in ('name', 'timeout-minutes'): + job['name' if section == 'name' else 'minutes'] = scalar(m[2] or '') + continue + m = re.fullmatch(r' (- | )([A-Za-z_-]+):(?:\s+(.*))?', line) if job and section == 'steps' else None + if not m: + continue + if m[1] == '- ': + cur = {'name': None, 'minutes': None} + job['steps'].append(cur) + if cur is not None and m[2] in ('name', 'timeout-minutes'): + cur['name' if m[2] == 'name' else 'minutes'] = scalar(m[3] or '') + for j in jobs.values(): + for x in [j] + j['steps']: + x['minutes'] = int(x['minutes']) if re.fullmatch(r'[0-9]{1,5}', x['minutes'] or '') else None + return jobs + + +def shown(template, text, matrix=False): + """Is `text` the name GitHub shows for this name template (each ${{ }} any text; a job whose name has none may + carry its matrix values in parentheses)?""" + parts = re.split(r'\$\{\{.*?\}\}', template) + tail = r'(?: \(.*\))?' if matrix and len(parts) == 1 else '' + return bool(re.fullmatch('.*'.join(map(re.escape, parts)) + tail, text, re.S)) + + +def limit_of(workflow, job_name, step=None): + """The timeout-minutes the workflow gives the job shown as job_name (or its step named `step`), or None when it + gives none, or when the name matches no job or more than one, or names steps with different limits.""" + keys = [k for k, j in workflow.items() if shown(j['name'] or k, job_name, matrix=True)] + if len(keys) != 1: + return None + j = workflow[keys[0]] + if step is None: + return j['minutes'] + got = {s['minutes'] for s in j['steps'] if s['name'] is not None and shown(s['name'], step)} + return got.pop() if len(got) == 1 else None + + +def workflow_reader(cfg, gh, run): + """limits(job name, step) from CONFIG workflow's file at the run's own commit, read once and only when asked.""" + got = [] + + def limits(job_name, step=None): + if not got: + sha = str(run.get('head_sha') or '') + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError(f'run {run.get("id")}: no commit to read the workflow at') + reply = gh.call(f'contents/.github/workflows/{cfg["workflow"]}?ref={sha}') + if reply.get('encoding') != 'base64': + raise ValueError(f'{cfg["workflow"]} at {sha[:12]}: not readable through the contents API') + got.append(workflow_limits(base64.b64decode(reply['content']).decode('utf-8'))) + return limit_of(got[0], job_name, step) + return limits + + +def limit_hit(cfg, job, annotations, limits=None): + """GitHub's time-limit message, when the job's record agrees: its minutes are the job's (or the step's) + timeout-minutes in the workflow (limits), the job (or the step it names) ran at least that long and no other + step failed. (message, None), or (None, why) when there is no such message.""" + why = None + for a in annotations: + if not from_runner(a): + continue + msg = str(a.get('message') or '').strip() + for p in cfg['signs']['limit']: + m = re.fullmatch(p, msg, re.S) + if not m: + continue + bound = int(m['minutes']) * 60 - cfg['limit_slack_seconds'] + step = m.groupdict().get('step') + others = [s for s in failed_steps(job) if s != step] + if others: + why = f'GitHub\'s time-limit text, but step {others[0]!r} failed' + continue + own = limits(job.get('name') or '', step) if limits else None + if own != int(m['minutes']): + why = (f'GitHub\'s time-limit text says {m["minutes"]} min, but the workflow gives ' + f'{"step " + repr(step) if step is not None else "the job"} {own} min: not GitHub\'s own') + continue + if step is not None: + ran = [took(s) or 0 for s in job.get('steps') or [] if s.get('name') == step + and s.get('conclusion') in ('failure', 'cancelled')] + if not any(t >= bound for t in ran): + why = f'GitHub\'s time-limit text, but no step {step!r} ran {m["minutes"]} min and ended red' + continue + elif (took(job) or 0) < bound: + why = f'GitHub\'s time-limit text, but the job ran {int((took(job) or 0) // 60)} min, not {m["minutes"]}' + continue + return msg, None + return None, why + + +def numbers(note): + """The note's numbers, for the log.""" + h = note['host'] + out = [f'machine: full {h.get("full_share")}% some {h.get("some_share")}% of {note.get("seconds")} s, ' + f'full avg10/60/300 {h.get("full_avg10")}/{h.get("full_avg60")}/{h.get("full_avg300")}%, ' + f'available {h.get("available_mb")} MB, swap used {h.get("swap_used_mb")} MB, OOM kills {h.get("oom_kill")}'] + out += [f'box {c.get("path")}: max {c.get("max_mb")} MB high {c.get("high_mb")} MB use {c.get("current_mb")} MB, ' + f'events high {c.get("high")} max {c.get("max")} oom_kill {c.get("oom_kill")}, full {c.get("full_share")}%' + for c in note['cgroups']] + return '; '.join(out) + + +def job_sign(cfg, job, annotations, limits=None): + """(sign, evidence) when a failed job shows an infrastructure sign, else (None, why not).""" + cancel = github_says(cfg, 'cancelled_by', annotations) + if cancel: + return None, f'GitHub: "{cancel[0][:200]}": a person\'s cancel stands' + note, why = note_of(cfg, job, annotations) + lost = github_says(cfg, 'lost', annotations) + if lost: + msg = f'GitHub: "{lost[0][:200]}"' + if job.get('conclusion') != 'failure': + return None, f'{msg}, but the job ended {job.get("conclusion")}, not failure' + if failed_steps(job): + return None, f'{msg}, but step {failed_steps(job)[0]!r} failed: the text is a step\'s output' + if github_says(cfg, 'limit', annotations): + return None, f'{msg}, but GitHub also says it ran past a time limit: a step that hung, not a lost runner' + runner = lost[1].groupdict().get('runner') + if runner is not None and job.get('runner_name') and runner != job['runner_name']: + return None, f'{msg}, but the job ran on {job["runner_name"]!r}: not its runner' + if end_step_ran(cfg, job): + return None, f'{msg}, but its "{cfg["probe_end_step"]}" step completed: the runner was not lost' + return 'lost', f'{msg}; no step failed and the job never reached its end step (no numbers: the runner died)' + full = memory_full(cfg, job, note) if note else None + if full: + return 'memory full', f'{full} ({numbers(note)})' + hit, not_hit = limit_hit(cfg, job, annotations, limits) + if hit and note: + stall = stalled(cfg, note) + if stall: + return 'limit while stalled', f'GitHub: "{hit[:200]}"; {stall} ({numbers(note)})' + return None, f'ran past its time limit, but the machine was not stalled ({numbers(note)})' + if hit: + return None, f'ran past its time limit, and {why}: cannot tell it was stalled' + return None, ('no infrastructure sign (a failure of its own, or a cancel)' + (f'; {not_hit}' if not_hit else '') + + (f'; {why}' if not note else '')) + + +def annotations_of(gh, job): + m = CHECK_RUN.search(job.get('check_run_url') or '') + return gh.pages(f'check-runs/{int(m[1]) if m else int(job["id"])}/annotations', most=3) + + +def current(gh, run): + """Is the run's commit still the head of its branch? A newer push makes an old run moot.""" + try: + branch = gh.call(f'branches/{urllib.parse.quote(run.get("head_branch") or "", safe="")}') + except urllib.error.HTTPError as e: + if e.code == 404: + return False # the branch is gone (merged and deleted) + raise + return (branch.get('commit') or {}).get('sha') == run.get('head_sha') + + +def decide(cfg, gh, run, env): + """('quiet', None) for a run that is no candidate, ('leave', why), ('report', evidence) or ('rerun', evidence).""" + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + if run.get('status') != 'completed' or run.get('conclusion') not in RED: + return 'quiet', None + actors = cfg['fleet']['trusted']['actors'] + by = login(run.get('triggering_actor')) + if attempt > 1 and by not in actors: + return 'leave', f'attempt {attempt} was triggered by {by!r} (an automatic re-run): a repeat red stays red' + why = FLEET.run_refused(cfg['fleet'], gh, run) + if why: + return 'leave', f'not a trusted run: {why}' + for n in range(attempt - 1, 0, -1): + by = login(gh.call(f'actions/runs/{rid}/attempts/{n}').get('triggering_actor')) + if by not in actors: + return 'leave', f'attempt {n} was triggered by {by!r} (an automatic re-run): never twice' + jobs = gh.pages(f'actions/runs/{rid}/jobs?filter=latest', 'jobs') + red = [j for j in jobs if j.get('conclusion') in RED and j.get('name') not in cfg['followers']] + if not red: + return 'leave', 'no failed job of its own (only followers failed, or nothing did)' + evidence, limits = [], workflow_reader(cfg, gh, run) + for j in red: + sign, why = job_sign(cfg, j, annotations_of(gh, j), limits) + if not sign: + return 'leave', f'{j.get("name")} ended {j.get("conclusion")}: {why}' + evidence.append(f'{j.get("name")} ({j.get("conclusion")}): {sign}: {why}') + if not current(gh, run): + return 'leave', f'{run.get("head_branch")!r} has moved on or is gone; it qualified: ' + ' | '.join(evidence) + skip = cfg['skip_when'] + if (env.get(skip['variable']) or '').strip() in skip['is']: + return 'report', ' | '.join(evidence) + return 'rerun', ' | '.join(evidence) + + +# ---------------------------------------------------------------- the tracking issue + +class Notice: + """The one tracking issue (CONFIG notice): the durable, visible record of every automatic re-run outside this + sweep's own run. Only an open issue github-actions opened with exactly that title is ours, never one a person + opened; with none open, the first re-run opens it. Each re-run (and each fleet-mode report) is one comment with a + marker line (MARK: run, attempt, what), so a pass never writes the same one twice and the count reads them.""" + + def __init__(self, cfg, gh, now): + self.cfg, self.gh, self.now, self.hours = cfg['notice'], gh, now, cfg['window_hours'] + self.issue, self.comments = None, None + + def _ours(self): + n = self.cfg + listed = self.gh.pages(f'issues?state=open&creator={urllib.parse.quote(n["author"], safe="")}' + f'&sort=created&direction=asc') + return sorted(int(i['number']) for i in listed if isinstance(i, dict) and 'pull_request' not in i + and i.get('title') == n['title'] and login(i.get('user')) == n['author']) + + def load(self, since=None): + if self.comments is not None: + return + ours = self._ours() + self.issue = ours[0] if ours else None + start = iso(min(self.now() - self.hours * 3600, since or self.now())) + self.comments = [] if self.issue is None else [ + c for c in self.gh.pages(f'issues/{self.issue}/comments?since={start}') + if isinstance(c, dict) and login(c.get('user')) == self.cfg['author']] + + def marks(self): + out = {} + for c in self.comments or []: + m = MARK.match(str(c.get('body') or '')) # only a comment's first line: quoted text is never one + if m: + out.setdefault((int(m[1]), int(m[2]), m[3]), c.get('created_at')) + return out + + def has(self, rid, attempt, what): + self.load() + return (int(rid), int(attempt), what) in self.marks() + + def count(self): + """Re-runs this sweep asked for in the last window_hours, from the issue's markers.""" + self.load() + since = self.now() - self.hours * 3600 + return sum(1 for (_, _, what), at in self.marks().items() if what == 'rerun' and (stamp(at) or 0) >= since) + + def where(self): + return f'issue #{self.issue}' if self.issue else 'no tracking issue yet' + + def _open(self): + n = self.cfg + got = self.gh.call('issues', method='POST', data={ + 'title': n['title'], 'labels': n['labels'], + 'body': ('Each comment below is one automatic re-run of a failed run (or, in fleet mode, a run that ' + 'would have had one), with its jobs, the signs and the memory pressure numbers. Many of them ' + 'mean a machine that stalls or fills up: fix that, the re-run only hides it for one attempt. ' + 'Closing this issue is fine; the next re-run opens a new one. Rules and runbook: ' + '`.github/fork-ci-rerun.md`.')}) + mine = int(got['number']) + # A dispatched pass may run beside a scheduled one (fork-ci-rerun.yml); if both opened one, the lowest wins + # and the other is closed, pointing at it, so there is only ever one. + ours = self._ours() + if ours and ours[0] < mine: + self.gh.call(f'issues/{mine}/comments', method='POST', data={'body': f'Duplicate of #{ours[0]}.'}) + self.gh.call(f'issues/{mine}', method='PATCH', data={'state': 'closed', 'state_reason': 'not_planned'}) + return ours[0] + return mine + + def add(self, run, what, text): + """Write one comment for this run's attempt; returns the issue's number.""" + self.load() + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + url = run.get('html_url') or f'run {rid}' + head = (f'**Re-run asked** for [run {rid}]({url}) attempt {attempt}: GitHub\'s "re-run failed jobs" makes ' + f'attempt {attempt + 1}; if the run has no attempt {attempt + 1}, the request did not go through ' + '(the sweep\'s log says why).' if what == 'rerun' else + f'**Not re-run (fleet mode)**: [run {rid}]({url}) attempt {attempt} qualifies for its one automatic ' + 're-run; re-run it by hand.') + body = (f'\n{head}\n\n' + f'{quoted(run.get("event"))} `{quoted(run.get("head_branch"))}` `{str(run.get("head_sha"))[:12]}`\n\n' + + ''.join(f'- {quoted(part.strip())}\n' for part in text.split(' | ')))[:LONGEST] + if self.issue is None: + self.issue = self._open() + got = self.gh.call(f'issues/{self.issue}/comments', method='POST', data={'body': body}) + self.comments.append(dict(got if isinstance(got, dict) else {}, body=body, + created_at=(got or {}).get('created_at') or iso(self.now()))) + return self.issue + + +def quoted(text): + """Text from annotations and notes, safe in the public comment: one line, no @mention (a zero-width joiner after + each @), and no HTML (so never a marker line, MARK, of another run's record).""" + return (str(text).replace('\r', ' ').replace('\n', ' ').replace('@', '@\u200d').replace('&', '&') + .replace('<', '<').replace('>', '>')) + + +def rerun(gh, run, record): + """GitHub's own re-run of failed jobs (and the jobs after them), once: (True, how) when this pass asked for it. + record() writes the durable notice first, so a reply lost after GitHub took the request still leaves the why.""" + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + fresh = gh.call(f'actions/runs/{rid}') + if int(fresh.get('run_attempt') or 1) != attempt or fresh.get('status') != 'completed': + return False, f'already re-run or running again (attempt {fresh.get("run_attempt")}, {fresh.get("status")})' + record() + try: + gh.call(f'actions/runs/{rid}/rerun-failed-jobs', method='POST') + except urllib.error.HTTPError as e: + again = gh.call(f'actions/runs/{rid}') + if int(again.get('run_attempt') or 1) != attempt or again.get('status') != 'completed': + return False, f'already re-run (GitHub refused a second request: HTTP {e.code})' + raise + return True, f'attempt {attempt + 1} asked for' + + +def summary(env, lines): + path = env.get('GITHUB_STEP_SUMMARY') + if path and lines: + with open(path, 'a', encoding='utf-8') as f: + f.write(''.join(f'{line}\n' for line in lines)) + + +def cell(text): + return str(text).replace('|', '\\|').replace('\n', ' ') + + +def sweep(cfg, env, gh, now=time.time, run_id=None, out=None): + notice = Notice(cfg, gh, now) + rows, trouble, asked = [], 0, 0 + try: + if run_id: + runs = [gh.call(f'actions/runs/{int(run_id)}')] + else: + since = iso(now() - cfg['list_hours'] * 3600) + runs = gh.pages(f'actions/workflows/{cfg["workflow"]}/runs?created=%3E%3D{since}', 'workflow_runs') + except TROUBLE as e: + runs, trouble = [], 1 + say(f'::warning::fork-ci re-run: the runs: {e.__class__.__name__}: {e}; the next pass tries again', out) + rows.append(f'| - | not read | {cell(e.__class__.__name__)}: {cell(e)} |') + recent = now() - cfg['catch_up_minutes'] * 60 + for run in runs: + rid = run.get('id') + try: + if not run_id and (stamp(run.get('updated_at')) or 0) < recent: + continue # ended before the catch-up window: too old to re-run now + what, text = decide(cfg, gh, run, env) + where = f'run {rid} ({run.get("event")} {run.get("head_branch")} {str(run.get("head_sha"))[:12]})' + attempt = int(run.get('run_attempt') or 1) + if what == 'quiet': + continue + if what == 'leave': + say(f'{where}: left red: {text}', out) + rows.append(f'| {rid} | left red | {cell(text)} |') + continue + notice.load(since=stamp(run.get('created_at'))) + if what == 'report': + skip = cfg['skip_when'] + state = f'{skip["variable"]}={env.get(skip["variable"])}' + if notice.has(rid, attempt, 'report'): + say(f'{where}: qualifies, not re-run ({state}); reported already ({notice.where()})', out) + rows.append(f'| {rid} | not re-run ({cell(state)}), reported already | {cell(text)} |') + continue + issue = notice.add(run, 'report', text) + say(f'::error title=fork-ci re-run::{where} qualifies for its one automatic re-run, but {state}: ' + f'not re-run; re-run it by hand (.github/fork-ci-rerun.md, issue #{issue}). {text}', out) + rows.append(f'| {rid} | not re-run ({cell(state)}), issue #{issue} | {cell(text)} |') + continue + + def record(): + if not notice.has(rid, attempt, 'rerun'): + notice.add(run, 'rerun', text) + done, how = rerun(gh, run, record) + if done: + asked += 1 + say(f'::warning title=fork-ci re-run::{where} re-run once ({how}, {notice.where()}), ' + f'infrastructure: {text}', out) + rows.append(f'| {rid} | **re-run** ({cell(how)}, {notice.where()}) | {cell(text)} |') + else: + say(f'{where}: {how}', out) + rows.append(f'| {rid} | {cell(how)} | {cell(text)} |') + except TROUBLE as e: + trouble += 1 + say(f'::warning::fork-ci re-run: run {rid}: {e.__class__.__name__}: {e}; the next pass tries again', out) + rows.append(f'| {rid} | not read | {cell(e.__class__.__name__)}: {cell(e)} |') + try: + count = (f'{asked} run(s) re-run by this pass; {notice.count()} automatic re-run(s) asked in the last ' + f'{cfg["window_hours"]} h ({notice.where()})') + except TROUBLE as e: + trouble += 1 + count = (f'{asked} run(s) re-run by this pass; the tracking issue could not be read for the ' + f'{cfg["window_hours"]} h count ({e.__class__.__name__}: {e}): red, the next pass tries again') + say(f'fork-ci re-run: {count}', out) + lines = ['### fork-ci re-run', '', count, ''] + if rows: + lines += ['| Run | Outcome | Why |', '| --- | --- | --- |'] + rows + summary(env, lines) + return 1 if trouble else 0 + + +def main(argv=None, env=os.environ): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument('verb', choices=('sweep', 'check')) + p.add_argument('--run-id', type=int) + p.add_argument('--config', default=CONFIG) + a = p.parse_args(argv) + cfg = load(a.config) + if a.verb == 'check': + say(f'{a.config}: ok') + return 0 + return sweep(cfg, env, GitHub(env.get('GITHUB_TOKEN'), cfg['fleet']['repository']), run_id=a.run_id) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/fork-ci-rerun.test.py b/.github/scripts/fork-ci-rerun.test.py new file mode 100644 index 0000000000000..d0bc7bb47063c --- /dev/null +++ b/.github/scripts/fork-ci-rerun.test.py @@ -0,0 +1,1100 @@ +#!/usr/bin/env python3 +"""Tests of fork-ci-rerun.py (the one automatic re-run of an infrastructure failure), with a fake GitHub that pages +like the real one (the script's own GitHub.pages reads it), and of its wiring in fork-ci-rerun.yml. Run: python3 .""" +import base64, copy, io, json, os, re, shutil, subprocess, sys, tempfile, time, unittest, urllib.error, urllib.parse +import importlib.util + +sys.dont_write_bytecode = True # importing the script must not leave __pycache__ in the tree + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SCRIPT = os.path.join(HERE, 'fork-ci-rerun.py') +_spec = importlib.util.spec_from_file_location('fork_ci_rerun', SCRIPT) +cr = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cr) +_spec = importlib.util.spec_from_file_location('fork_ci_pressure', os.path.join(HERE, 'fork-ci-pressure.py')) +probe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(probe) + +CFG = cr.load() +FLEET_CFG = CFG['fleet'] +REPO = FLEET_CFG['repository'] +OWNER = FLEET_CFG['trusted']['actors'][0] +BRANCH = FLEET_CFG['trusted']['push_branches'][0] +FOLLOWER = CFG['followers'][0] +END_STEP = CFG['probe_end_step'] +SKIP_VAR, SKIP_VALUE = CFG['skip_when']['variable'], CFG['skip_when']['is'][0] +TITLE = CFG['notice']['title'] +BOT = CFG['notice']['author'] +ACTIONS = 'github-actions' # the triggering actor of GitHub's re-run from a workflow token +HOSTED = CFG['hosted_runner_groups'][0] +T0 = 1_800_000_000 +SHA = '0123456789abcdef0123456789abcdef01234567' +NEWER = 'fedcba9876543210fedcba9876543210fedcba98' + +# The fleet's trust rule asks git whether the upstream sync made a pushed commit; here, a set. +SYNCED = set() +cr.FLEET.made_by_the_sync = lambda cfg, sha, run=None: (f'commit {sha[:12]} was pushed by the upstream sync' + if sha in SYNCED else None) + +# GitHub's own messages, word for word as its annotations carry them. +LOST = ('The self-hosted runner: fleet-vm-1 lost communication with the server. Verify the machine is running and ' + 'has a healthy network connection. Anything in your workflow that terminates the runner process, starves it ' + 'for CPU/Memory, or blocks its network access can cause this error.') +HOSTED_LOST = 'The hosted runner: GitHub Actions 7 lost communication with the server. Anything in your workflow...' +SHUTDOWN = 'The runner has received a shutdown signal. This can happen when the runner service is stopped.' +JOB_LIMIT = 'The job running on runner fleet-vm-1 has exceeded the maximum execution time of 15 minutes.' +STEP_LIMIT = "The action 'Test' has timed out after 10 minutes." +CANCEL_BY = f'The run was canceled by @{OWNER}.' +CANCELED = 'The operation was canceled.' +EXIT_1 = 'Process completed with exit code 1.' +EXIT_137 = 'Process completed with exit code 137.' +# The workflow file at the run's commit, as the contents API hands it back: the limits GitHub's messages must match. +WORKFLOW = f"""name: Fork tests +on: push +jobs: + test: + name: Test + strategy: + matrix: + shard: [1, 2] + timeout-minutes: 15 + steps: + - name: Checkout + - name: Note memory pressure at the start + timeout-minutes: 1 + run: | + python3 x.py + - name: Test + timeout-minutes: 10 # the step's own + - name: '{END_STEP}' + timeout-minutes: 1 + lint: + name: Lint + timeout-minutes: 20 + steps: + - run: true +""" +MEMORY_WORDS = 'OOM kill: memory.max hit, memory full, the box stalled (full) 90% of the job' + + +def iso(t): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t)) + + +def ann(message, level='failure', title='', path='.github'): + """An annotation as GitHub hands it back. A step's ::error::/::warning:: has the same fields as the runner's own.""" + return {'message': message, 'annotation_level': level, 'title': title, 'path': path} + + +def note(host=None, cgroups=None, status='failure', seconds=600): + """A pressure note as fork-ci-pressure.py writes it, decoded as GitHub hands it back.""" + h = {'full_share': 0.0, 'some_share': 0.5, 'full_avg10': 0.0, 'full_avg60': 0.0, 'full_avg300': 0.0, + 'some_avg60': 0.1, 'some_avg300': 0.1, 'available_mb': 9000, 'swap_used_mb': 0, 'oom_kill': 0} + h.update(host or {}) + boxes = cgroups if cgroups is not None else [box()] + body = json.dumps({'status': status, 'seconds': seconds, 'host': h, 'cgroups': boxes}, separators=(',', ':')) + return ann(cr.NOTE_HEAD + body, level='notice', title=cr.NOTE_TITLE) + + +def box(**kw): + b = {'path': '/fleet.slice/job-1.service', 'high_mb': 6144, 'max_mb': 8192, 'current_mb': 3000, + 'high': 0, 'max': 0, 'oom': 0, 'oom_kill': 0, 'full_share': 0.0} + b.update(kw) + return b + + +CALM = note() +FULL = note(cgroups=[box(oom_kill=1, full_share=30.0)]) # the box stalled and an OOM kill in it +STALLED = note(host={'full_share': 25.0}) + + +def step(name, conclusion='success', status='completed', took=5, at=0): + return {'name': name, 'status': status, 'conclusion': conclusion, + 'started_at': iso(T0 - 3600 + at), 'completed_at': iso(T0 - 3600 + at + took) if status == 'completed' + else None} + + +def ran(test='failure', took=60, end=True): + """A job that reached its end step (the note is its own).""" + return [step('Checkout'), step('Note memory pressure at the start', at=5), step('Test', test, took=took, at=10), + step(END_STEP, at=10 + took) if end else step(END_STEP, None, 'queued')] + + +def died(): + """A job whose runner went away mid-test: the test step never ended, the end step never ran.""" + return [step('Checkout'), step('Note memory pressure at the start', at=5), + step('Test', None, 'in_progress', at=10), step(END_STEP, None, 'queued')] + + +def run(rid=1, attempt=1, conclusion='failure', status='completed', event='push', branch=BRANCH, actor=OWNER, + by=OWNER, repo=REPO, head_repo=REPO, updated=T0 - 120, created=T0 - 3600, workflow='fork-tests.yml', pr=7): + r = {'id': rid, 'run_attempt': attempt, 'status': status, 'conclusion': conclusion, 'event': event, + 'head_branch': branch, 'head_sha': SHA, 'path': f'.github/workflows/{workflow}', + 'html_url': f'https://github.example/{repo}/actions/runs/{rid}', + 'repository': {'full_name': repo}, 'head_repository': {'full_name': head_repo}, + 'actor': {'login': actor}, 'triggering_actor': {'login': by}, 'updated_at': iso(updated), + 'created_at': iso(created), 'pull_requests': []} + if event == 'pull_request' and head_repo == repo: + r['pull_requests'] = [{'number': pr}] # GitHub lists none for a pull request from another fork + return r + + +def job(jid, name='Test (1)', conclusion='failure', steps=None, group='fleet', runner='fleet-vm-1', took=600): + return {'id': jid, 'name': name, 'conclusion': conclusion, 'status': 'completed', + 'steps': steps if steps is not None else ran(), 'runner_group_name': group, 'runner_name': runner, + 'started_at': iso(T0 - 3600), 'completed_at': iso(T0 - 3600 + took), + 'check_run_url': f'https://api.example/repos/{REPO}/check-runs/{jid}'} + + +class Fake: + """GitHub's REST API as fork-ci-rerun.py uses it. Listings page like GitHub's (page/per_page), read by the script's + own GitHub.pages; a re-run makes a new attempt triggered by github-actions; issues and comments are kept.""" + + pages = cr.GitHub.pages # the real paging code, over this fake's call() + + def __init__(self, runs, jobs=None, annotations=None, heads=None, pulls=None): + self.runs = {r['id']: r for r in runs} + self.jobs = jobs or {} + self.ann = annotations or {} + self.heads = heads if heads is not None else {BRANCH: SHA, 'feature': SHA} + self.pulls = pulls if pulls is not None else {7: {'head': {'repo': {'full_name': REPO}}, + 'base': {'repo': {'full_name': REPO}}, + 'user': {'login': OWNER}}} + self.issues = [] + self.attempts = {} + self.calls = [] + self.now = T0 + self.post_fails = None # an exception to raise for the next re-run POST + self.lost_reply = False # the re-run POST reaches GitHub, its reply never comes back + self.comment_fails = False # a comment POST fails + self.on_read = None # called with the run before GET actions/runs/N answers + self.workflow = WORKFLOW # the workflow file at every commit (None: not found) + self.comments_fail = False # reading an issue's comments fails + + def call(self, path, method='GET', data=None): + self.calls.append((method, path)) + base, _, query = path.partition('?') + q = urllib.parse.parse_qs(query) + + def paged(items): + if 'page' not in q: + return copy.deepcopy(items) + n, per = int(q['page'][0]), int(q['per_page'][0]) + return copy.deepcopy(items[(n - 1) * per:n * per]) + + if method == 'POST' and base == 'issues': + number = max([i['number'] for i in self.issues] + [40]) + 1 + self.issues.append({'number': number, 'title': data['title'], 'labels': data['labels'], + 'body': data['body'], 'state': 'open', 'user': {'login': BOT}, 'comments': []}) + return {'number': number} + m = re.fullmatch(r'issues/(\d+)/comments', base) + if m and method == 'POST': + if self.comment_fails: + raise urllib.error.URLError('comment: reset') + issue = self.issue(int(m[1])) + c = {'id': 9000 + sum(len(i['comments']) for i in self.issues), 'body': data['body'], + 'user': {'login': BOT}, 'created_at': iso(self.now)} + issue['comments'].append(c) + return copy.deepcopy(c) + m = re.fullmatch(r'issues/(\d+)', base) + if m and method == 'PATCH': + self.issue(int(m[1])).update(data) + return {} + if method == 'POST': + m = re.fullmatch(r'actions/runs/(\d+)/rerun-failed-jobs', path) + if not m: + raise AssertionError(f'unexpected POST {path}') + r = self.runs[int(m[1])] + if self.post_fails: + e, self.post_fails = self.post_fails, None + raise e() if callable(e) and not isinstance(e, BaseException) else e + if r['status'] != 'completed': + raise urllib.error.HTTPError(path, 403, 'This workflow is already running', {}, None) + self.attempts[(r['id'], r['run_attempt'])] = copy.deepcopy(r) + r.update(run_attempt=r['run_attempt'] + 1, status='queued', conclusion=None, + triggering_actor={'login': ACTIONS}) + if self.lost_reply: + raise urllib.error.URLError('connection reset') + return {} + if method != 'GET': + raise AssertionError(f'unexpected {method} {path}') + if base == f'contents/.github/workflows/{CFG["workflow"]}': + assert q['ref'] == [SHA], q + if self.workflow is None: + raise urllib.error.HTTPError(path, 404, 'Not Found', {}, None) + return {'encoding': 'base64', 'content': base64.b64encode(self.workflow.encode()).decode()} + if base == f'actions/workflows/{CFG["workflow"]}/runs': + since = cr.stamp(urllib.parse.unquote(q['created'][0]).lstrip('>=')) + listed = sorted((r for r in self.runs.values() if cr.stamp(r['created_at']) >= since), + key=lambda r: -r['id']) # newest first, as GitHub lists them + return {'total_count': len(listed), 'workflow_runs': paged(listed)} + m = re.fullmatch(r'actions/runs/(\d+)/jobs', base) + if m: + return {'jobs': paged(self.jobs.get(int(m[1]), []))} + m = re.fullmatch(r'actions/runs/(\d+)', base) + if m: + if self.on_read: + self.on_read(self.runs[int(m[1])]) + return copy.deepcopy(self.runs[int(m[1])]) + m = re.fullmatch(r'actions/runs/(\d+)/attempts/(\d+)', base) + if m: + got = self.attempts.get((int(m[1]), int(m[2]))) + if got is None: + got = dict(self.runs[int(m[1])], run_attempt=int(m[2]), triggering_actor={'login': OWNER}) + return copy.deepcopy(got) + m = re.fullmatch(r'check-runs/(\d+)/annotations', base) + if m: + return paged(self.ann.get(int(m[1]), [])) + m = re.fullmatch(r'branches/(.+)', base) + if m: + if m[1] not in self.heads: + raise urllib.error.HTTPError(path, 404, 'Branch not found', {}, None) + return {'commit': {'sha': self.heads[m[1]]}} + m = re.fullmatch(r'pulls/(\d+)', base) + if m: + return copy.deepcopy(self.pulls[int(m[1])]) + if base == 'issues': + assert q['state'] == ['open'] and q['creator'] == [BOT], q + # The script checks the author itself: here the creator filter is not applied. + listed = [{k: v for k, v in i.items() if k != 'comments'} for i in self.issues if i['state'] == 'open'] + return paged(listed) + m = re.fullmatch(r'issues/(\d+)/comments', base) + if m: + if self.comments_fail: + raise urllib.error.URLError('comments: reset') + since = cr.stamp(q['since'][0]) + # GitHub's `since` is by updated_at: an old comment edited lately is listed too. + return paged([c for c in self.issue(int(m[1]))['comments'] + if cr.stamp(c.get('updated_at') or c['created_at']) >= since]) + raise AssertionError(f'unexpected GET {path}') + + def issue(self, number): + return next(i for i in self.issues if i['number'] == number) + + def posts(self): + return [p for m, p in self.calls if m == 'POST' and p.startswith('actions/')] + + def comments(self): + return [c['body'] for i in self.issues for c in i['comments']] + + +def sweep(gh, env=None, run_id=None, cfg=CFG): + out = io.StringIO() + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, 'summary.md') + code = cr.sweep(cfg, dict(env or {}, GITHUB_STEP_SUMMARY=path), gh, now=lambda: gh.now, run_id=run_id, + out=out) + text = '' + if os.path.exists(path): + with open(path, encoding='utf-8') as f: + text = f.read() + return code, out.getvalue(), text + + +def one(annotations, job_kw=None, **kw): + """A failed run whose one failed job (id 11) carries these annotations, plus a passed job and the follower.""" + jobs = [job(11, **(job_kw or {})), job(12, 'Lint', 'success'), job(13, FOLLOWER, 'failure', steps=[])] + return Fake([run(**kw)], {1: jobs}, {11: annotations, 13: [ann(EXIT_1)]}) + + +def lost(**kw): + return one([ann(LOST), ann(CANCELED)], dict(steps=died()), **kw) + + +class Rerun(unittest.TestCase): + def assertReran(self, gh, out): + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertIn('::warning title=fork-ci re-run::run 1', out) + + def assertLeft(self, gh, code, out, why=None): + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('left red', out) + if why: + self.assertIn(why, out) + + +class Signs(Rerun): + def test_lost_runner_is_rerun_once_with_the_message_in_the_log(self): + for message, runner in ((LOST, 'fleet-vm-1'), (SHUTDOWN, 'fleet-vm-1'), (HOSTED_LOST, 'GitHub Actions 7')): + with self.subTest(message[:30]): + gh = one([ann(message), ann(CANCELED)], dict(steps=died(), runner=runner)) + code, out, text = sweep(gh) + self.assertEqual(code, 0, out) + self.assertReran(gh, out) + self.assertIn('lost', out) + self.assertIn('**re-run**', text) + + def test_a_full_box_is_rerun_with_its_numbers_in_the_log(self): + cases = { + 'box OOM kills while it stalled': FULL, + 'box memory.max hits while it stalled': note(cgroups=[box(max=3, full_share=12.0)]), + 'memory.high throttling while the box stalled': note(cgroups=[box(high=40, full_share=30.0)]), + 'a box with only a high limit': note(cgroups=[box(max_mb=None, oom_kill=2, full_share=15.0)]), + } + for label, n in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), n]) + code, out, text = sweep(gh) + self.assertReran(gh, out) + self.assertIn('memory full', out) + self.assertRegex(out, r'machine: full [0-9.]+% some [0-9.]+% of 600 s') + self.assertIn('box /fleet.slice/job-1.service: max', out) + self.assertIn('memory full', text) + + def test_one_event_or_a_stall_alone_is_no_memory_sign(self): + cases = { + 'one OOM kill, the box not stalled': note(cgroups=[box(oom_kill=1, full_share=2.0)]), + 'one memory.max hit, not stalled': note(cgroups=[box(max=1)]), + 'memory.high throttling, not stalled': note(host={'full_share': 2.0}, cgroups=[box(high=40)]), + 'an OOM kill on the machine, not in the box': note(host={'oom_kill': 3, 'full_share': 50.0}), + 'the machine stalled, the box had no event': note(host={'full_share': 50.0}, + cgroups=[box(full_share=50.0)]), + 'a cgroup with no memory limit': note(cgroups=[box(max_mb=None, high_mb=None, oom_kill=4, + full_share=80.0)]), + 'numbers that are not numbers': note(cgroups=[box(oom_kill=True, max='9', full_share='90')]), + } + for label, n in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), n]) + self.assertLeft(gh, *sweep(gh)[:2], 'no infrastructure sign') + + def test_an_oom_kill_on_a_github_hosted_runner_never_qualifies(self): + for group in (HOSTED, None, ''): + with self.subTest(group=group): + gh = one([ann(EXIT_137), FULL], dict(group=group)) + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_a_memory_sign_needs_the_job_and_its_note_to_have_failed(self): + for kw, status in ((dict(conclusion='cancelled'), 'failure'), ({}, 'cancelled'), ({}, 'success')): + with self.subTest(kw=kw, status=status): + n = note(cgroups=[box(oom_kill=1, full_share=30.0)], status=status) + gh = one([ann(EXIT_137), n], kw) + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_time_limit_while_stalled_is_rerun(self): + for stall in ({'full_share': 12.0}, {'full_avg300': 15.0}): + with self.subTest(stall=stall): + gh = one([ann(JOB_LIMIT), ann(CANCELED), note(host=stall, status='cancelled')], + dict(conclusion='cancelled', steps=ran('cancelled', took=880), took=900)) + code, out, _ = sweep(gh) + self.assertReran(gh, out) + self.assertIn('limit while stalled', out) + gh = one([ann(STEP_LIMIT), note(cgroups=[box(full_share=50.0)])], dict(steps=ran('failure', took=600))) + self.assertReran(gh, sweep(gh)[1]) + + def test_a_timed_out_run_is_a_candidate(self): + gh = one([ann(JOB_LIMIT), STALLED], dict(conclusion='timed_out', steps=ran('cancelled', took=880), took=900), + conclusion='timed_out') + self.assertReran(gh, sweep(gh)[1]) + + def test_a_time_limit_message_needs_the_time_really_spent(self): + cases = { + 'the job ran 3 of its 15 minutes': (JOB_LIMIT, dict(steps=ran('cancelled', took=170), took=180)), + 'the step ran 1 of its 10 minutes': (STEP_LIMIT, dict(steps=ran('failure', took=60))), + 'the step it names never ran': ("The action 'Other' has timed out after 10 minutes.", + dict(steps=ran('failure', took=600))), + 'another step failed': (JOB_LIMIT, dict(steps=ran('failure', took=880), took=900)), + } + for label, (message, kw) in cases.items(): + with self.subTest(label): + gh = one([ann(message), STALLED], kw) + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_a_time_limit_message_counts_only_with_the_workflows_own_minutes(self): + """A failing step can print GitHub's exact words with any number of minutes: "after 1 minutes" makes a step + that ran 10 minutes look 10 times over its limit. Only the workflow's own timeout-minutes counts.""" + cases = { + "a failing step printing GitHub's timeout line": ("The action 'Test' has timed out after 1 minutes.", + dict(steps=ran('failure', took=600))), + 'the job form with the wrong minutes': (JOB_LIMIT.replace('15 minutes', '1 minutes'), + dict(conclusion='cancelled', steps=ran('cancelled', took=880), + took=900)), + 'a step with no limit of its own': ("The action 'Checkout' has timed out after 1 minutes.", + dict(steps=[step('Checkout', 'failure', took=600)] + + ran('skipped')[1:])), + } + for label, (message, kw) in cases.items(): + with self.subTest(label): + gh = one([ann(message), note(cgroups=[box(full_share=50.0)])], kw) + self.assertLeft(gh, *sweep(gh)[:2], 'not GitHub\'s own') + with self.subTest('a job the workflow does not have'): + gh = one([ann(STEP_LIMIT), STALLED], dict(name='Gone (1)', steps=ran('failure', took=600))) + self.assertLeft(gh, *sweep(gh)[:2], 'the workflow gives step \'Test\' None min') + with self.subTest('the workflow at that commit cannot be read'): + gh = one([ann(STEP_LIMIT), STALLED], dict(steps=ran('failure', took=600))) + gh.workflow = None + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (1, [], []), out) + + def test_the_workflow_reader_reads_every_limit_of_the_real_workflow(self): + import yaml + with open(os.path.join(ROOT, '.github/workflows', CFG['workflow']), encoding='utf-8') as f: + text = f.read() + real, mine = yaml.safe_load(text)['jobs'], cr.workflow_limits(text) + self.assertEqual(sorted(mine), sorted(real)) + for key, j in real.items(): + self.assertEqual((mine[key]['name'], mine[key]['minutes']), (j.get('name'), j.get('timeout-minutes')), key) + self.assertEqual([(s['name'], s['minutes']) for s in mine[key]['steps']], + [(s.get('name'), s.get('timeout-minutes')) for s in j.get('steps') or []], key) + wf = cr.workflow_limits(text) + self.assertEqual(cr.limit_of(wf, 'Lint', CFG['probe_end_step']), 1) + self.assertEqual(cr.limit_of(wf, 'Lint'), real['lint']['timeout-minutes']) + self.assertIsNone(cr.limit_of(wf, 'Lint', 'Check'), 'a step without timeout-minutes has none of its own') + + def test_a_stalled_note_without_its_status_is_no_limit_sign(self): + n = dict(STALLED, message=STALLED['message'].replace('"status":"failure",', '')) + self.assertNotIn('status', n['message']) + gh = one([ann(STEP_LIMIT), n], dict(steps=ran('failure', took=600))) + self.assertLeft(gh, *sweep(gh)[:2], 'garbled') + + def test_time_limit_on_a_calm_machine_is_left_red(self): + gh = one([ann(JOB_LIMIT), ann(CANCELED), CALM], dict(conclusion='cancelled', steps=ran('cancelled', took=880), + took=900)) + code, out, text = sweep(gh) + self.assertLeft(gh, code, out, 'not stalled') + self.assertIn('left red', text) + + def test_time_limit_without_a_note_is_left_red_fail_closed(self): + gh = one([ann(JOB_LIMIT), ann(CANCELED)], dict(conclusion='cancelled', took=900, + steps=ran('cancelled', took=880, end=False))) + self.assertLeft(gh, *sweep(gh)[:2], 'cannot tell it was stalled') + + def test_a_test_failure_is_never_rerun(self): + gh = one([ann(EXIT_1), CALM]) + code, out, text = sweep(gh) + self.assertLeft(gh, code, out, 'Test (1) ended failure: no infrastructure sign') + self.assertIn('| 1 | left red |', text) + + def test_one_job_without_a_sign_leaves_the_whole_run_red(self): + jobs = [job(11, steps=died()), job(12, 'Test (2)'), job(13, FOLLOWER, steps=[])] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_1), CALM]}) + self.assertLeft(gh, *sweep(gh)[:2], 'Test (2) ended failure') + + def test_every_failed_job_with_a_sign_is_rerun_together(self): + jobs = [job(11, steps=died()), job(12, 'Build and Typecheck'), job(13, FOLLOWER, steps=[])] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_137), FULL]}) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertIn('Test (1) (failure): lost', out) + self.assertIn('Build and Typecheck (failure): memory full', out) + + def test_only_a_follower_failed_is_left_red(self): + gh = Fake([run()], {1: [job(12, 'Lint', 'success'), job(13, FOLLOWER, steps=died())]}, {13: [ann(LOST)]}) + self.assertLeft(gh, *sweep(gh)[:2], 'no failed job of its own') + + def test_a_job_skipped_after_a_failed_one_needs_no_sign(self): + jobs = [job(11, steps=died()), job(12, 'Lint', 'skipped'), job(13, FOLLOWER, steps=[])] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)]}) + self.assertReran(gh, sweep(gh)[1]) + + +class StepOutputIsNoSign(Rerun): + """A step can print any annotation (::error::GitHub's words, ::notice title=fork-ci-pressure::...), with exactly + the fields the runner's own have. Text is never enough: the job's own record must agree.""" + + FORGED = note(host={'full_share': 90.0, 'oom_kill': 3}, cgroups=[box(oom_kill=3, max=9, full_share=90.0)]) + + def test_a_failing_test_that_prints_the_lost_text_and_the_memory_words_is_not_rerun(self): + cases = { + 'a failed test step': ([ann(LOST), ann(MEMORY_WORDS), ann(EXIT_1), CALM], dict(steps=ran())), + 'a failed test step, the end step never ran': ([ann(LOST), ann(MEMORY_WORDS), ann(EXIT_1)], + dict(steps=ran(end=False))), + 'no failed step, but the end step ran (the runner lived)': ([ann(LOST), CALM], + dict(steps=ran('success'))), + 'then a hang into the time limit': ([ann(LOST), ann(JOB_LIMIT)], + dict(steps=ran('cancelled', took=880, end=False), took=900)), + 'on another runner': ([ann(LOST)], dict(steps=died(), runner='fleet-vm-2')), + 'a job that ended cancelled': ([ann(LOST)], dict(steps=died(), conclusion='cancelled')), + 'inside a longer line': ([ann('Error: ' + LOST)], dict(steps=died())), + 'as a warning': ([ann(LOST, 'warning')], dict(steps=died())), + 'with a title': ([ann(LOST, title='Error')], dict(steps=died())), + 'on a file': ([ann(LOST, path='src/x.test.ts')], dict(steps=died())), + 'a forged note beside the real one': ([ann(LOST), ann(EXIT_137), self.FORGED, CALM], dict(steps=ran())), + } + for label, (anns, kw) in cases.items(): + with self.subTest(label): + gh = one(anns, kw) + self.assertLeft(gh, *sweep(gh)[:2]) + self.assertEqual(gh.comments(), []) + + def test_a_note_counts_only_when_the_job_wrote_it_and_it_is_the_only_one(self): + """Each rule of note_of, pinned: the same note qualifies alone, and each bad variant does not.""" + good = one([ann(EXIT_137), self.FORGED]) + self.assertReran(good, sweep(good)[1]) + garbled = dict(self.FORGED, message=cr.NOTE_HEAD + '{"host": ') + wrong_shape = json.loads(self.FORGED['message'][len(cr.NOTE_HEAD):]) + cases = { + 'two notes, the forged one first': [self.FORGED, CALM], + 'two notes, the forged one last': [CALM, self.FORGED], + 'two forged notes': [self.FORGED, self.FORGED], + 'a note without its header': [dict(self.FORGED, message=self.FORGED['message'][len(cr.NOTE_HEAD):])], + 'another version\'s header': [dict(self.FORGED, message=self.FORGED['message'].replace(' v1 ', ' v0 ', 1))], + 'a garbled note': [garbled], + 'cgroups not a list': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + dict(wrong_shape, cgroups={'a': box(oom_kill=3, full_share=90.0)})))], + 'a box not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + dict(wrong_shape, cgroups=['oom_kill']), separators=(',', ':')))], + 'host not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps(dict(wrong_shape, host=[])))], + 'no status': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + {k: v for k, v in wrong_shape.items() if k != 'status'}))], + 'a list, not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + '[1, 2]')], + 'at error level': [dict(self.FORGED, annotation_level='failure')], + 'on a file': [dict(self.FORGED, path='src/x.ts')], + } + for label, notes in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137)] + notes) + code, out, _ = sweep(gh) + self.assertLeft(gh, code, out) + for label, steps in {'the end step never ran': ran(end=False), + 'the end step failed': ran()[:-1] + [step(END_STEP, 'failure')], + 'no steps listed': []}.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), self.FORGED], dict(steps=steps)) + self.assertLeft(gh, *sweep(gh)[:2], 'did not complete') + + +class Cancel(Rerun): + def test_a_run_a_person_cancelled_never_qualifies(self): + cases = { + 'a cancel while the box was full (an OOM elsewhere)': ([ann(CANCEL_BY), ann(CANCELED), FULL], + dict(conclusion='cancelled')), + 'a cancel while the job failed for memory': ([ann(CANCEL_BY), ann(EXIT_137), FULL], {}), + 'a cancel on a stalled machine at the time limit': ([ann(CANCEL_BY), ann(JOB_LIMIT), STALLED], + dict(conclusion='cancelled', took=900, + steps=ran('cancelled', took=880))), + 'a cancel after a lost runner': ([ann(CANCEL_BY), ann(LOST)], dict(steps=died())), + 'a cancel with no message': ([ann(CANCELED), FULL], dict(conclusion='cancelled')), + } + for label, (anns, kw) in cases.items(): + with self.subTest(label): + gh = one(anns, kw, conclusion='cancelled') + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_the_cancel_message_names_why(self): + gh = one([ann(CANCEL_BY), ann(EXIT_137), FULL]) + self.assertLeft(gh, *sweep(gh)[:2], "a person's cancel stands") + + +class NeverTwice(Rerun): + def test_a_second_failure_after_the_automatic_rerun_stays_red(self): + gh = lost(attempt=2, by=ACTIONS) + code, out, text = sweep(gh) + self.assertLeft(gh, code, out, 'a repeat red stays red') + self.assertFalse([p for _, p in gh.calls if '/jobs' in p or 'pulls/' in p], 'it costs no further read') + + def test_a_person_rerunning_after_the_automatic_rerun_gets_no_second_one(self): + for bot_attempt in (1, 2): # the loop reads every earlier attempt, down to the first + with self.subTest(bot_attempt=bot_attempt): + gh = lost(attempt=3, by=OWNER) + gh.attempts[(1, bot_attempt)] = run(attempt=bot_attempt, by=ACTIONS) + code, out, _ = sweep(gh) + self.assertLeft(gh, code, out, f'attempt {bot_attempt} was triggered by') + self.assertIn('never twice', out) + + def test_a_person_rerun_without_an_automatic_one_may_get_it(self): + gh = lost(attempt=2, by=OWNER) # attempt 1 by the owner too (the fake's default) + self.assertReran(gh, sweep(gh)[1]) + + def test_the_sweep_runs_again_on_the_run_it_reran(self): + gh = lost() + sweep(gh) + self.assertEqual(len(gh.posts()), 1) + code, out, _ = sweep(gh) # the re-run is queued: not completed, nothing to do + self.assertEqual((code, len(gh.posts()), len(gh.comments())), (0, 1, 1), out) + gh.runs[1].update(status='completed', conclusion='failure') # and it failed again, the same way + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.comments())), (0, 1, 1), out) + self.assertIn('a repeat red stays red', out) + + def test_the_request_sent_twice_github_refuses_the_second(self): + gh = lost() + first = copy.deepcopy(gh.runs[1]) + sweep(gh) + # A second pass (or the fleet sweep) read the run before the first re-ran it and asks again. + done, how = cr.rerun(gh, first, lambda: self.fail('no second record')) + self.assertFalse(done) + self.assertIn('already re-run', how) + self.assertEqual(len(gh.posts()), 1, 'the second request is not even sent: the run is read again first') + + def test_a_run_started_again_after_it_was_listed_is_not_asked_for(self): + gh = lost() + gh.on_read = lambda r: r.update(status='in_progress') # same attempt, running again (a person's re-run) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (0, [], []), out) + self.assertIn('already re-run or running again', out) + + def test_a_run_rerun_and_red_again_after_it_was_listed_is_not_asked_for(self): + """Between the listing and the fresh read, another pass re-ran it and attempt 2 already ended red: the listed + attempt 1 still qualifies, so only the fresh read's attempt number stops an attempt 3.""" + gh = lost() + gh.on_read = lambda r: r.update(run_attempt=2, status='completed', conclusion='failure', + triggering_actor={'login': ACTIONS}) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (0, [], []), out) + self.assertIn('already re-run or running again (attempt 2, completed)', out) + + def test_github_refusing_a_rerun_that_already_started_is_no_trouble(self): + gh = lost() + def race(): # another pass re-ran it between this pass's last read and its request + gh.runs[1].update(run_attempt=2, status='queued', triggering_actor={'login': ACTIONS}) + return urllib.error.HTTPError('x', 403, 'This workflow is already running', {}, None) + gh.post_fails = race + code, out, _ = sweep(gh) + self.assertEqual(code, 0, out) + self.assertIn('already re-run (GitHub refused a second request: HTTP 403)', out) + + def test_a_refusal_with_the_run_unchanged_is_trouble_and_tried_again_with_one_comment(self): + gh = lost() + gh.post_fails = urllib.error.HTTPError('x', 500, 'Server Error', {}, None) + code, out, text = sweep(gh) + self.assertEqual(code, 1, out) + self.assertIn('the next pass tries again', out) + self.assertIn('not read', text) + self.assertEqual(len(gh.comments()), 1, 'the reason is on record before the request') + code, out, _ = sweep(gh) # the next pass + self.assertEqual(code, 0, out) + self.assertEqual((len(gh.posts()), gh.runs[1]['run_attempt'], len(gh.comments())), (2, 2, 1)) + + def test_stopped_half_way_the_request_never_reached_github(self): + gh = lost() + gh.post_fails = urllib.error.URLError('timed out') + code, out, _ = sweep(gh) + self.assertEqual((code, gh.runs[1]['run_attempt']), (1, 1), out) + code, out, _ = sweep(gh) # the next pass finishes it, once + self.assertEqual((code, gh.runs[1]['run_attempt']), (0, 2), out) + sweep(gh) + self.assertEqual((len(gh.posts()), len(gh.comments())), (2, 1)) + + def test_a_reply_lost_after_github_took_the_request_still_leaves_the_why(self): + gh = lost() + gh.lost_reply = True # GitHub made attempt 2; the sweep never heard back + code, out, _ = sweep(gh) + self.assertEqual(code, 1, out) + [comment] = gh.comments() + self.assertIn('', comment) + self.assertIn('lost', comment) + self.assertIn(gh.runs[1]['html_url'], comment) + gh.lost_reply = False + for _ in range(2): # queued, never started: the next passes leave it alone + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts())), (0, 1), out) + gh.runs[1].update(status='completed', conclusion='cancelled') # it never ran (cancelled while queued) + code, out, _ = sweep(gh) + self.assertEqual((len(gh.posts()), len(gh.comments())), (1, 1), out) + self.assertIn('a repeat red stays red', out) + self.assertIn('1 automatic re-run(s) asked in the last 24 h', out) + + def test_no_record_no_rerun(self): + gh = lost() + gh.comment_fails = True # the notice cannot be written: the re-run would be hidden, so none + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (1, []), out) + gh.comment_fails = False + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.comments()), len(gh.issues)), (0, 1, 1, 1), out) + + +class TheNotice(Rerun): + def test_every_rerun_is_a_comment_on_one_issue_opened_once(self): + jobs = {1: [job(11, steps=died()), job(13, FOLLOWER, steps=[])], 2: [job(21), job(23, FOLLOWER, steps=[])]} + gh = Fake([run(1), run(2, branch='feature', event='pull_request')], jobs, + {11: [ann(LOST)], 21: [ann(EXIT_137), FULL]}) + code, out, text = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.issues)), (0, 2, 1), out) + [issue] = gh.issues + self.assertEqual((issue['title'], issue['labels'], issue['user']['login']), (TITLE, CFG['notice']['labels'], + BOT)) + bodies = {re.search(r'run=([0-9]+)', c['body'])[1]: c['body'] for c in issue['comments']} + self.assertEqual(sorted(bodies), ['1', '2']) + self.assertIn('what=rerun', bodies['1']) + second = {'body': bodies['2']} + self.assertIn('box /fleet.slice/job-1.service: max 8192 MB high 6144 MB', second['body']) + self.assertIn('OOM kill', second['body']) + self.assertIn(f'issue #{issue["number"]}', text) + self.assertIn('2 automatic re-run(s) asked in the last 24 h', text) + # A later pass, a later run: the same issue. + gh.runs[3] = run(3, created=T0 - 60) + gh.jobs[3] = [job(31, steps=died())] + gh.ann[31] = [ann(LOST)] + sweep(gh) + self.assertEqual((len(gh.issues), len(issue['comments'])), (1, 3)) + + def test_only_an_open_issue_the_bot_opened_with_exactly_the_title_is_ours(self): + gh = lost() + gh.issues = [ + {'number': 3, 'title': TITLE, 'state': 'open', 'user': {'login': OWNER}, 'comments': []}, + {'number': 4, 'title': TITLE + ' (old)', 'state': 'open', 'user': {'login': BOT}, 'comments': []}, + {'number': 5, 'title': TITLE, 'state': 'closed', 'user': {'login': BOT}, 'comments': []}, + {'number': 6, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': [], + 'pull_request': {}}, + ] + sweep(gh) + [ours] = [i for i in gh.issues if i['comments']] + self.assertNotIn(ours['number'], (3, 4, 5, 6)) + self.assertEqual((ours['title'], ours['user']['login']), (TITLE, BOT)) + + def test_two_passes_opening_it_at_once_keep_the_lowest(self): + gh = lost() + real = gh.call + def call(path, method='GET', data=None): + got = real(path, method, data) + if method == 'POST' and path == 'issues' and len(gh.issues) == 1: + # A dispatched pass opened one too, just before this one. + gh.issues.insert(0, {'number': 10, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, + 'comments': []}) + return got + gh.call = call + sweep(gh) + lowest, mine = sorted(gh.issues, key=lambda i: i['number']) + self.assertEqual((lowest['number'], len(lowest['comments'])), (10, 1)) + self.assertEqual((mine['state'], mine['comments'][0]['body']), ('closed', 'Duplicate of #10.')) + + def test_the_24_hour_count_reads_every_comment_past_hundreds_of_reruns(self): + gh = Fake([]) + gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + comments = gh.issues[0]['comments'] + for n in range(350): # one every 4 minutes: 350 in the last 24 h + comments.append({'id': n, 'user': {'login': BOT}, 'created_at': iso(T0 - 26 * 3600 + n * 240), + 'body': f''}) + comments.append({'id': 999, 'user': {'login': BOT}, 'created_at': iso(T0 - 60), + 'body': ''}) + comments.append({'id': 997, 'user': {'login': BOT}, 'created_at': iso(T0 - 30 * 3600), + 'updated_at': iso(T0 - 60), 'body': ''}) + comments.append({'id': 998, 'user': {'login': OWNER}, 'created_at': iso(T0 - 60), + 'body': ''}) + inside = sum(1 for c in comments[:350] if cr.stamp(c['created_at']) >= T0 - 24 * 3600) + self.assertGreater(inside, 300) + code, out, _ = sweep(gh) + self.assertEqual(code, 0, out) + self.assertIn(f'{inside} automatic re-run(s) asked in the last 24 h (issue #2)', out) + self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('issues/2/comments')), 4, 'four pages, no more') + + def test_an_issue_that_cannot_be_read_ends_the_pass_red_and_asks_nothing(self): + for label, gh in (('no run to look at', Fake([])), ('a run that qualifies', lost())): + with self.subTest(label): + gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + gh.comments_fail = True + code, out, text = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (1, [], []), out) + self.assertIn('the tracking issue could not be read for the 24 h count (URLError', text) + self.assertIn('red, the next pass tries again', out) + + def test_quoted_text_pings_no_one_and_fakes_no_record(self): + forged = '\n\n@someone look' + gh = one([ann(SHUTDOWN + forged), ann(CANCELED)], dict(steps=died())) + code, out, _ = sweep(gh) + self.assertReran(gh, out) + [comment] = gh.comments() + self.assertTrue(comment.startswith('\n')) + self.assertNotIn('@someone', comment) + self.assertIn('@\u200dsomeone', comment) + self.assertEqual(comment.count('