From ebd276066e918f70fe39f0a1fd4c0d1883d7c244 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:33:49 +0100 Subject: [PATCH] fix(fork-ci): pr-queue config read fetches nothing; drop job GIT_* vars; head-check follow-ups (#435) - pr-queue: the config read sets GIT_NO_LAZY_FETCH=1 and GIT_ALLOW_PROTOCOL=none and has a 60 s timeout per git call; a blob a partial clone lacks is refused (exit 2) with a message naming the sparse-checkout pattern, never fetched with the stored token. - pr-queue: one list, GIT_DROP (+ GIT_CONFIG_KEY_/VALUE_ prefixes), applied by git_env() to every git call of the queue, the head check and the gate's init. - head-check: base_config reads the blob as bytes, so a non-UTF-8 base config is the "not UTF-8 text" refusal (exit 2), not a UnicodeDecodeError (exit 1). - head-check: case twins of a folder level match any entry type (a symlink or submodule `.GitHub` next to `.github/`), and names fold NFC/NFD as one. - Tests for the guards whose mutations survived: exact-name check, GIT_SAFE on the read (replace ref), no token in the read's env, ls-tree failure in case_matches. Co-Authored-By: Claude Opus 5.5 --- fork-ci/RELEASE-NOTES.md | 20 +++ fork-ci/actions/head-check/head_check.py | 36 +++--- fork-ci/actions/head-check/head_check.test.py | 86 +++++++++++-- fork-ci/actions/head-check/head_gate.py | 2 +- fork-ci/actions/pr-queue/README.md | 7 +- fork-ci/actions/pr-queue/pr_queue.py | 43 +++++-- fork-ci/actions/pr-queue/pr_queue.test.py | 115 ++++++++++++++++++ 7 files changed, 277 insertions(+), 32 deletions(-) diff --git a/fork-ci/RELEASE-NOTES.md b/fork-ci/RELEASE-NOTES.md index 78c4aa1f97294..cf5325437e4fe 100644 --- a/fork-ci/RELEASE-NOTES.md +++ b/fork-ci/RELEASE-NOTES.md @@ -13,6 +13,26 @@ nothing. ## After ci/v14 +### pr-queue and head-check: follow-ups of the git config read (#435) + +- The queue's config read fetches nothing and is timed. In a partial clone (actions/checkout's sparse checkout) a + config whose blob the sparse-checkout pattern did not check out was fetched from origin with the token the checkout + stored, with no timeout. It is now a refusal (exit 2) that names the cause; each git call of the read has 60 + seconds. +- Every git call of the queue and the head check drops the job's `GIT_*` variables that point git at another + repository, object store, index, config or pathspec rule (`GIT_DIR`, `GIT_WORK_TREE`, `GIT_OBJECT_DIRECTORY`, + `GIT_ALTERNATE_OBJECT_DIRECTORIES`, `GIT_INDEX_FILE`, `GIT_NAMESPACE`, `GIT_CONFIG_PARAMETERS`, `GIT_CONFIG_COUNT` + with its `GIT_CONFIG_KEY_*`/`GIT_CONFIG_VALUE_*`, and the like: `pr_queue.GIT_DROP`). +- A base config that is not UTF-8 is a refusal in the head check (exit 2, "not UTF-8 text", as the queue says), not + a crash (exit 1). +- The case-twin rule also refuses a file, symlink or submodule entry that differs only in case from one of the + config's folders (`.GitHub` next to `.github/`), and it counts Unicode NFC and NFD forms of a name as one. + +**Upgrade:** a caller whose checkout leaves the config's blob out must check the config out. The reusable workflow's +sparse pattern is the config's path, which matches itself unless the name holds `[ ] * ?`, a leading `!` or `#`, a +trailing space or a backslash: rename such a config. A base that already holds a twin of a config folder is not a +lock: pull requests that leave or delete it pass. + ### pr-queue: reads its config with git, never from the checkout's file (#424) The queue now reads its config as the commit checked out at `config` holds it (HEAD): the blob, with `git ls-tree` and diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index 27209940f7aef..070bfb3da348d 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -38,7 +38,7 @@ head come from differs (Checkout). `--required-lenses` takes one lens name per line (a name may hold a comma: `data-integrity (DI-1, DI-2)`). """ -import argparse, json, os, re, subprocess, sys, tempfile, time +import argparse, json, os, re, subprocess, sys, tempfile, time, unicodedata sys.dont_write_bytecode = True # no __pycache__ next to the scripts sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pr-queue')) @@ -81,7 +81,7 @@ def git(self, *a, errors='strict', timeout=None): with tempfile.TemporaryDirectory(prefix='head-check-') as empty: return subprocess.run(['git', *GIT_SAFE, *self.auth, '--git-dir', self.git_dir, *a], cwd=empty, text=True, errors=errors, capture_output=True, timeout=timeout, - env={**os.environ, 'GIT_TERMINAL_PROMPT': '0', **pr_queue.GIT_ENV}) + env=pr_queue.git_env(os.environ)) def problem(self): """Why the repository cannot be read as git made it, or None: not a git folder, shallow (the base's history is @@ -422,12 +422,12 @@ def base_config(checkout, path, problems): return None # Bare, the base is the tip the gate read from GitHub's API, fetched by sha: what the pull request's branch says # about its own config is never read. - r = checkout.git('show', f'{parents[0]}:{repo_path(path)}') + r = checkout.git('show', f'{parents[0]}:{repo_path(path)}', errors='surrogateescape') if r.returncode != 0: problems.append(f'the base commit {parents[0][:12]} has no config {path}') return None - try: - return config.parse(r.stdout, f'{path} at the base {parents[0][:12]}') + try: # as bytes: config.parse refuses one that is not UTF-8, as the queue does + return config.parse(r.stdout.encode('utf-8', 'surrogateescape'), f'{path} at the base {parents[0][:12]}') except config.ConfigError as e: problems.append(str(e)) return None @@ -470,12 +470,17 @@ def tree_entry(checkout, tree, rel): return tuple(fields) +def folded(name): + """`name` as a case-insensitive file system compares it: case folded, NFC and NFD as one (APFS, HFS+).""" + return unicodedata.normalize('NFD', unicodedata.normalize('NFD', name).casefold()) + + def case_matches(checkout, tree, rel): - """The paths in `tree` that name `rel` ignoring case, `rel` itself too: at each level the entries whose name - casefolds to the same as `rel`'s, the last of any type (a folder there collides too). Raises ValueError when git - fails (the caller refuses).""" - parts = rel.casefold().split('/') - level = [('', tree)] + """The paths in `tree` that name `rel` or one of its folders ignoring case (folded), `rel` itself too: at each level + the entries of any type whose name folds to the same as `rel`'s there (a file, a symlink or a submodule next to the + folder collides with it too); only folders are read on. Raises ValueError when git fails (the caller refuses).""" + parts = [folded(p) for p in rel.split('/')] + level, matches = [('', tree)], set() for i, want in enumerate(parts): found = [] for prefix, oid in level: @@ -486,10 +491,13 @@ def case_matches(checkout, tree, rel): for record in filter(None, (r.stdout or '').split('\0')): meta, _, name = record.partition('\t') fields = meta.split(' ') - if name.casefold() == want and (i == len(parts) - 1 or fields[1:2] == ['tree']): - found.append((prefix + name, fields[-1])) - level = [(p + '/', oid) for p, oid in found] if i < len(parts) - 1 else found - return {p for p, _ in level} + if folded(name) == want: + if i < len(parts) - 1 and fields[1:2] == ['tree']: + found.append((prefix + name + '/', fields[-1])) + else: + matches.add(prefix + name) + level = found + return matches def _merged_config_problem(checkout, path, base, head): diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index 4768f60c76f19..8d01a76b35282 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -972,24 +972,34 @@ def test_a_base_config_over_the_caps_still_judges_heads_that_leave_it_alone(self self.assertEqual(code, H.REFUSED, out) # a change to it is read under the caps self.assertIn('bytes the check reads', out) - def entry(self, commit, mode, oid, parent): - """`commit`'s tree with the config's entry made (mode, oid), committed on `parent`: a symlink (120000) or a - submodule (160000), which World.commit (it writes files) cannot make.""" + def entry(self, commit, mode, oid, parent, path=CONFIG_PATH): + """`commit`'s tree with the entry `path` (the config's) made (mode, oid), committed on `parent`: a symlink + (120000) or a submodule (160000), which World.commit (it writes files) cannot make.""" env = {**os.environ, **GIT_ENV, 'GIT_INDEX_FILE': os.path.join(self.w.dir, 'entry.index')} def git(*a): return subprocess.run(['git', *a], cwd=self.w.work, env=env, text=True, capture_output=True, check=True).stdout.strip() git('read-tree', commit) - git('update-index', '--add', '--cacheinfo', f'{mode},{oid},{CONFIG_PATH}') + git('update-index', '--add', '--cacheinfo', f'{mode},{oid},{path}') sha = git('commit-tree', git('write-tree'), '-p', parent, '-m', 'entry') self.w.n += 1 git('push', '-q', 'origin', f'{sha}:refs/keep/{self.w.n}') return sha def blob(self, text): - return subprocess.run(['git', 'hash-object', '-w', '--stdin'], cwd=self.w.work, input=text, text=True, - capture_output=True, check=True).stdout.strip() + data = text.encode() if isinstance(text, str) else text + return subprocess.run(['git', 'hash-object', '-w', '--stdin'], cwd=self.w.work, input=data, + capture_output=True, check=True).stdout.decode().strip() + + def test_a_base_config_that_is_not_utf8_is_a_refusal_never_a_crash(self): # #435 item 3: as the queue says + m = self.base(mode=None) + oid = self.blob(json.dumps(CFG_RAW).encode('utf-16')) + base = self.entry(m, '100644', oid, m) + head = self.entry(self.head(base, mode=None, **{'docs/a.md': 'a\n'}), '100644', oid, base) + code, out = self.check(base, head, EVERY_LENS) + self.assertEqual(code, H.REFUSED, out) + self.assertIn(f'{CONFIG_PATH} at the base {base[:12]}: not UTF-8 text', out) def test_a_config_that_is_no_regular_file_is_refused(self): # #413 item 1: the queue open()s it and would stop base = self.base(mode=None) @@ -1047,7 +1057,7 @@ def test_a_name_that_differs_from_the_config_only_in_case_is_refused(self): # self.assertEqual(code, H.REFUSED, out) self.assertIn(f'merged, this pull request adds {twin.split("/x.json")[0]}, which differs from ' f'{CONFIG_PATH} only in case', out) - for name in ('.github/pr-queue.json.bak', '.github/other.json', 'pr-queue.json', '.GITHUB'): # others pass + for name in ('.github/pr-queue.json.bak', '.github/other.json', 'pr-queue.json', '.GITHUBX'): # others pass with self.subTest(name): self.assertEqual(self.check(base, self.head(base, mode=None, **{name: 'x\n'}), EVERY_LENS)[0], 0) old = self.base(mode=None, **{'.github/PR-QUEUE.json': 'x\n'}) # old state: a base that holds one already @@ -1062,6 +1072,43 @@ def test_a_name_that_differs_from_the_config_only_in_case_is_refused(self): # self.assertEqual(code, H.REFUSED, out) # a second one is still new self.assertIn('adds .github/pr-QUEUE.json, which differs', out) + def test_git_variables_of_the_job_change_nothing_the_check_reads(self): # #435 item 6: pr_queue.GIT_DROP + base = self.base(mode=None) + head = self.head(base, mode=None, **{'docs/a.md': 'a\n'}) + path = self.w.checkout(base, head) + other = os.path.join(self.w.dir, 'other') + os.makedirs(os.path.join(other, 'objects')) + hostile = {'GIT_OBJECT_DIRECTORY': os.path.join(other, 'objects'), 'GIT_INDEX_FILE': os.path.join(other, 'i'), + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.repositoryFormatVersion', + 'GIT_CONFIG_VALUE_0': '99', 'GIT_NAMESPACE': 'x', 'GIT_ICASE_PATHSPECS': '1'} + with unittest.mock.patch.dict(os.environ, hostile): + code, out = run_cli(path, fake_gh((0, json.dumps(pr([self.p.lens(n, head) for n in EVERY_LENS], + head=head))))) + made, why = head_gate.bare_repository(self.w.dir, 'hostile') + self.assertEqual(code, 0, out) + self.assertIsNone(why) + self.assertTrue(os.path.isdir(os.path.join(made, 'objects', 'info')), 'git init wrote its objects elsewhere') + self.assertEqual(os.listdir(os.path.join(other, 'objects')), []) + + def test_a_twin_of_a_parent_folder_is_refused_whatever_its_type(self): # #435 item 4 + # A case-insensitive checkout holds one of `.GitHub` and the folder `.github/`: a link there could lead the + # config's path anywhere. + base = self.base(mode=None) + docs = self.head(base, mode=None, **{'docs/a.md': 'a\n'}) + for kind, mode, oid in (('a file', '100644', self.blob('x\n')), ('a symlink', '120000', self.blob('/tmp')), + ('a submodule', '160000', base)): + for twin in ('.GitHub', '.GITHUB'): + with self.subTest(kind, twin=twin): + code, out = self.check(base, self.entry(docs, mode, oid, base, path=twin), EVERY_LENS) + self.assertEqual(code, H.REFUSED, out) + self.assertIn(f'merged, this pull request adds {twin}, which differs from {CONFIG_PATH} only in ' + 'case', out) + old = self.entry(docs, '120000', self.blob('/tmp'), base, path='.GitHub') # old state: the base holds one + keeps = self.entry(self.head(old, mode=None, **{'docs/b.md': 'b\n'}), '120000', self.blob('/tmp'), old, + path='.GitHub') + code, out = self.check(old, keeps, EVERY_LENS) + self.assertEqual(code, 0, out) # a head that leaves it + def test_the_caps_hold_at_their_edge(self): # #413 item 4: the documented values, exactly at and one over base = self.base('shadow') at = 'd/' * 62 + '*.py' # 128 characters @@ -1238,6 +1285,31 @@ def test_text_that_breaks_the_parser_is_a_refusal(self): with self.subTest(name): self.assertIn(why, H.config_text_problem(text, CONFIG_PATH)) + def test_a_case_twin_listing_that_fails_is_a_refusal(self): # #435 item 5: fewer twins would pass + text = json.dumps(CFG_RAW) + + def ls_tree(*a): + if '--' in a: # tree_entry: the config is a file + return types.SimpleNamespace(returncode=0, stdout=f'100644 blob {"2" * 40}\t{a[-1]}\0', stderr='') + return types.SimpleNamespace(returncode=128, stdout='', stderr='fatal: not a tree object') + answers = {('cat-file', '-s'): (0, f'{len(text)}\n'), ('cat-file', 'blob'): (0, text)} + why = H.merged_config_problem(reading({**answers, 'ls-tree': ls_tree}), CONFIG_PATH) + self.assertIn('could not be checked, so it is not passed: ValueError: git ls-tree', why or '') + + def test_case_twins_fold_nfc_and_nfd(self): # #435 item 4: macOS file systems hold both forms as one name + sub = 'e' * 40 + trees = {C: f'040000 tree {sub}\t.github\0', + sub: ''.join(f'100644 blob {B}\t{n}\0' for n in ('caf\u00e9.json', 'cafe\u0301.json', 'CAFE\u0301.json', + 'cafe.json'))} + + class Listing: + def git(self, *a, **kw): + return types.SimpleNamespace(returncode=0, stdout=trees[a[2]], stderr='') + for rel in ('.github/caf\u00e9.json', '.github/cafe\u0301.json'): + with self.subTest(rel=ascii(rel)): + self.assertEqual(H.case_matches(Listing(), C, rel), {'.github/caf\u00e9.json', '.github/cafe\u0301.json', + '.github/CAFE\u0301.json'}) + def test_a_config_that_is_not_utf8_is_a_refusal(self): # strict: a config is text, and base_config reads it so def blob(*a): b'{"x": "\xff"}'.decode() diff --git a/fork-ci/actions/head-check/head_gate.py b/fork-ci/actions/head-check/head_gate.py index e9fd56e653996..01ed46b9e9ece 100644 --- a/fork-ci/actions/head-check/head_gate.py +++ b/fork-ci/actions/head-check/head_gate.py @@ -651,7 +651,7 @@ def bare_repository(work, name): except OSError as e: return None, f'could not make a folder in {work}: {e}' r = subprocess.run(['git', *pr_queue.GIT_SAFE, 'init', '-q', '--bare', '--template=', path], text=True, - capture_output=True, env={**os.environ, **pr_queue.GIT_ENV}) + capture_output=True, env=pr_queue.git_env(os.environ)) if r.returncode != 0: return None, f'git init failed: {r.stderr.strip()[:300]}' with open(os.path.join(path, 'config'), 'w', encoding='utf-8') as f: diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index f85e6c4319fcb..54c226e5d69c1 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -17,6 +17,10 @@ The queue reads the config with git, as the commit checked out at its path without touching the config itself. So the path must be inside a git checkout, and the config must be committed there as a regular file: an edit that is not committed is not read, and a symlink, a submodule entry or a folder is refused. +The read fetches nothing: in a partial clone (a sparse checkout) the config's +blob must be checked out, so the sparse-checkout pattern has to match its name. +A name with `[ ] * ?`, a leading `!` or `#`, a trailing space or a backslash +does not match itself as a pattern, and the queue then refuses to start. ```json { @@ -168,7 +172,8 @@ head, the one that fixes it too, and the queue stops (an owner then has to push the fix by hand). The gate therefore refuses a pull request whose merged config (GitHub's merge commit, or `git merge-tree` of base and head in the gate's bare mode) does not parse, is deleted, is no regular file (a folder, a symlink or a -submodule entry), gains a twin whose name differs from it only in case, or is +submodule entry), gains a twin whose name differs from it or one of its +folders only in case (Unicode NFC and NFD forms count as one name), or is in `enforce` with a rule order `enforce` refuses (the order checks above), and caps what it reads of a config a pull request changes (32 KiB, 128 characters per glob, 2,048 in all). That one check reads the diff --git a/fork-ci/actions/pr-queue/pr_queue.py b/fork-ci/actions/pr-queue/pr_queue.py index 0e45736174b73..99d1afa94d2a6 100755 --- a/fork-ci/actions/pr-queue/pr_queue.py +++ b/fork-ci/actions/pr-queue/pr_queue.py @@ -78,6 +78,16 @@ # ~/.gitconfig or ~/.config/git/attributes. The same # two as ../git-clean/git_clean.py's (git_clean.test.py pins GIT_SAFE and GIT_ENV to its). GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} +# Never passed on to git (git_env): variables of the job that point git at another repository, object store, index, +# config or pathspec rule than the call names. The job's environment is the owner's, but each git call says alone what +# it reads. +GIT_DROP = ('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_INDEX_FILE', 'GIT_NAMESPACE', 'GIT_CONFIG', 'GIT_CONFIG_PARAMETERS', 'GIT_CONFIG_COUNT', + 'GIT_REPLACE_REF_BASE', 'GIT_GRAFT_FILE', 'GIT_SHALLOW_FILE', 'GIT_QUARANTINE_PATH', 'GIT_PREFIX', + 'GIT_IMPLICIT_WORK_TREE', 'GIT_INTERNAL_SUPER_PREFIX', 'GIT_LITERAL_PATHSPECS', 'GIT_GLOB_PATHSPECS', + 'GIT_NOGLOB_PATHSPECS', 'GIT_ICASE_PATHSPECS') +GIT_DROP_PREFIXES = ('GIT_CONFIG_KEY_', 'GIT_CONFIG_VALUE_') +GIT_READ_SECONDS = 60 # each git call of the config read # The cache's whole config, as the queue writes it: a cache with any other config (a planted `url.*.insteadOf`, helper # or hook path) is rebuilt, never run. CACHE_CONFIG = '[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = true\n' @@ -297,6 +307,12 @@ def no_token_env(): return {k: v for k, v in os.environ.items() if k not in TOKEN_VARS} +def git_env(env): + """`env` for a git call: without GIT_DROP, with GIT_ENV and no prompt. Tokens are the caller's to drop.""" + return {**{k: v for k, v in env.items() if k not in GIT_DROP and not k.startswith(GIT_DROP_PREFIXES)}, + 'GIT_TERMINAL_PROMPT': '0', **GIT_ENV} + + def git_problem(run=subprocess.run): """Why this git cannot run the queue (merge-tree --write-tree needs GIT_MIN), or None.""" try: @@ -310,21 +326,27 @@ def git_problem(run=subprocess.run): return f'git {".".join(map(str, GIT_MIN))} or newer is needed (merge-tree --write-tree); found: {out[:80] or "none"}' -def committed_config(path, run=subprocess.run): +def committed_config(path, run=None): """The config at `path` as the commit checked out there (HEAD) holds it: its blob, read with git, never the file. A checkout writes that file through filters and `.gitattributes` (a `working-tree-encoding` makes it UTF-16, an `ident` rewrites it), which a pull request can change without changing the config's own entry, so the head check would pass it. A path outside a git checkout, a commit without the config, an entry that is no regular file - (a symlink, a submodule, a folder) or a blob that is not UTF-8 JSON is a ConfigError, never a crash.""" + (a symlink, a submodule, a folder), a blob the checkout does not hold or that is not UTF-8 JSON is a ConfigError, + never a crash. Nothing is fetched: a partial clone's missing blob would come from origin with the token the + checkout stored.""" full = os.path.abspath(path) where = os.path.dirname(full) while not os.path.isdir(where): # the folder is missing when the commit has no config: git still says so where = os.path.dirname(where) def git(cwd, *a): - try: - r = run(['git', *GIT_SAFE, '-C', cwd, *a], capture_output=True, - env={**no_token_env(), 'GIT_TERMINAL_PROMPT': '0', **GIT_ENV}) + try: # no lazy fetch (git 2.44+), and no transport at all for an older git that ignores the variable + r = (run or subprocess.run)(['git', *GIT_SAFE, '-C', cwd, *a], capture_output=True, + timeout=GIT_READ_SECONDS, env={**git_env(no_token_env()), + 'GIT_NO_LAZY_FETCH': '1', + 'GIT_ALLOW_PROTOCOL': 'none'}) + except subprocess.TimeoutExpired: + raise config.ConfigError(f'{path}: git {a[0]} took over {GIT_READ_SECONDS} seconds') from None except OSError as e: raise config.ConfigError(f'{path}: cannot run git to read it: {e}') from None return r.returncode, r.stdout, os.fsdecode(r.stderr or b'').strip()[:200] @@ -359,7 +381,10 @@ def git(cwd, *a): '100755): make it a regular file') code, out, err = git(top, 'cat-file', 'blob', oid) if code != 0: - raise config.ConfigError(f'{path}: cannot read {rel} at {sha[:12]}: {err}') + raise config.ConfigError(f'{path}: cannot read {rel} at {sha[:12]}: its blob {oid[:12]} is not in this checkout ' + '(the queue fetches nothing). In a partial clone, check the config out: the ' + 'sparse-checkout pattern must match its name, which glob characters ([ ] * ?), a ' + f'leading ! or #, a trailing space or a backslash keep it from doing ({err})') return config.parse(out, f'{path} at {sha[:12]}') @@ -442,9 +467,9 @@ def git(self, *a): """git in the cache. It only fetches: with the run's read token, and with no token at all when there is none (never gh's own login of the machine).""" token = self.read_token - env = self.gh_env(token) if token else {k: v for k, v in os.environ.items() if k not in TOKEN_VARS} + env = self.gh_env(token) if token else no_token_env() return subprocess.run(['git', *GIT_SAFE, *(GIT_AUTH if token else GIT_AUTH[:2]), '--git-dir', self.cache, *a], - text=True, capture_output=True, env={**env, 'GIT_TERMINAL_PROMPT': '0', **GIT_ENV}) + text=True, capture_output=True, env=git_env(env)) # --- step 1 and 4: the queue's own git repository ------------------------------------------------------------- def cache_problem(self): @@ -500,7 +525,7 @@ def own_cache(self): print(f'could not make {new}: {e}') return False r = subprocess.run(['git', *GIT_SAFE, 'init', '-q', '--bare', '--template=', new], text=True, - capture_output=True, env={**no_token_env(), **GIT_ENV}) + capture_output=True, env=git_env(no_token_env())) if r.returncode != 0: print(f'could not make {self.cache}: {r.stderr.strip()[:300]}') return False diff --git a/fork-ci/actions/pr-queue/pr_queue.test.py b/fork-ci/actions/pr-queue/pr_queue.test.py index a7bd1cbababec..daf5e05269407 100644 --- a/fork-ci/actions/pr-queue/pr_queue.test.py +++ b/fork-ci/actions/pr-queue/pr_queue.test.py @@ -3237,6 +3237,121 @@ def test_a_config_outside_a_git_checkout_or_before_its_first_commit_is_refused(s self.config = os.path.join(self.repo, '.github', 'pr-queue.json') self.refused('has no commit checked out') + def test_a_config_blob_missing_from_a_partial_clone_is_refused_never_fetched(self): # #435 item 1 + # actions/checkout's sparse checkout is a --filter=blob:none clone: a pattern that does not match the config's + # own name ([1] is a glob) leaves its blob on the remote, and cat-file would fetch it with the stored token. + rel = '.github/q[1].json' + self.commit({rel: json.dumps(CFG_RAW)}) + remote, pc = os.path.join(self.d, 'remote.git'), os.path.join(self.d, 'pc') + self.git('clone', '-q', '--bare', self.repo, remote) + subprocess.run(['git', '-C', remote, 'config', 'uploadpack.allowFilter', 'true'], check=True) + subprocess.run(['git', 'clone', '-q', '--filter=blob:none', '--no-checkout', 'file://' + remote, pc], + check=True, capture_output=True) + subprocess.run(['git', '-C', pc, 'sparse-checkout', 'set', '--no-cone', rel], check=True, capture_output=True) + subprocess.run(['git', '-C', pc, 'checkout', '-q', 'main'], check=True, capture_output=True) + oid = subprocess.run(['git', '-C', pc, 'rev-parse', f'HEAD:{rel}'], check=True, capture_output=True, + text=True).stdout.strip() + + def local(sha): # without fetching it + return subprocess.run(['git', '-C', pc, 'cat-file', '-e', sha], capture_output=True, + env={**os.environ, 'GIT_NO_LAZY_FETCH': '1', 'GIT_ALLOW_PROTOCOL': 'none'}).returncode == 0 + self.assertFalse(local(oid), 'the clone holds the blob: the test reads nothing missing') + self.config = os.path.join(pc, rel) + for _ in range(2): # runs again: the same refusal, still nothing fetched + self.refused(f'{rel} at ', 'not in this checkout', 'sparse-checkout pattern') + self.assertFalse(local(oid), 'the read fetched the blob') + subprocess.run(['git', '-C', pc, 'sparse-checkout', 'disable'], check=True, capture_output=True) # the fix + code, out = self.cli() + self.assertEqual((code, self.built[-1][0][1]), (0, CFG), out) + + def test_every_git_call_of_the_read_has_a_timeout(self): # #435 item 1: a stuck git never holds the job + calls = [] + + def run(cmd, **kw): + calls.append(kw.get('timeout')) + if 'cat-file' in cmd: + raise subprocess.TimeoutExpired(cmd, kw.get('timeout')) + return subprocess.run(cmd, **kw) + with self.assertRaisesRegex(config.ConfigError, r'pr-queue\.json: git cat-file took over \d+ seconds'): + Q.committed_config(self.config, run=run) + self.assertTrue(calls and all(t == Q.GIT_READ_SECONDS for t in calls), calls) + + def test_a_config_path_with_pathspec_magic_reads_only_that_name(self): # #435 item 2: the exact-name check + self.commit({'x.json': json.dumps(CFG_RAW)}) # ls-tree -- ':(top)x.json' lists x.json + self.config = os.path.join(self.repo, ':(top)x.json') + self.refused('did not list one entry') + + def test_a_leftover_replace_ref_does_not_change_the_config_read(self): # #435 item 2: GIT_SAFE on the read + oid = self.git('rev-parse', 'HEAD:.github/pr-queue.json').decode().strip() + other = subprocess.run(['git', '-C', self.repo, 'hash-object', '-w', '--stdin'], check=True, capture_output=True, + input=json.dumps({**CFG_RAW, 'replaced': 1}).encode()).stdout.decode().strip() + self.git('replace', oid, other) # a self-hosted runner's .git can outlive a job + self.assertIn(b'replaced', self.git('cat-file', 'blob', oid), 'the replace ref does nothing') + code, out = self.cli() + self.assertEqual((code, self.built[-1][0][1]), (0, CFG), out) + + def envs_of_the_read(self, extra): + envs = [] + real = subprocess.run + + def run(cmd, **kw): + envs.append(kw.get('env')) + return real(cmd, **kw) + with unittest.mock.patch.dict(os.environ, extra), unittest.mock.patch.object(subprocess, 'run', run): + code, out = self.cli() + self.assertEqual((code, self.built[-1][0][1]), (0, CFG), out) + self.assertTrue(envs and None not in envs, envs) + return envs + + def test_the_config_read_passes_no_token_to_git(self): # #435 item 2 + tokens = {k: 'secret-' + k for k in Q.TOKEN_VARS} + for env in self.envs_of_the_read(tokens): + self.assertEqual(set(env) & set(tokens), set()) + # never a fetch: git 2.44+ honours the first, an older git still refuses every transport + self.assertEqual((env.get('GIT_NO_LAZY_FETCH'), env.get('GIT_ALLOW_PROTOCOL')), ('1', 'none')) + + def test_the_config_read_ignores_git_variables_of_the_job(self): # #435 item 6 + other = os.path.join(self.d, 'other') + subprocess.run(['git', 'init', '-q', '-b', 'main', other], check=True) + os.makedirs(os.path.join(other, '.github')) + with open(os.path.join(other, '.github', 'pr-queue.json'), 'w') as f: + json.dump({**CFG_RAW, 'merge_method': 'merge'}, f) + subprocess.run(['git', '-C', other, 'add', '-A'], check=True) + subprocess.run(['git', '-C', other, 'commit', '-qm', 'other'], check=True, env={**os.environ, **GIT_ENV}) + hostile = {'GIT_DIR': os.path.join(other, '.git'), 'GIT_WORK_TREE': self.repo, + 'GIT_OBJECT_DIRECTORY': os.path.join(other, '.git', 'objects'), 'GIT_NAMESPACE': 'x', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES': os.path.join(other, '.git', 'objects'), + 'GIT_INDEX_FILE': os.path.join(self.d, 'index'), 'GIT_CONFIG_PARAMETERS': "'core.bare'='true'", + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.bare', 'GIT_CONFIG_VALUE_0': 'true', + 'GIT_ICASE_PATHSPECS': '1'} + for env in self.envs_of_the_read(hostile): + self.assertEqual(set(env) & set(hostile), set()) + + def test_the_queue_drops_one_list_of_git_variables(self): # #435 item 6: every git call of the queue + hostile = {k: 'x' for k in Q.GIT_DROP} + hostile.update(GIT_CONFIG_KEY_3='core.bare', GIT_CONFIG_VALUE_3='true', GIT_TRACE='0', PATH='/bin') + env = Q.git_env(hostile) + self.assertEqual(env, {'GIT_TRACE': '0', 'PATH': '/bin', 'GIT_TERMINAL_PROMPT': '0', **Q.GIT_ENV}) + for name in ('GIT_DIR', 'GIT_WORK_TREE', 'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_CONFIG_PARAMETERS', 'GIT_CONFIG_COUNT', 'GIT_INDEX_FILE', 'GIT_NAMESPACE', 'GIT_COMMON_DIR'): + self.assertIn(name, Q.GIT_DROP) + q = object.__new__(Q.Queue) # the cache's git and init + q.cache, q.read_token = os.path.join(self.d, 'cache', 'repo.git'), None + envs, real = [], subprocess.run + + def run(cmd, **kw): + envs.append(kw.get('env')) + return real(cmd, **kw) + with unittest.mock.patch.dict(os.environ, {'GIT_DIR': self.repo + '/.git', 'GIT_CONFIG_COUNT': '1', + 'GIT_CONFIG_KEY_0': 'core.bare', 'GIT_CONFIG_VALUE_0': 'false'}), \ + unittest.mock.patch.object(subprocess, 'run', run), \ + unittest.mock.patch('sys.stdout', new_callable=io.StringIO) as out: + self.assertTrue(q.own_cache(), out.getvalue()) + self.assertEqual(q.git('rev-parse', '--is-bare-repository').stdout.strip(), 'true') + self.assertTrue(envs) + for env in envs: + self.assertFalse({'GIT_DIR', 'GIT_CONFIG_COUNT', 'GIT_CONFIG_KEY_0'} & set(env), env) + def test_an_old_git_is_refused(self): self.refused('git 2.38 or newer', git='git 2.38 or newer is needed (merge-tree --write-tree); found: git 2.34')