diff --git a/.github/scripts/fork-follow.sh b/.github/scripts/fork-follow.sh index 3483200b265fa..f3d24482b30c2 100755 --- a/.github/scripts/fork-follow.sh +++ b/.github/scripts/fork-follow.sh @@ -195,7 +195,8 @@ schema_check "$base" "The series" # is a fork with a list of its own). After a rebase (--after-rebase), a "pending" ceiling or "lost" # count above the file's lines is only a note: upstream taking a fork line drops it in the rebase, # and the next run (from the new base) would refuse it. So the series gets a commit that lowers each -# one (#325); a count that rose is still refused, as raising one is the owner's review. +# one (#325), and excludes in "forkFiles" each upstream file a pull request's run would take for the +# fork's own (#432); a count that rose is still refused, as raising one is the owner's review. seam_refusal="Fork changes to upstream files must stay within the seam lines" # What a lowering commit changed, for the run's summary: written only once the plan has finished, # as a later check can still stop it and then nothing is pushed. @@ -218,7 +219,7 @@ seam_check() { # seam_check BASE WHAT [--after-rebase] as_fork commit -q -m "fork seams: lower counts after the rebase onto $target_tag" -m "$lowered" printf '%s\n' "$2 lowered the seam list (.github/fork-seams.json) in a commit of its own:" "$lowered" >&2 lowered_summary=("### Seam list lowered: $2" "" "$lowered" "" \ - "Committed on top of the rebased series (the candidate); the fork lines or upstream lines these counted are gone." "") + "Committed on top of the rebased series (the candidate); the fork lines or upstream lines these counted are gone, and each excluded file is upstream's." "") # The lowered list is judged again: it must pass and lower nothing more. status=0 seam_run "$1" ${3:+"$3"} || status=$? diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index b46d1517c6f3d..e75be33d8605b 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -693,6 +693,24 @@ plan check "plan: re-run of a rebase that lowered the list -> the same files, advance to the earlier release" \ eval 'action_is advance && [[ "$(out candidate)" == "$released" ]] && grep -q "lowered the seam list" "$root/out"' +# #432: upstream adds a file named like the fork's (upstream's own "fork" feature names files so). +# A pull request's run would take it for the fork's own, so the run from upstream refused it and +# every fork-follow stopped until a pull request excluded it. After the rebase the plan excludes +# it itself, in the lowering commit, and the next run passes. +new_upstream +new_fork v0.1.10 +up_release v0.1.11 'echo up > fork-up.txt' +plan +check "plan: an upstream file named like a fork file is excluded in the lowered list, not a stop" \ + eval 'action_is rebase && [[ ! -e "$root/refused" ]] \ + && [[ "$(lowered_list | jq -c .forkFiles)" == "[\"!fork-up.txt\"]" ]] \ + && grep -qx -- "- fork-up.txt: \"forkFiles\" \"!fork-up.txt\" added (upstream has it; a pull request'"'"'s run took it for a fork file)" <<< "$(git -C "$root/work" log -1 --format=%b "$(out candidate)")"' +work git reset -q --hard "$(out candidate)" +work git push -q --force origin HEAD:main +plan +check "plan: the next run from the new base passes the list with the exclusion" \ + eval 'action_is release && [[ ! -e "$root/report.md" ]] && hasnt "lowered the seam list" "$root/out"' + # The lowered list is checked again and must lower nothing more. A seam check that always lowers # (in the fork's series, which the plan runs) stops the plan: the report says what was lowered, and # the run's summary claims no commit, as nothing goes on from here. diff --git a/.github/scripts/fork-seam-check.cjs b/.github/scripts/fork-seam-check.cjs index 5419641efc8fb..9140b45b14872 100644 --- a/.github/scripts/fork-seam-check.cjs +++ b/.github/scripts/fork-seam-check.cjs @@ -555,16 +555,20 @@ function findListedGone(config, { atBase, atHead }) { } /** - * Upstream's files the head still has that a pull request's run would take for the fork's own (a - * hole in that run). `upstreamFiles`: every path upstream has; `atHead`: a path's text in head. + * Upstream files in head that a pull request's run would take for the fork's own. Refused, except + * right after a rebase that writes a lowered list: there each goes to `lowered.hidden` and lowerList + * excludes it in "forkFiles", which only narrows what a pull request's run takes for the fork's own. */ -function findHiddenUpstreamFiles(upstreamFiles, isForkFile, atHead) { - return upstreamFiles - .filter((file) => isForkFile(file) && atHead(file) !== undefined) - .map( - (file) => - `${file}: upstream has it, but a pull request's run takes it for a fork file (its name or "forkFiles"); exclude it with "!${file}".`, - ); +function findHiddenUpstreamFiles(upstreamFiles, isForkFile, atHead, lowered) { + const hidden = upstreamFiles.filter((file) => isForkFile(file) && atHead(file) !== undefined); + if (lowered !== undefined) { + for (const file of hidden) lowered.hidden.add(file); + return []; + } + return hidden.map( + (file) => + `${file}: upstream has it, but a pull request's run takes it for a fork file (its name or "forkFiles"); exclude it with "!${file}".`, + ); } /** @@ -877,10 +881,10 @@ const MOVED_WHY = 'Was "pending": fork-follow moved it here once it had no fork line left, as its "lost" entry still counts upstream lines it loses. No seam line: a fork line goes in a file of the fork\'s own.'; /** - * `text` with the counts in `lowered` ({ pending, lost }: Maps of file to count) lowered, as { text, changes }, - * or undefined when nothing changes. Only lowers; a count at 0 removes the entry. A "pending" file with no fork - * line left that "lost" still names moves to "files" with no seam line (no fork line may come back, no line may - * be lost). Two-space JSON, parsed again before it is returned, so a lowered list is never one the check refuses. + * `text` with `lowered` ({ pending, lost } Maps of counts, `hidden` Set of upstream files) applied, as + * { text, changes }, or undefined when nothing changes. Only lowers; a count at 0 removes the entry; a + * "pending" file with no fork line left moves to "files"; each hidden file stops being a fork file + * (its own "forkFiles" entry goes, or a `!` entry excludes it). Parsed again before it is returned. */ function lowerList(text, lowered) { const raw = JSON.parse(text); @@ -927,6 +931,21 @@ function lowerList(text, lowered) { }), ); } + for (const file of lowered.hidden ?? []) { + const patterns = raw.forkFiles ?? []; + const kept = patterns.filter((pattern) => pattern !== file); + const named = forkFileTest(kept)(file); + if (named) kept.push(`!${file}`); + raw.forkFiles = kept; + const what = !named + ? "entry removed" + : kept.length === patterns.length + ? `entry replaced by "!${file}"` + : `"!${file}" added`; + changes.push( + `${file}: "forkFiles" ${what} (upstream has it; a pull request's run took it for a fork file)`, + ); + } // Appended through defineProperty: a file named `__proto__` stays an own key. for (const file of moved) { Object.defineProperty(raw.files, file, { @@ -1038,7 +1057,9 @@ function main(argv) { } } const lowered = - parsed.loweredList === undefined ? undefined : { pending: new Map(), lost: new Map() }; + parsed.loweredList === undefined + ? undefined + : { pending: new Map(), lost: new Map(), hidden: new Set() }; const unlisted = findUnlistedChanges(config, { changed, atBase, @@ -1087,6 +1108,7 @@ function main(argv) { git(["ls-tree", "-r", "-z", "--name-only", mergeBase]).split("\0").filter(Boolean), isForkFile, atHead, + lowered, ) : []), ]; @@ -1127,7 +1149,18 @@ function main(argv) { if (listProblem !== undefined) return 1; if (lowered !== undefined) { // After a rebase only: fork-follow commits the lowered list into the rebased series. - const lower = lowerList(textAt(configRef, CONFIG), lowered); + const listText = textAt(configRef, CONFIG); + const lower = lowerList(listText, lowered); + const isStillFork = forkFileTest(parseConfig(lower?.text ?? listText).forkFiles); + const still = [...lowered.hidden].filter((file) => isStillFork(file)); + if (still.length > 0) { + for (const file of still) { + console.error( + printable(`${file}: the lowered list still takes this upstream file for a fork file.`), + ); + } + return 1; + } if (lower !== undefined) { fs.writeFileSync(parsed.loweredList, lower.text); for (const change of lower.changes) console.log(printable(`Lowered: ${change}`)); diff --git a/.github/scripts/fork-seam-check.test.cjs b/.github/scripts/fork-seam-check.test.cjs index 3817c70267e9a..cdedc49c08aed 100644 --- a/.github/scripts/fork-seam-check.test.cjs +++ b/.github/scripts/fork-seam-check.test.cjs @@ -913,6 +913,90 @@ test("from upstream, an upstream file named like a fork file fails on the comman }); }); +// A file upstream names like a fork file stopped every fork-follow after the rebase until a pull +// request excluded it. Right after a rebase that writes a lowered list, the list excludes it. +test("after a rebase, the lowered list excludes an upstream file a pull request's run takes for a fork file", () => { + withRepo(({ dir, commit, fromUpstream }) => { + const upstream = commit( + { "src/exact.ts": UP, "src/forkJoin.ts": UP, "src/usage/Up.ts": UP, ...BASE }, + "upstream", + ); + // Old state: a list with no "forkFiles" key, and one whose entries take upstream's files. + const forkFiles = ["src/usage/", "src/exact.ts", "src/fork-own.ts"]; + const series = commit( + { ...seams({ files: SEAM, forkFiles }), "src/fork-own.ts": UP }, + "fork series", + ); + const out = path.join(dir, "..", `${path.basename(dir)}-lowered.json`); + try { + // Before the rebase, or after it with no list to write, it is still refused: nothing would + // commit the exclusion, and main's pull request runs would keep the hole. + for (const args of [[], ["--after-rebase"]]) { + const refused = fromUpstream(...args, upstream, series); + assert.equal(refused.status, 1, args.join(" ")); + assert.match(refused.stderr, /- src\/forkJoin\.ts: upstream has it/); + } + const result = fromUpstream("--after-rebase", "--lowered-list", out, upstream, series); + assert.equal(result.status, 0, result.stderr); + const why = "(upstream has it; a pull request's run took it for a fork file)"; + assert.deepEqual(result.stdout.match(/^Lowered: .*$/gm), [ + `Lowered: src/exact.ts: "forkFiles" entry removed ${why}`, + `Lowered: src/forkJoin.ts: "forkFiles" "!src/forkJoin.ts" added ${why}`, + `Lowered: src/usage/Up.ts: "forkFiles" "!src/usage/Up.ts" added ${why}`, + ]); + const lowered = parseConfig(fs.readFileSync(out, "utf8")); + assert.deepEqual(lowered.forkFiles, [ + "src/usage/", + "src/fork-own.ts", + "!src/forkJoin.ts", + "!src/usage/Up.ts", + ]); + // Main's pull request runs judge upstream's three as upstream's; the fork's own stay its own. + const isForkFile = forkFileTest(lowered.forkFiles); + for (const file of ["src/exact.ts", "src/forkJoin.ts", "src/usage/Up.ts"]) { + assert.equal(isForkFile(file), false, file); + } + assert.equal(isForkFile("src/fork-own.ts"), true); + assert.equal(isForkFile("src/usage/Own.ts"), true); + // Runs again with the lowered list committed: nothing more to lower or refuse. + fs.rmSync(out); + const committed = commit({ [CONFIG]: JSON.stringify(lowered) }, "lowered"); + for (const args of [["--after-rebase", "--lowered-list", out], ["--after-rebase"], []]) { + const again = fromUpstream(...args, upstream, committed); + assert.equal(again.status, 0, `${args.join(" ")}: ${again.stderr}`); + assert.doesNotMatch(again.stdout, /Lowered/); + } + assert.equal(fs.existsSync(out), false); + // A fork change in such a file is refused, never excluded away, and nothing is written. + const changed = commit( + { [CONFIG]: JSON.stringify({ files: SEAM, forkFiles }), "src/forkJoin.ts": `${UP}x();\n` }, + "fork line", + ); + const stopped = fromUpstream("--after-rebase", "--lowered-list", out, upstream, changed); + assert.equal(stopped.status, 1); + assert.match(stopped.stderr, /- src\/forkJoin\.ts: upstream file with 1 fork line/); + assert.equal(fs.existsSync(out), false); + } finally { + fs.rmSync(out, { force: true }); + } + }); +}); + +test("lowerList excludes a hidden upstream file in a list with no forkFiles, or replaces its own entry", () => { + const none = `${JSON.stringify({ files: {} }, null, 2)}\n`; + const hidden = (...files) => ({ pending: new Map(), lost: new Map(), hidden: new Set(files) }); + assert.deepEqual(JSON.parse(lowerList(none, hidden("src/forkJoin.ts")).text), { + files: {}, + forkFiles: ["!src/forkJoin.ts"], + }); + // Named like the fork's and listed: the entry alone would not exclude it, so a `!` replaces it. + const listed = `${JSON.stringify({ files: {}, forkFiles: ["fork/up.ts"] }, null, 2)}\n`; + const result = lowerList(listed, hidden("fork/up.ts")); + assert.deepEqual(JSON.parse(result.text).forkFiles, ["!fork/up.ts"]); + assert.match(result.changes[0], /^fork\/up\.ts: "forkFiles" entry replaced by "!fork\/up\.ts"/); + assert.equal(lowerList(none, hidden()), undefined); +}); + // Both cases passed in both modes before. test("a rename or delete cannot hide an upstream file, from upstream or on a pull request", () => { withRepo(({ dir, git, commit, run, fromUpstream }) => { diff --git a/docs/fork/updates.md b/docs/fork/updates.md index 5c3bc6a1511b1..91a78dde571a6 100644 --- a/docs/fork/updates.md +++ b/docs/fork/updates.md @@ -122,9 +122,12 @@ watches; it changes nothing on the machine, and `0` turns it off. (the runbook below), regenerate the tree and commit it into the rebased series before pushing; a pull request that changes only the tree is judged line by line and refused. A pull request's run has no upstream commit, so it takes files named as the fork's (`fork`, `fork-*`, `forkName*`, `FORK_*`) and `forkFiles` - for the fork's own; from upstream, a pattern that matches an upstream file fails. Then remove the - entry (or exclude a fork-named path with `!path`): from then on a pull request's run judges the - file as upstream's, also before the rebase that brings upstream's copy. For the same + for the fork's own; from upstream, a pattern that matches an upstream file fails. Right after a + rebase it is not refused: fork-follow excludes the file itself in its lowering commit (the + file's own `forkFiles` entry goes; a fork-named path or one another pattern takes gets `!path`), + which only narrows what a pull request's run takes for the fork's own. Before a rebase (a list + on main that lets one through) it still fails: remove the entry or add the `!path` with a pull + request. Either way a pull request's run then judges the file as upstream's. For the same reason a pull request's run counts upstream's copies of a seam line against its allowance: a seam line upstream also has (`}`, `/**`) cannot be added by a pull request, only checked by fork-follow, so keep seam lines specific. A listed copy main's tip already has is spent too, not