From 598d8312fa2c4e7b172feaec6afb123cfa3a566e Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Tue, 18 Aug 2026 22:54:35 -0600 Subject: [PATCH 1/5] populate --sync-bodies: push file bodies to existing issues (#7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit populate was create-only, so editing (or unwrapping) a body in the plan file could never reach an issue that already exists on GitHub. --sync-bodies is the per-issue inverse of update: it pushes existing issues' bodies AND milestone descriptions (decision 2: it was easy) file -> GitHub, creates nothing, and joins the mutually exclusive stage group. Conflict stance per decision 1 — refuse-on-divergence by default, --force for last-writer-wins — operationalized without a stored base via a three-way classification per item: in-sync identical after CRLF/trailing-whitespace normalization (GitHub stores web edits with CRLF) -> skip reflow identical after unwrap() of both sides — the content matches, only line wrapping differs, so pushing cannot lose anyone's words -> pushed without --force; this is the case the flag exists for divergent the content itself differs; with no base the engine cannot tell which side moved -> refused per item (exit 1) unless --force Refusals are per-item, so one divergent body does not block the reflow-safe ones; --dry-run prints exactly the live plan and mutates nothing; the confirmation prompt states how many bodies/descriptions may be rewritten; a note reminds that sync never creates when plan issues are missing from GitHub. --keep-line-breaks composes (file-side bodies are the effective, by-default-unwrapped ones, matching what populate would have pushed at creation). Decision 3 (how the README/CLAUDE.md direction contract should describe this deliberate exception) is deferred to new issue #9, as requested. GitHubClient gains update_issue_body / update_milestone_description; the recorded fake gh learns 'issue edit' and milestone PATCH. Suite 145 -> 153 (pure classification tests; end-to-end reflow push + in-sync second run; divergence refusal then forced win; dry-run mutates nothing; sync-never-creates). The roadmap-upkeep skill documents the workflow. Engine VERSION 0.5.0. AI-assisted: Claude Fable 5 (Anthropic) --- .claude/skills/roadmap-upkeep/SKILL.md | 12 ++ scripts/VERSION | 2 +- scripts/_gh_project_lib.py | 20 ++++ scripts/gh_project_populate.py | 158 +++++++++++++++++++++++++ scripts/tests/fake_gh/gh | 20 ++++ scripts/tests/test_integration.py | 103 ++++++++++++++++ 6 files changed, 314 insertions(+), 1 deletion(-) diff --git a/.claude/skills/roadmap-upkeep/SKILL.md b/.claude/skills/roadmap-upkeep/SKILL.md index 5491e3f..545598d 100644 --- a/.claude/skills/roadmap-upkeep/SKILL.md +++ b/.claude/skills/roadmap-upkeep/SKILL.md @@ -51,6 +51,18 @@ Issue bodies of already-created issues are edited ON GITHUB (the region mirror would overwrite in-file edits on the next update). Prose outside regions is edited in the file. +## Pushing body edits to existing issues (`--sync-bodies`) + +populate is create-only, so editing a body in the plan file never +reaches an issue that already exists. When file-side bodies should win +(e.g. after reflowing an already-populated plan), run +`python3 /scripts/gh_project_populate.py docs/GITHUB_PROJECT.md --sync-bodies` +(add `--dry-run` first). It pushes existing issues' bodies and +milestone descriptions file → GitHub, per item: identical content +skips; wrapping-only differences push safely; content divergence is +REFUSED (the engine stores no base, so it cannot tell which side moved) +unless `--force` makes the file win. It never creates anything. + ## Organically-filed issues (the `unplanned` region) Issues filed directly on GitHub without a `[MN-k]` title prefix never diff --git a/scripts/VERSION b/scripts/VERSION index 1d0ba9e..8f0916f 100644 --- a/scripts/VERSION +++ b/scripts/VERSION @@ -1 +1 @@ -0.4.0 +0.5.0 diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 00d1713..99be547 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -984,6 +984,26 @@ def create_milestone(self, ms: Milestone) -> Result[Milestone, PipelineError]: "-f", f"description={ms.description}", ).and_then(_parse_milestone_create_response).map(ms.with_gh_number) + def update_issue_body( + self, gh_number: int, body: str + ) -> Result[None, PipelineError]: + """Replace issue `gh_number`'s body (populate --sync-bodies).""" + return self._run( + "issue", "edit", str(gh_number), + "--repo", self.repo, + "--body", body, + ).map(lambda _: None) + + def update_milestone_description( + self, gh_number: int, description: str + ) -> Result[None, PipelineError]: + """Replace milestone `gh_number`'s description (populate --sync-bodies).""" + return self._run( + "api", f"repos/{self.repo}/milestones/{gh_number}", + "-X", "PATCH", + "-f", f"description={description}", + ).map(lambda _: None) + def create_issue( self, issue: Issue, milestone_title: Optional[str] ) -> Result[int, PipelineError]: diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index b861cdf..c04e541 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -41,6 +41,13 @@ --issues-only Only create issues (milestones must already exist) --skip-labels Skip label creation --start-from ID Create only issues with ID >= this (e.g. M1-3) + --sync-bodies Create nothing; push existing issues' bodies and + milestone descriptions from the file to GitHub — + the per-issue inverse of update. Content + divergence is refused per item unless --force + (last writer wins); wrapping-only differences push + safely without it + --force With --sync-bodies: push divergent content anyway --keep-line-breaks Push bodies exactly as authored; by default hard line breaks in prose are stripped so GitHub soft-wraps it @@ -150,6 +157,83 @@ def issue_blocker( return None +def _normalized(body: str) -> str: + """Comparison form: GitHub stores web edits with CRLF, and trailing + whitespace is never meaningful.""" + return body.replace("\r\n", "\n").strip() + + +def classify_sync(file_body: str, github_body: str) -> str: + """How the plan file's body relates to GitHub's, with no stored base. + + - "in-sync": identical after CRLF/trailing-whitespace normalization. + - "reflow": identical after unwrap() of both sides — the content + matches and only line wrapping differs, so pushing + cannot lose anyone's words. This is the case + --sync-bodies exists for. + - "divergent": the content itself differs. The engine stores no + base version, so it cannot tell which side moved — + refused by default; --force pushes the file's + version (last writer wins). + """ + ours, theirs = _normalized(file_body), _normalized(github_body) + if ours == theirs: + return "in-sync" + if unwrap(ours) == unwrap(theirs): + return "reflow" + return "divergent" + + +def execute_sync_bodies( + client: GitHubClient, + issue_pairs: list[tuple], + milestone_pairs: list[tuple], + force: bool, + dry_run: bool, + delay: float, +) -> int: + """Push plan-file bodies/descriptions to their existing GitHub + counterparts. Returns the number of failures + refusals. + + Each pair is (label, file_text, github_text, push_callable). Every + item is classified before anything mutates, dry-run prints exactly + the live plan, and refusals are per-item — one divergent body does + not block the reflow-safe ones. + """ + in_sync = pushed = refused = failed = 0 + for label, file_text, github_text, push in issue_pairs + milestone_pairs: + verdict = classify_sync(file_text, github_text) + if verdict == "in-sync": + in_sync += 1 + print(f" = in sync: {label}") + continue + if verdict == "divergent" and not force: + refused += 1 + print( + f" ! divergent: {label} — content differs beyond line " + f"wrapping and no base is stored to tell which side moved; " + f"re-run with --force to push the file's version" + ) + continue + tag = "forced push" if verdict == "divergent" else "reflow" + if dry_run: + pushed += 1 + print(f" ~ would push ({tag}): {label}") + continue + result = push() + if result.is_ok: + pushed += 1 + print(f" ~ pushed ({tag}): {label}") + else: + failed += 1 + print(f" ! {label}: {result.unwrap_err()}") + time.sleep(delay) + print() + print(f" sync: {pushed} pushed, {in_sync} already in sync, " + f"{refused} divergent (refused), {failed} failed") + return refused + failed + + def print_issue_plan( plan: IssuePlan, available_milestones: set[int], @@ -298,6 +382,67 @@ def execute_issues( return text, created, failed +def run_sync_bodies( + client: GitHubClient, + snapshot: RepoSnapshot, + milestone_plan: MilestonePlan, + issue_plan: IssuePlan, + args: argparse.Namespace, +) -> int: + """The --sync-bodies mode: push file bodies to EXISTING GitHub + counterparts; never creates anything (issue #7).""" + issue_pairs = [ + ( + f"issue {desired.id} (#{live.gh_number})", + desired.body, + live.body, + lambda n=live.gh_number, b=desired.body: + client.update_issue_body(n, b), + ) + for desired, live in issue_plan.existing + if live.gh_number is not None + ] + live_desc = {m.gh_number: m.description for m in snapshot.milestones} + milestone_pairs = [ + ( + f"milestone {ms.title} (#{ms.gh_number})", + ms.description, + live_desc.get(ms.gh_number, ""), + lambda n=ms.gh_number, d=ms.description: + client.update_milestone_description(n, d), + ) + for ms in milestone_plan.existing + if ms.gh_number + ] + + _stage_header( + "SYNCING BODIES (file → GitHub, existing items only)" + + (" [dry run]" if args.dry_run else "") + ) + if not issue_pairs and not milestone_pairs: + print(" nothing to sync: no plan issue or milestone exists on " + "GitHub yet") + if not args.dry_run and (issue_pairs or milestone_pairs) and not args.yes: + print(f"This may rewrite {len(issue_pairs)} issue bodies and " + f"{len(milestone_pairs)} milestone descriptions on GitHub.") + response = input("Continue? [y/N] ").strip().lower() + if response not in ("y", "yes"): + print("Aborted.") + return 0 + + problems = execute_sync_bodies( + client, issue_pairs, milestone_pairs, + force=args.force, dry_run=args.dry_run, delay=args.delay, + ) + if issue_plan.to_create: + print( + f" note: {len(issue_plan.to_create)} plan issue(s) do not " + f"exist on GitHub — --sync-bodies never creates; run populate " + f"without it first" + ) + return 1 if problems else 0 + + # ── Main ───────────────────────────────────────────────────────────────────── def build_parser() -> argparse.ArgumentParser: @@ -326,6 +471,14 @@ def build_parser() -> argparse.ArgumentParser: stage.add_argument("--milestones-only", action="store_true") stage.add_argument("--labels-only", action="store_true") stage.add_argument("--issues-only", action="store_true") + stage.add_argument("--sync-bodies", action="store_true", + help="create nothing; push existing issues' bodies " + "(and milestone descriptions) from the plan file " + "to GitHub — the per-issue inverse of update. " + "Refuses on content divergence unless --force") + parser.add_argument("--force", action="store_true", + help="with --sync-bodies: push even when the GitHub " + "side has content changes (last writer wins)") parser.add_argument("--skip-labels", action="store_true") parser.add_argument("--keep-line-breaks", action="store_true", help="push bodies exactly as authored; by default " @@ -423,6 +576,11 @@ def main() -> int: milestone_plan = plan_milestones(plan.milestones, snapshot.milestones) issue_plan = plan_issues(desired_issues, snapshot.issues) + if args.sync_bodies: + return run_sync_bodies( + client, snapshot, milestone_plan, issue_plan, args + ) + # Availability as it WILL stand when the issues stage runs, given # the selected stages: entities already on GitHub, plus those this # run will create first. Feeding these to the shared issue_blocker diff --git a/scripts/tests/fake_gh/gh b/scripts/tests/fake_gh/gh index b7a5ce5..96d12e0 100755 --- a/scripts/tests/fake_gh/gh +++ b/scripts/tests/fake_gh/gh @@ -83,6 +83,16 @@ def main() -> int: return 0 if args[0] == "api" and "milestones" in args[1]: + if "PATCH" in flag_values(args, "-X"): + number = int(args[1].rsplit("/", 1)[1]) + fields = kv_values(args, "-f") + milestones = load("milestones.json", []) + target = next(m for m in milestones if m["number"] == number) + target["description"] = fields.get("description", + target["description"]) + save("milestones.json", milestones) + print(json.dumps(target)) + return 0 if "POST" in flag_values(args, "-X"): fields = kv_values(args, "-f") milestones = load("milestones.json", []) @@ -103,6 +113,16 @@ def main() -> int: print(json.dumps(load("issues.json", []))) return 0 + if args[:2] == ["issue", "edit"]: + number = int(args[2]) + issues = load("issues.json", []) + target = next(i for i in issues if i["number"] == number) + body = flag_value(args, "--body") + if body is not None: + target["body"] = body + save("issues.json", issues) + return 0 + if args[:2] == ["issue", "create"]: call_index = bump("issue_create_calls") fail_at = STATE / "fail_create_issue_at" diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py index 1363792..4cb257a 100644 --- a/scripts/tests/test_integration.py +++ b/scripts/tests/test_integration.py @@ -405,6 +405,109 @@ def test_no_hint_after_a_partial_failure(self) -> None: self.assertNotIn(self.HINT, proc.stdout) +class SyncBodiesClassification(unittest.TestCase): + """Pure three-way classification behind --sync-bodies (issue #7).""" + + def setUp(self) -> None: + sys.path.insert(0, str(SCRIPTS_DIR)) + from gh_project_populate import classify_sync + self.classify = classify_sync + + def test_identical_is_in_sync(self) -> None: + self.assertEqual(self.classify("Body.\n", "Body.\n"), "in-sync") + + def test_crlf_and_trailing_whitespace_normalize(self) -> None: + self.assertEqual(self.classify("Body.\n", "Body.\r\n\r\n"), "in-sync") + + def test_wrapping_only_difference_is_reflow(self) -> None: + self.assertEqual( + self.classify("One long joined line here.", + "One long\njoined line\nhere."), + "reflow", + ) + + def test_content_change_is_divergent(self) -> None: + self.assertEqual( + self.classify("One long joined line here.", + "One long joined line THERE."), + "divergent", + ) + + +class SyncBodies(FakeGhHarness): + """--sync-bodies end to end: reflow pushes, divergence refuses + without --force, dry-run predicts, milestones included.""" + + WRAPPED_BODY = "Body of\nM1-1 wrapped over\nthree lines." + WRAPPED_DESC = "Build the\ncore, wrapped." + + def populate_wrapped(self) -> None: + self.plan_path.write_text( + PLAN.replace("Body of M1-1.", self.WRAPPED_BODY) + .replace("Build the core.", self.WRAPPED_DESC), + encoding="utf-8", + ) + proc = self.populate("--keep-line-breaks") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def m11(self) -> dict: + return next(i for i in self.issues_on_fake_github() + if i["title"].startswith("[M1-1]")) + + def core_milestone(self) -> dict: + milestones = json.loads((self.state / "milestones.json").read_text()) + return next(m for m in milestones if m["title"] == "1. Core") + + def test_reflow_pushes_and_second_run_is_in_sync(self) -> None: + self.populate_wrapped() + self.assertIn("\n", self.m11()["body"].split("wrapped")[0] + "wrapped") + + proc = self.populate("--sync-bodies") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("pushed (reflow)", proc.stdout) + self.assertTrue(self.m11()["body"].startswith( + "Body of M1-1 wrapped over three lines.")) + self.assertTrue(self.core_milestone()["description"].startswith( + "Build the core, wrapped.")) + + again = self.populate("--sync-bodies") + self.assertEqual(again.returncode, 0, again.stderr) + self.assertIn("0 pushed", again.stdout) + self.assertNotIn("pushed (", again.stdout) + + def test_divergence_refuses_then_force_wins(self) -> None: + self.populate_wrapped() + issues = self.issues_on_fake_github() + target = next(i for i in issues if i["title"].startswith("[M1-1]")) + target["body"] = "Rewritten independently on GitHub." + (self.state / "issues.json").write_text(json.dumps(issues)) + + refused = self.populate("--sync-bodies") + self.assertEqual(refused.returncode, 1, refused.stdout) + self.assertIn("divergent: issue M1-1", refused.stdout) + self.assertEqual(self.m11()["body"], + "Rewritten independently on GitHub.") + + forced = self.populate("--sync-bodies", "--force") + self.assertEqual(forced.returncode, 0, forced.stdout + forced.stderr) + self.assertIn("pushed (forced push)", forced.stdout) + self.assertTrue(self.m11()["body"].startswith("Body of M1-1")) + + def test_dry_run_predicts_and_mutates_nothing(self) -> None: + self.populate_wrapped() + before = json.dumps(self.issues_on_fake_github()) + proc = self.populate("--sync-bodies", "--dry-run") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("would push (reflow)", proc.stdout) + self.assertEqual(json.dumps(self.issues_on_fake_github()), before) + + def test_sync_never_creates(self) -> None: + proc = self.populate("--sync-bodies") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.issues_on_fake_github(), []) + self.assertIn("never creates", proc.stdout) + + class OrganicIssues(FakeGhHarness): """Issue #3 end to end: organically-filed issues render into the `unplanned` region, stay out of populate's way, and cannot inject From 7829904b4371eec7ded032e95cab209b032fed11 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Tue, 18 Aug 2026 23:19:02 -0600 Subject: [PATCH 2/5] classify_sync: leading whitespace is meaningful; GitHub-side hard breaks diverge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both findings from Copilot's PR #10 review were legitimate holes in the classification's safety guarantee: - _normalized used strip(), which also eats LEADING whitespace: a GitHub body whose first line is four-space-indented (a code block) compared in-sync with an unindented file body — and since in-sync short-circuits, even --force could not push. rstrip() now preserves leading whitespace; such a pair classifies divergent and --force can make the file win. - unwrap() deliberately removes markdown hard breaks (two trailing spaces, or a backslash, before a newline), so an INTENTIONAL
on the GitHub side classified as reflow and was overwritten without --force, contrary to refuse-on-divergence. GitHub-side hard-break syntax now forces divergent whenever the normalized bodies differ. Deliberately asymmetric: a hard break on the FILE side alone still reflows, because the push carries it to GitHub intact — nothing GitHub-side is lost. Five regression cases pin both (leading-indent divergence and its in-sync counterpart; two-space and backslash hard breaks; identical hard-break bodies still in-sync; file-side hard break still reflow). Suite 153 -> 156. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/gh_project_populate.py | 39 +++++++++++++++++++++++-------- scripts/tests/test_integration.py | 22 +++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index c04e541..359f53e 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -68,6 +68,7 @@ from __future__ import annotations import argparse +import re import sys import textwrap import time @@ -158,27 +159,45 @@ def issue_blocker( def _normalized(body: str) -> str: - """Comparison form: GitHub stores web edits with CRLF, and trailing - whitespace is never meaningful.""" - return body.replace("\r\n", "\n").strip() + """Comparison form: GitHub stores web edits with CRLF, and TRAILING + whitespace is never meaningful. Leading whitespace stays — a + four-space-indented first line is a code block, and eating it would + make genuinely different bodies compare equal (and in-sync + short-circuits even --force).""" + return body.replace("\r\n", "\n").rstrip() + + +# Markdown hard breaks: two+ trailing spaces, or a backslash, before a +# newline. unwrap() deliberately removes them, so they are invisible +# to the reflow equivalence — but on the GitHub side they are intended +# rendering (
) that a push would destroy. +_HARD_BREAK_RE = re.compile(r"(?:[ ]{2,}|\\)\n") def classify_sync(file_body: str, github_body: str) -> str: """How the plan file's body relates to GitHub's, with no stored base. - - "in-sync": identical after CRLF/trailing-whitespace normalization. + - "in-sync": identical after CRLF/trailing-whitespace + normalization (leading whitespace is meaningful and + preserved). - "reflow": identical after unwrap() of both sides — the content matches and only line wrapping differs, so pushing - cannot lose anyone's words. This is the case - --sync-bodies exists for. - - "divergent": the content itself differs. The engine stores no - base version, so it cannot tell which side moved — - refused by default; --force pushes the file's - version (last writer wins). + loses neither words nor GitHub-side formatting. This + is the case --sync-bodies exists for. + - "divergent": the content differs, or GitHub's body carries + explicit hard-break syntax (trailing double-space or + backslash before a newline) that reflow would + silently destroy. The engine stores no base version, + so it cannot tell which side moved — refused by + default; --force pushes the file's version (last + writer wins). Hard breaks on the FILE side alone do + not diverge: the push carries them to GitHub intact. """ ours, theirs = _normalized(file_body), _normalized(github_body) if ours == theirs: return "in-sync" + if _HARD_BREAK_RE.search(theirs): + return "divergent" if unwrap(ours) == unwrap(theirs): return "reflow" return "divergent" diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py index 4cb257a..7b2df53 100644 --- a/scripts/tests/test_integration.py +++ b/scripts/tests/test_integration.py @@ -433,6 +433,28 @@ def test_content_change_is_divergent(self) -> None: "divergent", ) + def test_leading_indentation_is_meaningful(self) -> None: + # A four-space-indented first line on GitHub is a code block; + # eating it made different bodies compare in-sync — and in-sync + # short-circuits even --force. + self.assertEqual(self.classify("code\nrest.", " code\nrest."), + "divergent") + self.assertEqual(self.classify(" code\nrest.", " code\nrest."), + "in-sync") + + def test_github_side_hard_breaks_are_divergent(self) -> None: + # `a \nb` renders a
; unwrap() would erase it, so reflow + # must not silently overwrite it (two-space and backslash forms). + self.assertEqual(self.classify("a b", "a \nb"), "divergent") + self.assertEqual(self.classify("a b", "a\\\nb"), "divergent") + # Identical bodies with hard breaks are still just in-sync. + self.assertEqual(self.classify("a \nb", "a \nb"), "in-sync") + + def test_file_side_hard_breaks_still_reflow(self) -> None: + # The push carries the file's hard break to GitHub intact — + # nothing GitHub-side is lost, so no --force is demanded. + self.assertEqual(self.classify("a \nb", "a\nb"), "reflow") + class SyncBodies(FakeGhHarness): """--sync-bodies end to end: reflow pushes, divergence refuses From 752027a1b2c0a333ed7afe41b24a0e39b389ef67 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Tue, 18 Aug 2026 23:44:27 -0600 Subject: [PATCH 3/5] sync-bodies: revalidate before each mutation; backslash file-breaks reflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both round-2 findings were legitimate: - TOCTOU: the divergence verdict came from the snapshot fetched before the interactive confirmation, while push() mutated unconditionally — an edit made while the prompt sat open (or during earlier items' delays) was overwritten undetected, outside the refuse-on-divergence guarantee. Live runs now re-fetch each target immediately before mutating (GitHubClient.get_issue_body / get_milestone_description — one targeted GET per pushed item, a different beast from the O(n^2) list-per-item that #293 banned) and classify against the fresh text; dry runs keep previewing from the snapshot without fetching. GitHub's API offers no conditional issue update, so a milliseconds-wide window remains; the docstring says so plainly. - The documented file-side hard-break reflow only held for the two-space form: the backslash form survives unwrap's join ('a\\ b') and spuriously demanded --force for a push that preserves the break verbatim. The equivalence comparison now defangs file-side hard-break markers (the GitHub-side guard is untouched). The recorded fake gh learns the two single-resource GETs. Regression tests: fresh-divergent-while-snapshot-reflow is refused with nothing mutated, dry runs never fetch, and the backslash counterpart joins the file-side reflow case. Suite 156 -> 157. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 40 ++++++++++++++++++++++++++ scripts/gh_project_populate.py | 34 ++++++++++++++++++---- scripts/tests/fake_gh/gh | 15 ++++++++++ scripts/tests/test_integration.py | 47 +++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 6 deletions(-) diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 99be547..a43f4bf 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -984,6 +984,25 @@ def create_milestone(self, ms: Milestone) -> Result[Milestone, PipelineError]: "-f", f"description={ms.description}", ).and_then(_parse_milestone_create_response).map(ms.with_gh_number) + def get_issue_body(self, gh_number: int) -> Result[str, PipelineError]: + """Fetch ONE issue's current body — the revalidation read + --sync-bodies performs immediately before each mutation, so the + divergence verdict covers edits made after the run's snapshot + (e.g. while the confirmation prompt sat open).""" + return self._run( + "api", f"repos/{self.repo}/issues/{gh_number}", "-X", "GET", + ).and_then(lambda out: _parse_object_field(out, "body", "issue")) + + def get_milestone_description( + self, gh_number: int + ) -> Result[str, PipelineError]: + """Fetch ONE milestone's current description (see get_issue_body).""" + return self._run( + "api", f"repos/{self.repo}/milestones/{gh_number}", "-X", "GET", + ).and_then( + lambda out: _parse_object_field(out, "description", "milestone") + ) + def update_issue_body( self, gh_number: int, body: str ) -> Result[None, PipelineError]: @@ -1150,6 +1169,27 @@ def _parse_json(stdout: str, kind: str) -> Result[list[dict], PipelineError]: return Result.ok(items) +def _parse_object_field( + stdout: str, field: str, kind: str +) -> Result[str, PipelineError]: + """Extract one string field from a single-object `gh api` response.""" + try: + data = json.loads(stdout) + except json.JSONDecodeError as e: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"failed to decode {kind} JSON from `gh`", + cause=e, + context={"stdout_preview": stdout[:500]}, + )) + if not isinstance(data, dict): + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"expected a JSON object for {kind}, got {type(data).__name__}", + )) + return Result.ok(data.get(field) or "") + + def _parse_milestone_create_response(stdout: str) -> Result[int, PipelineError]: try: data = json.loads(stdout) diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index 359f53e..ae3665e 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -198,7 +198,11 @@ def classify_sync(file_body: str, github_body: str) -> str: return "in-sync" if _HARD_BREAK_RE.search(theirs): return "divergent" - if unwrap(ours) == unwrap(theirs): + # File-side hard breaks are defanged for the EQUIVALENCE test only: + # the two-space form vanishes under unwrap's rstrip anyway, but the + # backslash form would survive the join ("a\\ b") and spuriously + # demand --force for a push that preserves the break verbatim. + if unwrap(_HARD_BREAK_RE.sub("\n", ours)) == unwrap(theirs): return "reflow" return "divergent" @@ -214,13 +218,29 @@ def execute_sync_bodies( """Push plan-file bodies/descriptions to their existing GitHub counterparts. Returns the number of failures + refusals. - Each pair is (label, file_text, github_text, push_callable). Every - item is classified before anything mutates, dry-run prints exactly - the live plan, and refusals are per-item — one divergent body does - not block the reflow-safe ones. + Each pair is (label, file_text, snapshot_text, fetch, push). + Dry runs classify against the run's snapshot; LIVE runs re-fetch + each target immediately before mutating and classify against that + fresh text, so the refuse-on-divergence guarantee also covers edits + made after the snapshot — e.g. while the confirmation prompt sat + open. (GitHub's API has no conditional issue update, so a + milliseconds-wide window remains; the revalidation shrinks it from + prompt-sized to that.) Refusals are per-item — one divergent body + does not block the reflow-safe ones. """ in_sync = pushed = refused = failed = 0 - for label, file_text, github_text, push in issue_pairs + milestone_pairs: + for label, file_text, snapshot_text, fetch, push in ( + issue_pairs + milestone_pairs): + if dry_run: + github_text = snapshot_text + else: + fresh = fetch() + if fresh.is_err: + failed += 1 + print(f" ! {label}: revalidation failed: " + f"{fresh.unwrap_err()}") + continue + github_text = fresh.unwrap() verdict = classify_sync(file_text, github_text) if verdict == "in-sync": in_sync += 1 @@ -415,6 +435,7 @@ def run_sync_bodies( f"issue {desired.id} (#{live.gh_number})", desired.body, live.body, + lambda n=live.gh_number: client.get_issue_body(n), lambda n=live.gh_number, b=desired.body: client.update_issue_body(n, b), ) @@ -427,6 +448,7 @@ def run_sync_bodies( f"milestone {ms.title} (#{ms.gh_number})", ms.description, live_desc.get(ms.gh_number, ""), + lambda n=ms.gh_number: client.get_milestone_description(n), lambda n=ms.gh_number, d=ms.description: client.update_milestone_description(n, d), ) diff --git a/scripts/tests/fake_gh/gh b/scripts/tests/fake_gh/gh index 96d12e0..d30cfd0 100755 --- a/scripts/tests/fake_gh/gh +++ b/scripts/tests/fake_gh/gh @@ -24,6 +24,7 @@ from __future__ import annotations import json import os +import re import sys from pathlib import Path @@ -82,6 +83,14 @@ def main() -> int: save("labels.json", labels) return 0 + if args[0] == "api" and re.search(r"/milestones/\d+$", args[1]) \ + and "PATCH" not in flag_values(args, "-X"): + number = int(args[1].rsplit("/", 1)[1]) + milestones = load("milestones.json", []) + print(json.dumps(next(m for m in milestones + if m["number"] == number))) + return 0 + if args[0] == "api" and "milestones" in args[1]: if "PATCH" in flag_values(args, "-X"): number = int(args[1].rsplit("/", 1)[1]) @@ -113,6 +122,12 @@ def main() -> int: print(json.dumps(load("issues.json", []))) return 0 + if args[0] == "api" and re.search(r"/issues/\d+$", args[1]): + number = int(args[1].rsplit("/", 1)[1]) + issues = load("issues.json", []) + print(json.dumps(next(i for i in issues if i["number"] == number))) + return 0 + if args[:2] == ["issue", "edit"]: number = int(args[2]) issues = load("issues.json", []) diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py index 7b2df53..2b9cbc2 100644 --- a/scripts/tests/test_integration.py +++ b/scripts/tests/test_integration.py @@ -453,7 +453,54 @@ def test_github_side_hard_breaks_are_divergent(self) -> None: def test_file_side_hard_breaks_still_reflow(self) -> None: # The push carries the file's hard break to GitHub intact — # nothing GitHub-side is lost, so no --force is demanded. + # Both markdown forms: trailing double-space AND backslash (the + # backslash survives unwrap's join, so it needs explicit + # defanging in the equivalence comparison). self.assertEqual(self.classify("a \nb", "a\nb"), "reflow") + self.assertEqual(self.classify("a\\\nb", "a\nb"), "reflow") + + def test_live_runs_classify_against_revalidated_text(self) -> None: + # The refuse-on-divergence guarantee must cover edits made + # after the snapshot (e.g. while the confirmation prompt sat + # open): the executor re-fetches each target immediately before + # mutating and classifies against THAT. + import contextlib + import io + from gh_project_populate import execute_sync_bodies + from _utils import Result + + quiet = io.StringIO() + pushes: list[str] = [] + pair = ( + "issue M1-1 (#1)", + "Body, reflowed onto one line.", + "Body,\nreflowed onto\none line.", # snapshot: reflow-safe + lambda: Result.ok("Edited on GitHub meanwhile."), # fresh: divergent + lambda: (pushes.append("pushed"), Result.ok(None))[1], + ) + with contextlib.redirect_stdout(quiet): + problems = execute_sync_bodies( + None, [pair], [], force=False, dry_run=False, delay=0, + ) + self.assertIn("divergent", quiet.getvalue()) + self.assertEqual(problems, 1) # refused on the FRESH text + self.assertEqual(pushes, []) # nothing mutated + + # Dry runs preview from the snapshot and never fetch. + fetches: list[str] = [] + pair = ( + "issue M1-1 (#1)", + "Body, reflowed onto one line.", + "Body,\nreflowed onto\none line.", + lambda: (fetches.append("fetched"), Result.ok(""))[1], + lambda: Result.ok(None), + ) + with contextlib.redirect_stdout(io.StringIO()): + problems = execute_sync_bodies( + None, [pair], [], force=False, dry_run=True, delay=0, + ) + self.assertEqual(problems, 0) + self.assertEqual(fetches, []) class SyncBodies(FakeGhHarness): From f2af04f5ee1593c9beedf10a5f164adaa17f3e87 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Tue, 18 Aug 2026 23:58:32 -0600 Subject: [PATCH 4/5] Self-review of PR #10: escape-aware classification and three polish items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewing my own diff surfaced one real correctness gap and three smaller improvements. The gap: update writes the DEFANGED form of marker-bearing GitHub bodies into the file (''), so the post-update mirror of any such issue classified divergent forever — and --force would 'resolve' it by pushing our escape artifacts over the real body. classify_sync now (a) treats the exact escaped mirror as in-sync, and (b) gives raw GitHub-side marker text no reflow shortcut (the file can only ever hold the escaped form, so any push sends escapes upstream; --force remains true last-writer-wins, escapes included, and says so). neutralize_markers moves to _gh_project_lib since both scripts now need it; update re-imports it. Pinned by classification units plus an end-to-end round trip: GitHub body edited to contain raw marker text, update defangs it into the file, sync reports in-sync and GitHub's raw text survives. (The test also documents why the plan-authoring path cannot produce this: lint correctly refuses raw markers in the file.) Polish: --force without --sync-bodies now errors instead of being silently ignored; execute_sync_bodies drops its never-used client parameter; the confirmation prompt says 'up to N' since in-sync items are counted but never rewritten. Suite 157 -> 161. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 16 ++++++++++ scripts/gh_project_populate.py | 24 ++++++++++++--- scripts/gh_project_update.py | 14 +-------- scripts/tests/test_integration.py | 49 +++++++++++++++++++++++++++++-- 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index a43f4bf..396303f 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -464,6 +464,22 @@ def record_issue_number(text: str, issue_id: str, gh_number: int) -> tuple[str, return "".join(out), found +# ── Marker neutralization (shared by update's renderer and populate's +# --sync-bodies classifier) ──────────────────────────────────────────────── + +# A live GitHub body/title may contain anything — including text shaped +# like this file's own region markers. Rendered verbatim, such a line +# would close (or open) a region on the next parse and corrupt the +# file, so the comment-opening form is defanged before insertion. The +# escape is deterministic, keeping `--check` stable across renders. +MARKER_IN_BODY_RE = re.compile(r"") diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index ae3665e..e47320f 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -82,6 +82,8 @@ sys.path.insert(0, str(_SCRIPT_DIR)) from _gh_project_lib import ( # noqa: E402 + MARKER_IN_BODY_RE, + neutralize_markers, GitHubClient, IssuePlan, LabelPlan, @@ -196,8 +198,20 @@ def classify_sync(file_body: str, github_body: str) -> str: ours, theirs = _normalized(file_body), _normalized(github_body) if ours == theirs: return "in-sync" + # update writes the DEFANGED form of marker-bearing GitHub bodies + # into the file (neutralize_markers), so a file that is exactly the + # escaped mirror of GitHub is in sync — pushing it would replace the + # real body with our escape artifacts. + if ours == neutralize_markers(theirs): + return "in-sync" if _HARD_BREAK_RE.search(theirs): return "divergent" + # Raw region-marker text on the GitHub side gets no reflow shortcut + # either: the file can only ever hold the escaped form, so any push + # would send escape artifacts upstream. --force remains true + # last-writer-wins, escapes included. + if MARKER_IN_BODY_RE.search(theirs): + return "divergent" # File-side hard breaks are defanged for the EQUIVALENCE test only: # the two-space form vanishes under unwrap's rstrip anyway, but the # backslash form would survive the join ("a\\ b") and spuriously @@ -208,7 +222,6 @@ def classify_sync(file_body: str, github_body: str) -> str: def execute_sync_bodies( - client: GitHubClient, issue_pairs: list[tuple], milestone_pairs: list[tuple], force: bool, @@ -464,15 +477,15 @@ def run_sync_bodies( print(" nothing to sync: no plan issue or milestone exists on " "GitHub yet") if not args.dry_run and (issue_pairs or milestone_pairs) and not args.yes: - print(f"This may rewrite {len(issue_pairs)} issue bodies and " - f"{len(milestone_pairs)} milestone descriptions on GitHub.") + print(f"This may rewrite up to {len(issue_pairs)} issue bodies " + f"and {len(milestone_pairs)} milestone descriptions on GitHub.") response = input("Continue? [y/N] ").strip().lower() if response not in ("y", "yes"): print("Aborted.") return 0 problems = execute_sync_bodies( - client, issue_pairs, milestone_pairs, + issue_pairs, milestone_pairs, force=args.force, dry_run=args.dry_run, delay=args.delay, ) if issue_plan.to_create: @@ -542,6 +555,9 @@ def main() -> int: args = build_parser().parse_args() if args.no_env_prefix: args.env_prefix = False + if args.force and not args.sync_bodies: + print("error: --force only applies to --sync-bodies", file=sys.stderr) + return 2 do_labels = not args.issues_only and not args.milestones_only and not args.skip_labels do_milestones = not args.issues_only and not args.labels_only diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py index 82ded8a..68bdc16 100755 --- a/scripts/gh_project_update.py +++ b/scripts/gh_project_update.py @@ -67,6 +67,7 @@ GitHubClient, Issue, ParsedFile, + neutralize_markers, parse_file, parse_repository, ) @@ -134,19 +135,6 @@ def group_unplanned( _ID_PREFIX_RE = re.compile(r"^\[M\d+-\d+[a-z]?\]\s+(.+)$") -# A live GitHub body may contain anything — including text shaped like -# this file's own region markers. Rendered verbatim, such a line would -# close (or open) a region on the next parse and corrupt the file, so -# the comment-opening form is defanged before insertion. The escape is -# deterministic, keeping `--check` stable across repeated renders. -_MARKER_IN_BODY_RE = re.compile(r" markers." + file_mirror = "See markers." + self.assertEqual(self.classify(file_mirror, github), "in-sync") + + def test_raw_markers_on_github_get_no_reflow_shortcut(self) -> None: + # Wrapping-equivalent, but GitHub carries raw marker text the + # file can only hold escaped: any push would send escapes + # upstream, so it must demand --force. + github = "About\n markers\nhere." + file_side = "About markers here." + self.assertEqual(self.classify(file_side, github), "divergent") + def test_live_runs_classify_against_revalidated_text(self) -> None: # The refuse-on-divergence guarantee must cover edits made # after the snapshot (e.g. while the confirmation prompt sat @@ -480,7 +497,7 @@ def test_live_runs_classify_against_revalidated_text(self) -> None: ) with contextlib.redirect_stdout(quiet): problems = execute_sync_bodies( - None, [pair], [], force=False, dry_run=False, delay=0, + [pair], [], force=False, dry_run=False, delay=0, ) self.assertIn("divergent", quiet.getvalue()) self.assertEqual(problems, 1) # refused on the FRESH text @@ -497,7 +514,7 @@ def test_live_runs_classify_against_revalidated_text(self) -> None: ) with contextlib.redirect_stdout(io.StringIO()): problems = execute_sync_bodies( - None, [pair], [], force=False, dry_run=True, delay=0, + [pair], [], force=False, dry_run=True, delay=0, ) self.assertEqual(problems, 0) self.assertEqual(fetches, []) @@ -570,6 +587,34 @@ def test_dry_run_predicts_and_mutates_nothing(self) -> None: self.assertIn("would push (reflow)", proc.stdout) self.assertEqual(json.dumps(self.issues_on_fake_github()), before) + def test_force_requires_sync_bodies(self) -> None: + proc = self.populate("--force") + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("--force only applies to --sync-bodies", proc.stderr) + + def test_marker_bearing_body_round_trip_stays_in_sync(self) -> None: + # An authored plan cannot carry raw marker text (lint forbids + # it — the file grammar owns those), so the realistic path is a + # GitHub-side edit: someone writes marker-like text into a live + # body, update mirrors it back DEFANGED, and sync must then + # report in-sync rather than divergent-forever. + self.assertEqual(self.populate().returncode, 0) + issues = self.issues_on_fake_github() + target = next(i for i in issues if i["title"].startswith("[M1-1]")) + target["body"] = "Body with inline." + (self.state / "issues.json").write_text(json.dumps(issues)) + self.assertEqual(self.update().returncode, 0) + self.assertIn("END GENERATED (escaped): milestone-9", + self.plan_path.read_text(encoding="utf-8")) + + proc = self.populate("--sync-bodies") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("in sync: issue M1-1", proc.stdout) + self.assertIn("0 pushed", proc.stdout) + # GitHub's raw marker text was not replaced by escape artifacts. + self.assertIn("", + self.m11()["body"]) + def test_sync_never_creates(self) -> None: proc = self.populate("--sync-bodies") self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) From ca05b20a61ee62977907ffbb75126e18d8be8f26 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Wed, 19 Aug 2026 00:12:09 -0600 Subject: [PATCH 5/5] Address Copilot round 3: empty-body sentinel, dry-run wording, parser units MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three suppressed findings were legitimate: - The empty-body sentinel was the second update-artifact blind spot (the first was marker escapes, caught in self-review): update renders an empty GitHub body as '_(no description on GitHub)_', the parser hands that back as the desired body, sync diverged forever, and --force would write the placeholder into a deliberately empty issue. The sentinel is now a shared lib constant (EMPTY_BODY_PLACEHOLDER, used by update's two renderers), and classify_sync accepts allow_empty_placeholder — applied to ISSUE targets only, per Copilot's own refinement, since a milestone description could legitimately BE that text. Sync targets are now a frozen SyncTarget dataclass carrying kind (the 5-tuples were at their limit). End-to-end round trip pinned: empty plan body -> populate pushes '' -> update writes the sentinel into the file -> sync reports in-sync and the GitHub body stays empty. - The dry-run summary reported planned mutations as completed ('2 pushed'); it now says 'would push', matching the per-item lines. - _parse_object_field gained the pure-core unit tests the house rules demand — and writing them exposed that the declared Result[str, ...] contract lied: a non-string field value was returned as-is. It is now a PARSING_ERROR. Suite 161 -> 168. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 18 ++++- scripts/gh_project_populate.py | 108 +++++++++++++++++---------- scripts/gh_project_update.py | 5 +- scripts/tests/test_gh_project_lib.py | 31 ++++++++ scripts/tests/test_integration.py | 82 +++++++++++++++----- 5 files changed, 184 insertions(+), 60 deletions(-) diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 396303f..37d7d95 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -474,6 +474,11 @@ def record_issue_number(text: str, issue_id: str, gh_number: int) -> tuple[str, # escape is deterministic, keeping `--check` stable across renders. MARKER_IN_BODY_RE = re.compile(r"