From 916cce7b5e9be024b3fb2c2a3f34b0442405b229 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Sun, 16 Aug 2026 14:11:36 -0600 Subject: [PATCH 1/2] update: render organically-filed issues into an 'unplanned' region (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issues filed directly on GitHub — no [MN-k] title prefix — were invisible to the generated regions even when they carried labels and GitHub milestones; in agda-native-air that hid the entire active agda-mcp hardening wave (its #68). A region marked BEGIN/END GENERATED: unplanned now mirrors them: grouped by the GitHub milestone title verbatim (no-milestone group last, rendered under #### headers), sorted by ascending issue number within a group, in the same block shape as planning issues minus the id — the [MN-k] prefix stays the stable-identifier contract of the milestone regions, so here the GitHub number is the only handle. Update warns on stderr when unplanned issues exist but the file has no such region. The safety half of the design: the region's content is GitHub-owned mirror text, so nothing inside it may ever be read back as plan STRUCTURE. mask_unplanned_regions() blanks region interiors (offset-preserving, marker lines intact) before parse_project_plan and before lint's heading scans — an organic title like 'Issue M9-9: ...' cannot become a phantom plan issue, and one merely starting with 'Issue' cannot trip the malformed-heading lint. Issue records carry the new read-side milestone_title field; titles are now marker-neutralized in both renderers (bodies already were). Behavior change pinned by test: a planning issue whose prefix was stripped on GitHub now surfaces in the unplanned region instead of vanishing from the file (it still matches by recorded number on populate, so no duplicates). Verified against live agda-native-air data (read-only, on a copy of its plan): the #68 wave — 24 organic blocks across two milestone groups — renders and is --check-stable across consecutive runs, which is the issue's acceptance criterion. Suite 115 -> 132; example plan and the roadmap-upkeep skill document the region; engine VERSION 0.3.0. AI-assisted: Claude Fable 5 (Anthropic) --- .claude/skills/roadmap-upkeep/SKILL.md | 15 ++++ docs/GITHUB_PROJECT.md | 18 ++++ scripts/VERSION | 2 +- scripts/_gh_project_lib.py | 59 +++++++++++-- scripts/gh_project_update.py | 112 ++++++++++++++++++++++-- scripts/tests/test_gh_project_lib.py | 45 +++++++++- scripts/tests/test_gh_project_lint.py | 29 +++++- scripts/tests/test_gh_project_update.py | 97 +++++++++++++++++++- scripts/tests/test_integration.py | 77 +++++++++++++++- 9 files changed, 432 insertions(+), 22 deletions(-) diff --git a/.claude/skills/roadmap-upkeep/SKILL.md b/.claude/skills/roadmap-upkeep/SKILL.md index dc9484a..6b5bd09 100644 --- a/.claude/skills/roadmap-upkeep/SKILL.md +++ b/.claude/skills/roadmap-upkeep/SKILL.md @@ -45,6 +45,21 @@ Issue bodies of already-created issues are edited ON GITHUB (the region mirror would overwrite in-file edits on the next update). Prose outside regions is edited in the file. +## Organically-filed issues (the `unplanned` region) + +Issues filed directly on GitHub without a `[MN-k]` title prefix never +appear in milestone regions — the prefix is those regions' identifier +contract. A region marked `` / +`` mirrors them instead: grouped by +GitHub milestone (title verbatim, no-milestone group last), sorted by +issue number, same block shape minus the id. Everything inside it is +GitHub-owned mirror content — the parser and lint deliberately ignore +it, so organic titles can never inject plan structure. To promote an +organic issue into the plan, retitle it on GitHub with the next free +`[MN-k]` prefix; the next update moves it into its milestone region. +If unplanned issues exist and the file has no such region, update warns +on stderr. + ## The scheduled freshness workflow `.github/workflows/project-plan-update.yml` runs Mondays (and on diff --git a/docs/GITHUB_PROJECT.md b/docs/GITHUB_PROJECT.md index 45d967c..a05171f 100644 --- a/docs/GITHUB_PROJECT.md +++ b/docs/GITHUB_PROJECT.md @@ -162,6 +162,24 @@ compatibility guarantees. +## Unplanned work + +Issues filed organically on GitHub — no `[MN-k]` prefix in the title — +are invisible to the milestone regions above. This region mirrors +them, grouped by GitHub milestone and sorted by issue number, so +field-driven work still shows up in the plan. The `[MN-k]` prefix +remains the stable-identifier contract of the milestone regions; here +the GitHub number is the only handle. Delete this section if you want +planned-only rendering (update then warns whenever unplanned issues +exist). + + + +_(no organically-filed issues — every issue on GitHub carries a `[MN-k]` planning prefix)_ + + +--- + ## How to use this file 1. Replace the plan above with your project (keep the structure), set diff --git a/scripts/VERSION b/scripts/VERSION index 0ea3a94..0d91a54 100644 --- a/scripts/VERSION +++ b/scripts/VERSION @@ -1 +1 @@ -0.2.0 +0.3.0 diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index ab3b740..00d1713 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -136,6 +136,11 @@ class Issue: # from a recorded `(#N)` heading # suffix on the parse side assignees: tuple[str, ...] = () # GitHub logins; read side only + milestone_title: Optional[str] = None # GitHub milestone title verbatim; + # read side only — the unplanned + # region groups by it, since + # organically-filed issues have + # no plan index @dataclass(frozen=True) @@ -166,6 +171,31 @@ class RepoSnapshot: ISSUE_HEADING_RE = re.compile(r"^### Issue (M\d+-\d+[a-z]?): (.+?)[ \t]*$", re.MULTILINE) +# The `unplanned` generated region mirrors organically-filed issues — +# GitHub titles with no `[MN-k]` prefix, rendered by update. Its +# content is GitHub-owned mirror text, so nothing in it may ever be read +# back as plan STRUCTURE: an organic title that happens to look like a +# planning heading (`Issue M1-5: fix the parser`) must not become a +# phantom plan issue, and one that merely starts with "Issue" must not +# trip the malformed-heading lint. +UNPLANNED_REGION_RE = re.compile( + r"()(.*?)()", + re.DOTALL, +) + + +def mask_unplanned_regions(text: str) -> str: + """Blank the INTERIOR of every `unplanned` region for plan parsing. + + Marker lines survive (region structure stays checkable); every + non-newline character between them becomes a space, so offsets and + line numbers in diagnostics still point at the real file. + """ + def blank(m: re.Match) -> str: + interior = "".join(c if c == "\n" else " " for c in m.group(2)) + return m.group(1) + interior + m.group(3) + return UNPLANNED_REGION_RE.sub(blank, text) + # A recorded issue number at the end of a heading: `(#123)` as written # back by populate, or `(#123, closed)` as rendered by update. HEADING_NUMBER_SUFFIX_RE = re.compile(r"\s*\(#(\d+)(?:,\s*[a-z]+)?\)$") @@ -210,12 +240,18 @@ def parse_repository(text: str) -> Optional[str]: def parse_project_plan(text: str) -> ProjectPlan: - """Parse the structured project plan markdown into a ProjectPlan.""" + """Parse the structured project plan markdown into a ProjectPlan. + + The interiors of `unplanned` regions are masked first: they mirror + GitHub-owned content, never plan structure (see + mask_unplanned_regions). + """ + masked = mask_unplanned_regions(text) return ProjectPlan( - repository=parse_repository(text), - milestones=tuple(_parse_milestones(text)), - labels=tuple(_parse_labels(text)), - issues=tuple(_parse_issues(text)), + repository=parse_repository(masked), + milestones=tuple(_parse_milestones(masked)), + labels=tuple(_parse_labels(masked)), + issues=tuple(_parse_issues(masked)), ) @@ -692,10 +728,17 @@ def lint_plan(text: str) -> tuple[Problem, ...]: f"cannot tell which milestone issues M{n}-* belong to", )) + # The heading scans run on MASKED text: an `unplanned` region + # mirrors organic GitHub titles, and one that merely starts with + # "Issue" (or resembles a milestone heading) is mirror content, not + # a malformed plan heading. Offsets survive masking, so line + # numbers still point at the real file. + scannable = mask_unplanned_regions(text) + # A heading that says `### Issue` but does not parse is worse than a # missing one: populate never pushes it, and the next update erases # it when the surrounding generated region is rebuilt from GitHub. - for m in re.finditer(r"^### Issue\b[^\n]*$", text, re.MULTILINE): + for m in re.finditer(r"^### Issue\b[^\n]*$", scannable, re.MULTILINE): if not ISSUE_HEADING_RE.match(m.group(0)): problems.append(Problem( "error", @@ -711,7 +754,7 @@ def lint_plan(text: str) -> tuple[Problem, ...]: # headings are scanned — prose headings like `### Milestone # dependencies` are legitimate (the exemplar plans use them). milestone_heading_re = re.compile(r"^### Milestone (\d+) — (.+)$") - for m in re.finditer(r"^### Milestone\s+\d+\b[^\n]*$", text, re.MULTILINE): + for m in re.finditer(r"^### Milestone\s+\d+\b[^\n]*$", scannable, re.MULTILINE): if not milestone_heading_re.match(m.group(0)): problems.append(Problem( "error", @@ -1032,6 +1075,7 @@ def to_issues(data: list[dict]) -> list[Issue]: gh_title = gh_ms.get("title", "") tm = re.match(r"^(\d+)\.", gh_title) inferred = int(tm.group(1)) if tm else (ms_idx if ms_idx is not None else 0) + gh_milestone = item.get("milestone") or {} out.append(Issue( id=issue_id, title=title, @@ -1043,6 +1087,7 @@ def to_issues(data: list[dict]) -> list[Issue]: assignees=tuple( a["login"] for a in item.get("assignees", []) or [] ), + milestone_title=gh_milestone.get("title") or None, )) return out return _parse_json(stdout, "issues").map(to_issues) diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py index bd0d0f4..efdf9f3 100755 --- a/scripts/gh_project_update.py +++ b/scripts/gh_project_update.py @@ -102,6 +102,34 @@ def issue_sort_key(issue: Issue) -> tuple[int, int, str]: return (int(m.group(1)), int(m.group(2)), m.group(3) or "") if m else (10**9, 10**9, "") +def group_unplanned( + issues: list[Issue], +) -> tuple[tuple[str | None, tuple[Issue, ...]], ...]: + """Group organically-filed issues (id="") for the `unplanned` region. + + Grouped by the GitHub milestone title verbatim (organic issues have + no plan index to key on); groups ordered by title with the + no-milestone group last, issues within a group by ascending number. + Everything is deterministic, so repeated renders are `--check` + stable. + """ + by_title: dict[str | None, list[Issue]] = {} + for issue in issues: + if issue.id: + continue + by_title.setdefault(issue.milestone_title, []).append(issue) + ordered_titles = sorted( + by_title, key=lambda t: (t is None, t or "") + ) + return tuple( + ( + title, + tuple(sorted(by_title[title], key=lambda i: i.gh_number or 0)), + ) + for title in ordered_titles + ) + + # ── Region rendering ───────────────────────────────────────────────────────── _ID_PREFIX_RE = re.compile(r"^\[M\d+-\d+[a-z]?\]\s+(.+)$") @@ -127,7 +155,9 @@ def strip_id_prefix(title: str) -> str: def render_issue(issue: Issue) -> str: - title = strip_id_prefix(issue.title) + # Titles are dynamic GitHub content too: defang marker look-alikes + # exactly as in bodies. + title = neutralize_markers(strip_id_prefix(issue.title)) state_suffix = ", closed" if issue.state == "closed" else "" ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)" labels = ", ".join(f"`{lbl}`" for lbl in issue.labels) @@ -152,7 +182,56 @@ def render_issue(issue: Issue) -> str: ) -def render_region(region_id: str, issues_by_ms: dict[int, list[Issue]]) -> str: +def render_unplanned_issue(issue: Issue) -> str: + """The planning block shape, minus the id — organic issues have none. + + The `[MN-k]` prefix stays the stable-identifier contract of the + milestone regions; here the GitHub number is the only handle, so it + alone appears in the heading. + """ + state_suffix = ", closed" if issue.state == "closed" else "" + ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)" + labels = ", ".join(f"`{lbl}`" for lbl in issue.labels) + assignees = ( + f"**Assignees:** {', '.join('@' + a for a in issue.assignees)}\n\n" + if issue.assignees else "" + ) + body = issue.body.strip() if issue.body else "_(no description on GitHub)_" + body = neutralize_markers(body) + title = neutralize_markers(issue.title) + return ( + f"### {title} ({ref})\n" + f"\n" + f"**Labels:** {labels}\n" + f"\n" + f"{assignees}" + f"{body}\n" + ) + + +def render_unplanned_region( + unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...], +) -> str: + if not unplanned: + return ( + "\n_(no organically-filed issues — every issue on GitHub " + "carries a `[MN-k]` planning prefix)_\n" + ) + parts: list[str] = [] + for title, issues in unplanned: + header = title if title is not None else "(no milestone)" + blocks = "\n---\n\n".join(render_unplanned_issue(i) for i in issues) + parts.append(f"#### {header}\n\n{blocks}") + return "\n" + "\n\n".join(parts) + "\n" + + +def render_region( + region_id: str, + issues_by_ms: dict[int, list[Issue]], + unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...], +) -> str: + if region_id == "unplanned": + return render_unplanned_region(unplanned) m = re.match(r"^milestone-(\d+)$", region_id) if m is None: return f"\n\n" @@ -164,13 +243,17 @@ def render_region(region_id: str, issues_by_ms: dict[int, list[Issue]]) -> str: return "\n" + "\n---\n\n".join(blocks) + "\n" -def assemble_file(parsed: ParsedFile, issues_by_ms: dict[int, list[Issue]]) -> str: +def assemble_file( + parsed: ParsedFile, + issues_by_ms: dict[int, list[Issue]], + unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...] = (), +) -> str: parts: list[str] = [] for i, manual in enumerate(parsed.manuals[:-1]): rid = parsed.ids[i] parts.append(manual) parts.append(f"\n") - parts.append(render_region(rid, issues_by_ms)) + parts.append(render_region(rid, issues_by_ms, unplanned)) parts.append(f"") parts.append(parsed.manuals[-1]) return "".join(parts) @@ -196,12 +279,31 @@ def warn_orphan_milestones(parsed: ParsedFile, issues_by_ms: dict[int, list[Issu ) +def warn_unrendered_unplanned( + parsed: ParsedFile, + unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...], +) -> None: + """Stderr-warn when organically-filed issues exist but the file has + no `unplanned` region to show them in — the invisibility this + region exists to end (issue #3).""" + if not unplanned or "unplanned" in parsed.ids: + return + count = sum(len(issues) for _, issues in unplanned) + print( + f"warning: {count} organically-filed issue(s) (no [MN-k] prefix) " + f"have no region", + file=sys.stderr, + ) + + # ── Main ───────────────────────────────────────────────────────────────────── def _render(parsed: ParsedFile, issues: list[Issue]) -> str: issues_by_ms = group_by_milestone(issues) + unplanned = group_unplanned(issues) warn_orphan_milestones(parsed, issues_by_ms) - return assemble_file(parsed, issues_by_ms) + warn_unrendered_unplanned(parsed, unplanned) + return assemble_file(parsed, issues_by_ms, unplanned) def run(markdown: Path, repo: str | None, env_prefix: bool) -> Result[str, PipelineError]: diff --git a/scripts/tests/test_gh_project_lib.py b/scripts/tests/test_gh_project_lib.py index 84e6a88..f60e387 100644 --- a/scripts/tests/test_gh_project_lib.py +++ b/scripts/tests/test_gh_project_lib.py @@ -121,6 +121,13 @@ +## Unplanned work + + + +_(no organically-filed issues — every issue on GitHub carries a `[MN-k]` planning prefix)_ + + ## How to use Trailing prose that must not leak into any issue body. @@ -251,6 +258,40 @@ def test_milestone_idx_from_id(self) -> None: ) +class UnplannedMasking(unittest.TestCase): + """The unplanned region mirrors GitHub content; nothing inside it + may be read back as plan structure.""" + + MIRRORED = PLAN.replace( + "_(no organically-filed issues — every issue on GitHub carries " + "a `[MN-k]` planning prefix)_", + "#### 1. Core\n\n" + "### Issue M9-9: phantom planning heading (#77)\n\n" + "**Labels:** `not-a-plan-label`\n\n" + "Organic body.\n", + ) + + def test_mask_preserves_markers_and_offsets(self) -> None: + masked = lib.mask_unplanned_regions(self.MIRRORED) + self.assertEqual(len(masked), len(self.MIRRORED)) + self.assertEqual(masked.count("\n"), self.MIRRORED.count("\n")) + self.assertIn("", masked) + self.assertIn("", masked) + self.assertNotIn("phantom planning heading", masked) + + def test_no_phantom_plan_issue(self) -> None: + plan = parse_project_plan(self.MIRRORED) + self.assertEqual( + {i.id for i in plan.issues}, {"M1-1", "M1-2", "M2-1", "M2-1a"} + ) + + def test_milestone_regions_still_parse(self) -> None: + # Masking is surgical: planned regions keep their content. + plan = parse_project_plan(self.MIRRORED) + m11 = next(i for i in plan.issues if i.id == "M1-1") + self.assertIn("Body of M1-1.", m11.body) + + class RecordIssueNumber(unittest.TestCase): def test_adds_suffix(self) -> None: text, found = record_issue_number(PLAN, "M1-1", 7) @@ -290,8 +331,8 @@ def test_id_match_is_exact_not_prefix(self) -> None: class ParseFileRegions(unittest.TestCase): def test_round_trip_structure(self) -> None: parsed = parse_file(PLAN).unwrap() - self.assertEqual(parsed.ids, ("milestone-1", "milestone-2")) - self.assertEqual(len(parsed.manuals), 3) + self.assertEqual(parsed.ids, ("milestone-1", "milestone-2", "unplanned")) + self.assertEqual(len(parsed.manuals), 4) self.assertTrue(parsed.manuals[-1].startswith("\n\n## How to use")) def test_unterminated_begin(self) -> None: diff --git a/scripts/tests/test_gh_project_lint.py b/scripts/tests/test_gh_project_lint.py index 403eb38..db9dcf8 100644 --- a/scripts/tests/test_gh_project_lint.py +++ b/scripts/tests/test_gh_project_lint.py @@ -33,9 +33,16 @@ def test_fixture_is_clean(self) -> None: class MarkerProblems(unittest.TestCase): def test_unterminated(self) -> None: - broken = PLAN.replace("\n", "") + # Remove the LAST region's END: no later marker can mis-pair. + broken = PLAN.replace("\n", "") self.assertTrue(any("unterminated" in e for e in errors(broken))) + def test_deleted_end_mid_file_reports_mismatch(self) -> None: + # Removing an inner END makes its BEGIN close against the next + # region's END — reported as mismatched markers. + broken = PLAN.replace("\n", "") + self.assertTrue(any("mismatched" in e for e in errors(broken))) + def test_mismatched(self) -> None: broken = PLAN.replace( "", @@ -126,6 +133,26 @@ def test_exact_duplicate(self) -> None: for e in errors(broken))) +class UnplannedRegionIsMirror(unittest.TestCase): + def test_organic_headings_inside_region_are_not_flagged(self) -> None: + mirrored = PLAN.replace( + "_(no organically-filed issues — every issue on GitHub " + "carries a `[MN-k]` planning prefix)_", + "### Issue tracker went down twice (#12)\n\n" + "**Labels:** `ops`\n\nOrganic body.\n\n" + "### Milestone 3 retro notes (#13)\n\nMore organic body.\n", + ) + self.assertEqual(errors(mirrored), []) + + def test_same_heading_outside_region_is_still_flagged(self) -> None: + broken = PLAN.replace( + "## How to use", + "### Issue tracker went down twice (#12)\n\n## How to use", + ) + self.assertTrue(any("malformed issue heading" in e + for e in errors(broken))) + + class Advisories(unittest.TestCase): def test_missing_repository_is_warning(self) -> None: broken = PLAN.replace("**Repository**: `octocat/demo`\n", "") diff --git a/scripts/tests/test_gh_project_update.py b/scripts/tests/test_gh_project_update.py index c47a4e4..abbb30e 100644 --- a/scripts/tests/test_gh_project_update.py +++ b/scripts/tests/test_gh_project_update.py @@ -83,11 +83,11 @@ def test_marker_in_live_body_is_neutralized(self) -> None: class RenderRegion(unittest.TestCase): def test_unknown_region_id_fails_loudly(self) -> None: - out = upd.render_region("weird-id", {}) + out = upd.render_region("weird-id", {}, ()) self.assertIn("no rendering rule", out) def test_empty_milestone(self) -> None: - out = upd.render_region("milestone-3", {}) + out = upd.render_region("milestone-3", {}, ()) self.assertIn("no open or closed issues", out) def test_parent_before_children(self) -> None: @@ -98,7 +98,7 @@ def test_parent_before_children(self) -> None: issue("M2-7a", "[M2-7a] Child A", gh_number=2), ] } - out = upd.render_region("milestone-2", issues) + out = upd.render_region("milestone-2", issues, ()) self.assertLess(out.index("Parent"), out.index("Child A")) self.assertLess(out.index("Child A"), out.index("Child B")) @@ -124,6 +124,97 @@ def test_manuals_preserved_and_regions_rebuilt(self) -> None: self.assertEqual(reparsed.manuals, parsed.manuals) +class UnplannedRegion(unittest.TestCase): + """Issue #3: organically-filed issues render under `unplanned`.""" + + def test_grouping_by_milestone_title_none_last(self) -> None: + groups = upd.group_unplanned([ + issue("", "C", gh_number=9, milestone_title=None), + issue("", "A", gh_number=7, milestone_title="2. Polish"), + issue("", "B", gh_number=3, milestone_title="1. Core"), + issue("", "D", gh_number=1, milestone_title="1. Core"), + issue("M1-1", "[M1-1] planned", gh_number=2), # excluded + ]) + self.assertEqual( + [(t, [i.gh_number for i in xs]) for t, xs in groups], + [("1. Core", [1, 3]), ("2. Polish", [7]), (None, [9])], + ) + + def test_block_shape_minus_the_id(self) -> None: + out = upd.render_unplanned_issue( + issue("", "Fix flaky timeout", gh_number=68, + labels=("agda-mcp",), assignees=("octocat",), + body="Body.", state="closed") + ) + self.assertIn("### Fix flaky timeout (#68, closed)\n", out) + self.assertIn("**Labels:** `agda-mcp`\n", out) + self.assertIn("**Assignees:** @octocat\n", out) + self.assertIn("Body.", out) + self.assertNotIn("Issue :", out) + + def test_region_groups_under_milestone_headers(self) -> None: + groups = upd.group_unplanned([ + issue("", "One", gh_number=1, milestone_title="1. Core"), + issue("", "Two", gh_number=2, milestone_title=None), + ]) + out = upd.render_region("unplanned", {}, groups) + self.assertIn("#### 1. Core\n", out) + self.assertIn("#### (no milestone)\n", out) + self.assertLess(out.index("#### 1. Core"), out.index("#### (no milestone)")) + + def test_empty_region_placeholder(self) -> None: + out = upd.render_region("unplanned", {}, ()) + self.assertIn("no organically-filed issues", out) + + def test_render_is_deterministic(self) -> None: + groups = upd.group_unplanned([ + issue("", "One", gh_number=1, milestone_title="1. Core"), + issue("", "Two", gh_number=2), + ]) + self.assertEqual( + upd.render_region("unplanned", {}, groups), + upd.render_region("unplanned", {}, groups), + ) + + def test_marker_in_organic_title_is_neutralized(self) -> None: + hostile = issue( + "", "About markers", + gh_number=5, + ) + out = upd.render_unplanned_issue(hostile) + self.assertNotRegex(out, r"\n" + "\n" + ) + self.assertEqual( + self._warnings(content, [issue("", "Organic", gh_number=9)]), "") + + def test_silent_when_nothing_unplanned(self) -> None: + self.assertEqual(self._warnings("prose\n", []), "") + + class GroupByMilestone(unittest.TestCase): def test_non_planning_issues_are_excluded(self) -> None: # The snapshot carries id="" records (prefix-less titles) for diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py index d223473..25fcdea 100644 --- a/scripts/tests/test_integration.py +++ b/scripts/tests/test_integration.py @@ -183,12 +183,18 @@ def test_stripped_prefix_does_not_duplicate(self) -> None: self.assertEqual(len(self.issues_on_fake_github()), 4) self.assertIn(f"- exists: issue #{victim['number']}", rerun.stdout) - # Update still works: the prefix-less issue simply leaves the - # region (it has no stable identifier to render under). + # Update still works: the prefix-less issue leaves its milestone + # region (no stable identifier to render under) and surfaces in + # the unplanned region instead — visible, not vanished. upd = self.update() self.assertEqual(upd.returncode, 0, upd.stderr) text = self.plan_path.read_text(encoding="utf-8") - self.assertNotIn("renamed with no prefix", text) + unplanned = text.split("")[1] + unplanned = unplanned.split("")[0] + self.assertIn("renamed with no prefix", unplanned) + milestone_1 = text.split("")[1] + milestone_1 = milestone_1.split("")[0] + self.assertNotIn("renamed with no prefix", milestone_1) class MilestoneAvailability(FakeGhHarness): @@ -332,6 +338,71 @@ def test_update_failure_is_exit_2_not_drift(self) -> None: self.assertIn("no repository", proc.stderr) +class OrganicIssues(FakeGhHarness): + """Issue #3 end to end: organically-filed issues render into the + `unplanned` region, stay out of populate's way, and cannot inject + plan structure on reparse.""" + + def seed_organic(self) -> None: + issues = self.issues_on_fake_github() + issues += [ + {"number": 68, "title": "Harden the flaky timeout", + "body": "Field-driven fix.", + "labels": [{"name": "documentation"}], + "milestone": {"title": "1. Working core"}, + "state": "OPEN", "assignees": []}, + {"number": 70, "title": "Issue M9-9: looks like a plan heading", + "body": "Adversarial organic title.", + "labels": [], "milestone": None, + "state": "OPEN", "assignees": []}, + ] + (self.state / "issues.json").write_text(json.dumps(issues)) + + def unplanned_region(self) -> str: + text = self.plan_path.read_text(encoding="utf-8") + inner = text.split("")[1] + return inner.split("")[0] + + def test_rendered_grouped_and_check_stable(self) -> None: + self.assertEqual(self.populate().returncode, 0) + self.seed_organic() + self.assertEqual(self.update().returncode, 0) + + region = self.unplanned_region() + self.assertIn("#### 1. Working core", region) + self.assertIn("### Harden the flaky timeout (#68)", region) + self.assertIn("#### (no milestone)", region) + self.assertIn("### Issue M9-9: looks like a plan heading (#70)", region) + + # --check-stable across runs (the acceptance criterion). + self.assertEqual(self.update("--check").returncode, 0) + + # Closing an organic issue is drift, healed by the next update. + issues = self.issues_on_fake_github() + next(i for i in issues if i["number"] == 68)["state"] = "CLOSED" + (self.state / "issues.json").write_text(json.dumps(issues)) + self.assertEqual(self.update("--check").returncode, 1) + self.assertEqual(self.update().returncode, 0) + self.assertIn("(#68, closed)", self.unplanned_region()) + + def test_mirror_content_never_becomes_plan_structure(self) -> None: + self.assertEqual(self.populate().returncode, 0) + self.seed_organic() + self.assertEqual(self.update().returncode, 0) + + # The updated file — now mirroring an adversarial organic title + # and a plan-foreign label — still lints clean... + lint = self.run_script("gh_project_lint.py") + self.assertEqual(lint.returncode, 0, lint.stdout + lint.stderr) + + # ...and a re-populate plans nothing: no phantom M9-9, no + # duplicate of anything. + rerun = self.populate("--dry-run") + self.assertEqual(rerun.returncode, 0, rerun.stdout + rerun.stderr) + self.assertNotIn("+ create issue", rerun.stdout) + self.assertNotIn("M9-9", rerun.stdout) + + class PopulateRefusesBrokenPlans(FakeGhHarness): def test_lint_gate(self) -> None: self.plan_path.write_text( From 41c5277a727e9ee8ce0359e0a582e857898af661 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Sun, 16 Aug 2026 14:22:04 -0600 Subject: [PATCH 2/2] Neutralize markers in rendered milestone titles and label names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's review of PR #5 was right: render_unplanned_region() wrote GitHub milestone titles verbatim into the #### group headers, and the marker regexes are not line-anchored — a title containing '' becomes a real marker mid-line. Reproduced before fixing: the region closes early and the file's true END marker then hard-fails parse_file ('no matching BEGIN'), so the next update/lint breaks outright. Fixing it surfaced one more instance of the same class: label names are rendered verbatim in BOTH renderers, GitHub label names allow arbitrary punctuation up to 50 chars (the marker string is 33), and backticks are no protection since the regexes match raw text. All dynamic GitHub content in rendered output — bodies, titles, milestone titles, label names — now passes through neutralize_markers; assignee logins are alphanumeric-plus-hyphen and cannot carry markers. Three regression tests, including a round trip proving an assembled region with a hostile milestone title re-parses intact. Suite 132 -> 134. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/gh_project_update.py | 16 +++++++---- scripts/tests/test_gh_project_update.py | 35 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py index efdf9f3..82ded8a 100755 --- a/scripts/gh_project_update.py +++ b/scripts/gh_project_update.py @@ -155,12 +155,14 @@ def strip_id_prefix(title: str) -> str: def render_issue(issue: Issue) -> str: - # Titles are dynamic GitHub content too: defang marker look-alikes - # exactly as in bodies. + # Titles and label names are dynamic GitHub content too: defang + # marker look-alikes exactly as in bodies. (GitHub label names + # allow arbitrary punctuation, and backticks are no protection — + # the marker regexes match raw text, not rendered markdown.) title = neutralize_markers(strip_id_prefix(issue.title)) state_suffix = ", closed" if issue.state == "closed" else "" ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)" - labels = ", ".join(f"`{lbl}`" for lbl in issue.labels) + labels = neutralize_markers(", ".join(f"`{lbl}`" for lbl in issue.labels)) # Assignees are GitHub-owned state (like open/closed); render them so # the file answers "who is on this?" without a browser. Both # metadata lines use the canonical `**Word:**` spelling that the @@ -191,7 +193,7 @@ def render_unplanned_issue(issue: Issue) -> str: """ state_suffix = ", closed" if issue.state == "closed" else "" ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)" - labels = ", ".join(f"`{lbl}`" for lbl in issue.labels) + labels = neutralize_markers(", ".join(f"`{lbl}`" for lbl in issue.labels)) assignees = ( f"**Assignees:** {', '.join('@' + a for a in issue.assignees)}\n\n" if issue.assignees else "" @@ -219,7 +221,11 @@ def render_unplanned_region( ) parts: list[str] = [] for title, issues in unplanned: - header = title if title is not None else "(no milestone)" + # Milestone titles are dynamic GitHub content rendered into the + # file: defang marker look-alikes like titles/bodies/labels. + header = ( + neutralize_markers(title) if title is not None else "(no milestone)" + ) blocks = "\n---\n\n".join(render_unplanned_issue(i) for i in issues) parts.append(f"#### {header}\n\n{blocks}") return "\n" + "\n\n".join(parts) + "\n" diff --git a/scripts/tests/test_gh_project_update.py b/scripts/tests/test_gh_project_update.py index abbb30e..46245c0 100644 --- a/scripts/tests/test_gh_project_update.py +++ b/scripts/tests/test_gh_project_update.py @@ -185,6 +185,41 @@ def test_marker_in_organic_title_is_neutralized(self) -> None: self.assertNotRegex(out, r" era"), + ]) + out = upd.render_region("unplanned", {}, groups) + self.assertNotRegex(out, r" era", out) + # The assembled document must survive a round trip. + assembled = ( + "\n" + + out + + "\n" + ) + reparsed = parse_file(assembled).unwrap() + self.assertEqual(reparsed.ids, ("unplanned",)) + + def test_marker_in_label_name_is_neutralized(self) -> None: + # GitHub label names allow arbitrary punctuation up to 50 chars + # (the marker string is 33); backticks are no protection since + # the marker regexes match raw text. + hostile = ("",) + for render, args in ( + (upd.render_unplanned_issue, + issue("", "Organic", gh_number=5, labels=hostile)), + (upd.render_issue, + issue("M1-1", "[M1-1] Planned", gh_number=6, labels=hostile)), + ): + out = render(args) + self.assertNotRegex(out, r"