update: render organically-filed issues into an unplanned region (#3)

.claude/skills/roadmap-upkeep/SKILL.md

@@ -45,6 +45,21 @@ Issue bodies of already-created issues are edited ON GITHUB (the region
 mirror would overwrite in-file edits on the next update).  Prose outside
 regions is edited in the file.
 
+## Organically-filed issues (the `unplanned` region)
+
+Issues filed directly on GitHub without a `[MN-k]` title prefix never
+appear in milestone regions — the prefix is those regions' identifier
+contract.  A region marked `<!-- BEGIN GENERATED: unplanned -->` /
+`<!-- END GENERATED: unplanned -->` mirrors them instead: grouped by
+GitHub milestone (title verbatim, no-milestone group last), sorted by
+issue number, same block shape minus the id.  Everything inside it is
+GitHub-owned mirror content — the parser and lint deliberately ignore
+it, so organic titles can never inject plan structure.  To promote an
+organic issue into the plan, retitle it on GitHub with the next free
+`[MN-k]` prefix; the next update moves it into its milestone region.
+If unplanned issues exist and the file has no such region, update warns
+on stderr.
+
 ## The scheduled freshness workflow
 
 `.github/workflows/project-plan-update.yml` runs Mondays (and on

docs/GITHUB_PROJECT.md

@@ -162,6 +162,24 @@ compatibility guarantees.
 
 <!-- END GENERATED: milestone-2 -->
 
+## Unplanned work
+
+Issues filed organically on GitHub — no `[MN-k]` prefix in the title —
+are invisible to the milestone regions above.  This region mirrors
+them, grouped by GitHub milestone and sorted by issue number, so
+field-driven work still shows up in the plan.  The `[MN-k]` prefix
+remains the stable-identifier contract of the milestone regions; here
+the GitHub number is the only handle.  Delete this section if you want
+planned-only rendering (update then warns whenever unplanned issues
+exist).
+
+<!-- BEGIN GENERATED: unplanned -->
+
+_(no organically-filed issues — every issue on GitHub carries a `[MN-k]` planning prefix)_
+<!-- END GENERATED: unplanned -->
+
+---
+
 ## How to use this file
 
 1. Replace the plan above with your project (keep the structure), set

scripts/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

scripts/_gh_project_lib.py

@@ -136,6 +136,11 @@ class Issue:
                                               # from a recorded `(#N)` heading
                                               # suffix on the parse side
     assignees: tuple[str, ...] = ()           # GitHub logins; read side only
+    milestone_title: Optional[str] = None     # GitHub milestone title verbatim;
+                                              # read side only — the unplanned
+                                              # region groups by it, since
+                                              # organically-filed issues have
+                                              # no plan index
 
 
 @dataclass(frozen=True)
@@ -166,6 +171,31 @@ class RepoSnapshot:
 
 ISSUE_HEADING_RE = re.compile(r"^### Issue (M\d+-\d+[a-z]?): (.+?)[ \t]*$", re.MULTILINE)
 
+# The `unplanned` generated region mirrors organically-filed issues —
+# GitHub titles with no `[MN-k]` prefix, rendered by update.  Its
+# content is GitHub-owned mirror text, so nothing in it may ever be read
+# back as plan STRUCTURE: an organic title that happens to look like a
+# planning heading (`Issue M1-5: fix the parser`) must not become a
+# phantom plan issue, and one that merely starts with "Issue" must not
+# trip the malformed-heading lint.
+UNPLANNED_REGION_RE = re.compile(
+    r"(<!--\s*BEGIN GENERATED:\s*unplanned\s*-->)(.*?)(<!--\s*END GENERATED:\s*unplanned\s*-->)",
+    re.DOTALL,
+)
+
+
+def mask_unplanned_regions(text: str) -> str:
+    """Blank the INTERIOR of every `unplanned` region for plan parsing.
+
+    Marker lines survive (region structure stays checkable); every
+    non-newline character between them becomes a space, so offsets and
+    line numbers in diagnostics still point at the real file.
+    """
+    def blank(m: re.Match) -> str:
+        interior = "".join(c if c == "\n" else " " for c in m.group(2))
+        return m.group(1) + interior + m.group(3)
+    return UNPLANNED_REGION_RE.sub(blank, text)
+
 # A recorded issue number at the end of a heading: `(#123)` as written
 # back by populate, or `(#123, closed)` as rendered by update.
 HEADING_NUMBER_SUFFIX_RE = re.compile(r"\s*\(#(\d+)(?:,\s*[a-z]+)?\)$")
@@ -210,12 +240,18 @@ def parse_repository(text: str) -> Optional[str]:
 
 
 def parse_project_plan(text: str) -> ProjectPlan:
-    """Parse the structured project plan markdown into a ProjectPlan."""
+    """Parse the structured project plan markdown into a ProjectPlan.
+
+    The interiors of `unplanned` regions are masked first: they mirror
+    GitHub-owned content, never plan structure (see
+    mask_unplanned_regions).
+    """
+    masked = mask_unplanned_regions(text)
     return ProjectPlan(
-        repository=parse_repository(text),
-        milestones=tuple(_parse_milestones(text)),
-        labels=tuple(_parse_labels(text)),
-        issues=tuple(_parse_issues(text)),
+        repository=parse_repository(masked),
+        milestones=tuple(_parse_milestones(masked)),
+        labels=tuple(_parse_labels(masked)),
+        issues=tuple(_parse_issues(masked)),
     )
 
 
@@ -692,10 +728,17 @@ def lint_plan(text: str) -> tuple[Problem, ...]:
             f"cannot tell which milestone issues M{n}-* belong to",
         ))
 
+    # The heading scans run on MASKED text: an `unplanned` region
+    # mirrors organic GitHub titles, and one that merely starts with
+    # "Issue" (or resembles a milestone heading) is mirror content, not
+    # a malformed plan heading.  Offsets survive masking, so line
+    # numbers still point at the real file.
+    scannable = mask_unplanned_regions(text)
+
     # A heading that says `### Issue` but does not parse is worse than a
     # missing one: populate never pushes it, and the next update erases
     # it when the surrounding generated region is rebuilt from GitHub.
-    for m in re.finditer(r"^### Issue\b[^\n]*$", text, re.MULTILINE):
+    for m in re.finditer(r"^### Issue\b[^\n]*$", scannable, re.MULTILINE):
         if not ISSUE_HEADING_RE.match(m.group(0)):
             problems.append(Problem(
                 "error",
@@ -711,7 +754,7 @@ def lint_plan(text: str) -> tuple[Problem, ...]:
     # headings are scanned — prose headings like `### Milestone
     # dependencies` are legitimate (the exemplar plans use them).
     milestone_heading_re = re.compile(r"^### Milestone (\d+) — (.+)$")
-    for m in re.finditer(r"^### Milestone\s+\d+\b[^\n]*$", text, re.MULTILINE):
+    for m in re.finditer(r"^### Milestone\s+\d+\b[^\n]*$", scannable, re.MULTILINE):
         if not milestone_heading_re.match(m.group(0)):
             problems.append(Problem(
                 "error",
@@ -1032,6 +1075,7 @@ def to_issues(data: list[dict]) -> list[Issue]:
                 gh_title = gh_ms.get("title", "")
                 tm = re.match(r"^(\d+)\.", gh_title)
                 inferred = int(tm.group(1)) if tm else (ms_idx if ms_idx is not None else 0)
+            gh_milestone = item.get("milestone") or {}
             out.append(Issue(
                 id=issue_id,
                 title=title,
@@ -1043,6 +1087,7 @@ def to_issues(data: list[dict]) -> list[Issue]:
                 assignees=tuple(
                     a["login"] for a in item.get("assignees", []) or []
                 ),
+                milestone_title=gh_milestone.get("title") or None,
             ))
         return out
     return _parse_json(stdout, "issues").map(to_issues)

scripts/gh_project_update.py

@@ -102,6 +102,34 @@ def issue_sort_key(issue: Issue) -> tuple[int, int, str]:
     return (int(m.group(1)), int(m.group(2)), m.group(3) or "") if m else (10**9, 10**9, "")
 
 
+def group_unplanned(
+    issues: list[Issue],
+) -> tuple[tuple[str | None, tuple[Issue, ...]], ...]:
+    """Group organically-filed issues (id="") for the `unplanned` region.
+
+    Grouped by the GitHub milestone title verbatim (organic issues have
+    no plan index to key on); groups ordered by title with the
+    no-milestone group last, issues within a group by ascending number.
+    Everything is deterministic, so repeated renders are `--check`
+    stable.
+    """
+    by_title: dict[str | None, list[Issue]] = {}
+    for issue in issues:
+        if issue.id:
+            continue
+        by_title.setdefault(issue.milestone_title, []).append(issue)
+    ordered_titles = sorted(
+        by_title, key=lambda t: (t is None, t or "")
+    )
+    return tuple(
+        (
+            title,
+            tuple(sorted(by_title[title], key=lambda i: i.gh_number or 0)),
+        )
+        for title in ordered_titles
+    )
+
+
 # ── Region rendering ─────────────────────────────────────────────────────────
 
 _ID_PREFIX_RE = re.compile(r"^\[M\d+-\d+[a-z]?\]\s+(.+)$")
@@ -127,10 +155,14 @@ def strip_id_prefix(title: str) -> str:
 
 
 def render_issue(issue: Issue) -> str:
-    title = strip_id_prefix(issue.title)
+    # Titles and label names are dynamic GitHub content too: defang
+    # marker look-alikes exactly as in bodies.  (GitHub label names
+    # allow arbitrary punctuation, and backticks are no protection —
+    # the marker regexes match raw text, not rendered markdown.)
+    title = neutralize_markers(strip_id_prefix(issue.title))
     state_suffix = ", closed" if issue.state == "closed" else ""
     ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)"
-    labels = ", ".join(f"`{lbl}`" for lbl in issue.labels)
+    labels = neutralize_markers(", ".join(f"`{lbl}`" for lbl in issue.labels))
     # Assignees are GitHub-owned state (like open/closed); render them so
     # the file answers "who is on this?" without a browser.  Both
     # metadata lines use the canonical `**Word:**` spelling that the
@@ -152,7 +184,60 @@ def render_issue(issue: Issue) -> str:
     )
 
 
-def render_region(region_id: str, issues_by_ms: dict[int, list[Issue]]) -> str:
+def render_unplanned_issue(issue: Issue) -> str:
+    """The planning block shape, minus the id — organic issues have none.
+
+    The `[MN-k]` prefix stays the stable-identifier contract of the
+    milestone regions; here the GitHub number is the only handle, so it
+    alone appears in the heading.
+    """
+    state_suffix = ", closed" if issue.state == "closed" else ""
+    ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)"
+    labels = neutralize_markers(", ".join(f"`{lbl}`" for lbl in issue.labels))
+    assignees = (
+        f"**Assignees:** {', '.join('@' + a for a in issue.assignees)}\n\n"
+        if issue.assignees else ""
+    )
+    body = issue.body.strip() if issue.body else "_(no description on GitHub)_"
+    body = neutralize_markers(body)
+    title = neutralize_markers(issue.title)
+    return (
+        f"### {title} ({ref})\n"
+        f"\n"
+        f"**Labels:** {labels}\n"
+        f"\n"
+        f"{assignees}"
+        f"{body}\n"
+    )
+
+
+def render_unplanned_region(
+    unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...],
+) -> str:
+    if not unplanned:
+        return (
+            "\n_(no organically-filed issues — every issue on GitHub "
+            "carries a `[MN-k]` planning prefix)_\n"
+        )
+    parts: list[str] = []
+    for title, issues in unplanned:
+        # Milestone titles are dynamic GitHub content rendered into the
+        # file: defang marker look-alikes like titles/bodies/labels.
+        header = (
+            neutralize_markers(title) if title is not None else "(no milestone)"
+        )
+        blocks = "\n---\n\n".join(render_unplanned_issue(i) for i in issues)
+        parts.append(f"#### {header}\n\n{blocks}")
+    return "\n" + "\n\n".join(parts) + "\n"
+
+
+def render_region(
+    region_id: str,
+    issues_by_ms: dict[int, list[Issue]],
+    unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...],
+) -> str:
+    if region_id == "unplanned":
+        return render_unplanned_region(unplanned)
     m = re.match(r"^milestone-(\d+)$", region_id)
     if m is None:
         return f"\n<!-- region '{region_id}' has no rendering rule in gh_project_update.py -->\n"
@@ -164,13 +249,17 @@ def render_region(region_id: str, issues_by_ms: dict[int, list[Issue]]) -> str:
     return "\n" + "\n---\n\n".join(blocks) + "\n"
 
 
-def assemble_file(parsed: ParsedFile, issues_by_ms: dict[int, list[Issue]]) -> str:
+def assemble_file(
+    parsed: ParsedFile,
+    issues_by_ms: dict[int, list[Issue]],
+    unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...] = (),
+) -> str:
     parts: list[str] = []
     for i, manual in enumerate(parsed.manuals[:-1]):
         rid = parsed.ids[i]
         parts.append(manual)
         parts.append(f"<!-- BEGIN GENERATED: {rid} -->\n")
-        parts.append(render_region(rid, issues_by_ms))
+        parts.append(render_region(rid, issues_by_ms, unplanned))
         parts.append(f"<!-- END GENERATED: {rid} -->")
     parts.append(parsed.manuals[-1])
     return "".join(parts)
@@ -196,12 +285,31 @@ def warn_orphan_milestones(parsed: ParsedFile, issues_by_ms: dict[int, list[Issu
             )
 
 
+def warn_unrendered_unplanned(
+    parsed: ParsedFile,
+    unplanned: tuple[tuple[str | None, tuple[Issue, ...]], ...],
+) -> None:
+    """Stderr-warn when organically-filed issues exist but the file has
+    no `unplanned` region to show them in — the invisibility this
+    region exists to end (issue #3)."""
+    if not unplanned or "unplanned" in parsed.ids:
+        return
+    count = sum(len(issues) for _, issues in unplanned)
+    print(
+        f"warning: {count} organically-filed issue(s) (no [MN-k] prefix) "
+        f"have no <!-- BEGIN GENERATED: unplanned --> region",
+        file=sys.stderr,
+    )
+
+
 # ── Main ─────────────────────────────────────────────────────────────────────
 
 def _render(parsed: ParsedFile, issues: list[Issue]) -> str:
     issues_by_ms = group_by_milestone(issues)
+    unplanned = group_unplanned(issues)
     warn_orphan_milestones(parsed, issues_by_ms)
-    return assemble_file(parsed, issues_by_ms)
+    warn_unrendered_unplanned(parsed, unplanned)
+    return assemble_file(parsed, issues_by_ms, unplanned)
 
 
 def run(markdown: Path, repo: str | None, env_prefix: bool) -> Result[str, PipelineError]:

scripts/tests/test_gh_project_lib.py

@@ -121,6 +121,13 @@
 
 <!-- END GENERATED: milestone-2 -->
 
+## Unplanned work
+
+<!-- BEGIN GENERATED: unplanned -->
+
+_(no organically-filed issues — every issue on GitHub carries a `[MN-k]` planning prefix)_
+<!-- END GENERATED: unplanned -->
+
 ## How to use
 
 Trailing prose that must not leak into any issue body.
@@ -251,6 +258,40 @@ def test_milestone_idx_from_id(self) -> None:
         )
 
 
+class UnplannedMasking(unittest.TestCase):
+    """The unplanned region mirrors GitHub content; nothing inside it
+    may be read back as plan structure."""
+
+    MIRRORED = PLAN.replace(
+        "_(no organically-filed issues — every issue on GitHub carries "
+        "a `[MN-k]` planning prefix)_",
+        "#### 1. Core\n\n"
+        "### Issue M9-9: phantom planning heading (#77)\n\n"
+        "**Labels:** `not-a-plan-label`\n\n"
+        "Organic body.\n",
+    )
+
+    def test_mask_preserves_markers_and_offsets(self) -> None:
+        masked = lib.mask_unplanned_regions(self.MIRRORED)
+        self.assertEqual(len(masked), len(self.MIRRORED))
+        self.assertEqual(masked.count("\n"), self.MIRRORED.count("\n"))
+        self.assertIn("<!-- BEGIN GENERATED: unplanned -->", masked)
+        self.assertIn("<!-- END GENERATED: unplanned -->", masked)
+        self.assertNotIn("phantom planning heading", masked)
+
+    def test_no_phantom_plan_issue(self) -> None:
+        plan = parse_project_plan(self.MIRRORED)
+        self.assertEqual(
+            {i.id for i in plan.issues}, {"M1-1", "M1-2", "M2-1", "M2-1a"}
+        )
+
+    def test_milestone_regions_still_parse(self) -> None:
+        # Masking is surgical: planned regions keep their content.
+        plan = parse_project_plan(self.MIRRORED)
+        m11 = next(i for i in plan.issues if i.id == "M1-1")
+        self.assertIn("Body of M1-1.", m11.body)
+
+
 class RecordIssueNumber(unittest.TestCase):
     def test_adds_suffix(self) -> None:
         text, found = record_issue_number(PLAN, "M1-1", 7)
@@ -290,8 +331,8 @@ def test_id_match_is_exact_not_prefix(self) -> None:
 class ParseFileRegions(unittest.TestCase):
     def test_round_trip_structure(self) -> None:
         parsed = parse_file(PLAN).unwrap()
-        self.assertEqual(parsed.ids, ("milestone-1", "milestone-2"))
-        self.assertEqual(len(parsed.manuals), 3)
+        self.assertEqual(parsed.ids, ("milestone-1", "milestone-2", "unplanned"))
+        self.assertEqual(len(parsed.manuals), 4)
         self.assertTrue(parsed.manuals[-1].startswith("\n\n## How to use"))
 
     def test_unterminated_begin(self) -> None: