From 833c6dfe7ac189b990e1f2f2f465f67c142c47c3 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Thu, 13 Aug 2026 21:50:14 -0600 Subject: [PATCH 1/9] Vendor _utils (pipeline_types, file_ops, command_runner) with the #293 fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This repository is now the upstream of the shared functional-Python core; the three consumer projects re-vendor from here. The copies were identical across agda-native-air, agda-algebras, and williamdemeo.github.io as of 2026-08-13; this vendoring applies the five utils fixes from ualib/agda-algebras#293, each pinned by a regression test that fails against the unfixed copies (verified: 8 failures/errors before the fixes, 21/21 green after): 1. Result.unwrap() keys solely off _is_ok, so Ok(None) — routine, e.g. from write_text — unwraps to None instead of raising. unwrap_or() had the same defect class and is fixed alongside. 2. Result.map() lets exceptions from the mapped function propagate instead of synthesizing Result.err(Exception) in a Result[_, PipelineError]. 3. file_ops imports ProcessingStage, which calculate_file_metadata annotates; typing.get_type_hints raised NameError before. 4. command_runner stream_output=True drains stdout and stderr in concurrent threads; draining stdout to completion first deadlocked (deterministically, ~64KiB stderr) — reproduced before the fix. 5. stdout_file opens in a mode matching text=. (The TypeError claimed in #293 did not reproduce on Python 3.12 — the child writes to the descriptor directly — so the test pins the now-explicit contract.) text_processing is deliberately not vendored: it is doc-pipeline specific and the gh_project scripts do not use it; it remains owned by the downstream projects. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_utils/__init__.py | 69 ++++ scripts/_utils/command_runner.py | 140 ++++++++ scripts/_utils/file_ops.py | 188 ++++++++++ scripts/_utils/pipeline_types.py | 500 +++++++++++++++++++++++++++ scripts/tests/test_command_runner.py | 110 ++++++ scripts/tests/test_file_ops.py | 66 ++++ scripts/tests/test_pipeline_types.py | 99 ++++++ 7 files changed, 1172 insertions(+) create mode 100644 scripts/_utils/__init__.py create mode 100644 scripts/_utils/command_runner.py create mode 100644 scripts/_utils/file_ops.py create mode 100644 scripts/_utils/pipeline_types.py create mode 100644 scripts/tests/test_command_runner.py create mode 100644 scripts/tests/test_file_ops.py create mode 100644 scripts/tests/test_pipeline_types.py diff --git a/scripts/_utils/__init__.py b/scripts/_utils/__init__.py new file mode 100644 index 0000000..176e5ca --- /dev/null +++ b/scripts/_utils/__init__.py @@ -0,0 +1,69 @@ +""" +File: scripts/_utils/__init__.py + +Description: + Utility modules for functional programming and pipeline operations. + + This package provides immutable data structures, functional error + handling, and pure transformations. + +Provenance: + This repository is the upstream of this package. Originally adapted + from formalverification/agda-native-air (SHA 664b919); identical + across agda-native-air, agda-algebras, and williamdemeo.github.io as + of 2026-08-13, plus the ualib/agda-algebras#293 fixes. Downstream + projects re-vendor from here — see scripts/VERSION and the README's + upgrade section. +""" + +from .pipeline_types import ( + # Functional error handling + Result, + PipelineError, + ErrorType, + + # Pipeline state management + PipelineState, + PipelineStatistics, + ProcessedFile, + FileMetadata, + ProcessingStage, + + # Command execution + CommandResult, + + # Function types + ProcessingFunction, + FileTransformer, + PipelineStage, + + # Utility functions + sequence_results, + collect_errors, +) + +__all__ = [ + # Error handling + "Result", + "PipelineError", + "ErrorType", + + # State management + "PipelineState", + "PipelineStatistics", + "ProcessedFile", + "FileMetadata", + "ProcessingStage", + + # Command execution + "CommandResult", + + # Function types + "ProcessingFunction", + "FileTransformer", + "PipelineStage", + + # Utilities + "sequence_results", + "collect_errors", +] diff --git a/scripts/_utils/command_runner.py b/scripts/_utils/command_runner.py new file mode 100644 index 0000000..9c886ae --- /dev/null +++ b/scripts/_utils/command_runner.py @@ -0,0 +1,140 @@ +""" +File: scripts/_utils/command_runner.py + +Description: + Functional command execution utilities. + +Provenance: + This repository is the upstream of this module. Originally adapted + from formalverification/agda-native-air (SHA 664b919); identical + across agda-native-air, agda-algebras, and williamdemeo.github.io as + of 2026-08-13. This copy additionally fixes ualib/agda-algebras#293 + items 4 (stream_output deadlock: both pipes are now drained + concurrently) and 5 (stdout_file honors the text flag). Downstream + projects re-vendor from here. +""" +from __future__ import annotations +import subprocess +import logging +import threading +from pathlib import Path +from typing import List, Optional + +from .pipeline_types import Result, PipelineError, ErrorType + +def run_command( + command: List[str], + cwd: Optional[Path] = None, + capture_output: bool = False, + text: bool = False, + stdout_file: Optional[Path] = None, + stream_output: bool = False # --- NEW: Parameter to enable live streaming +) -> Result[subprocess.CompletedProcess, PipelineError]: + """ + Runs a shell command and returns a Result object. + Can either capture output or stream it live to the logger. + """ + command_str = ' '.join(map(str, command)) + logging.debug(f"Running: {command_str}") + + try: + # --- Live streaming output --- + if stream_output: + # Use Popen for live output streaming + process = subprocess.Popen( + [str(arg) for arg in command], + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding='utf-8' + ) + + # Drain both pipes concurrently. Reading stdout to + # completion before touching stderr deadlocks as soon as the + # child fills the OS stderr pipe buffer (~64 KiB) while its + # stdout is still open: the child blocks on write, this + # process on read, and neither can proceed. + def _drain(pipe, emit) -> None: + for line in iter(pipe.readline, ''): + emit(line.strip()) + + readers = [ + threading.Thread( + target=_drain, + args=(process.stdout, lambda s: logging.info(f" > {s}")), + ), + threading.Thread( + target=_drain, + args=(process.stderr, lambda s: logging.warning(f" ! {s}")), + ), + ] + for reader in readers: + reader.start() + for reader in readers: + reader.join() + + # Wait for the process to complete and get the return code + process.wait() + if process.returncode != 0: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Streamed command failed with exit code {process.returncode}", + context={"command": command_str, "return_code": process.returncode} + )) + # Return a CompletedProcess-like object on success + return Result.ok(subprocess.CompletedProcess( + args=command, returncode=0, stdout="", stderr="" + )) + + # --- Original logic for capturing output --- + else: + stdout_target = subprocess.PIPE if capture_output or not stdout_file else None + if stdout_file: + stdout_file.parent.mkdir(parents=True, exist_ok=True) + # The file's mode must match `text`; the child writes to + # the descriptor directly, so a text-mode wrapper on a + # text=False run misdescribes what lands in the file. + stdout_target = open( + stdout_file, + "w" if text else "wb", + encoding="utf-8" if text else None, + ) + + process = subprocess.run( + [str(arg) for arg in command], + cwd=cwd, stdout=stdout_target, stderr=subprocess.PIPE, + text=text, check=False, encoding='utf-8' if text else None + ) + + if process.stderr: + logging.debug(f"Stderr for '{command_str}':\n{process.stderr}") + if process.returncode != 0: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Command failed with exit code {process.returncode}", + context={ + "command": command_str, + "return_code": process.returncode, + "stderr": process.stderr.strip() if process.stderr else "N/A", + "cwd": str(cwd) if cwd else "N/A" + } + )) + return Result.ok(process) + + except FileNotFoundError as e: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Command not found: {command[0]}", + cause=e + )) + except Exception as e: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"An unexpected error occurred running command: {e}", + cause=e + )) + finally: + if not stream_output and stdout_file and 'stdout_target' in locals() and stdout_target: + if not isinstance(stdout_target, int): # Not a PIPE + stdout_target.close() diff --git a/scripts/_utils/file_ops.py b/scripts/_utils/file_ops.py new file mode 100644 index 0000000..6853ab3 --- /dev/null +++ b/scripts/_utils/file_ops.py @@ -0,0 +1,188 @@ +""" +File: scripts/_utils/file_ops.py + +Description: + Provides pure, functional wrappers for file system operations. + +Provenance: + This repository is the upstream of this module. Originally adapted + from formalverification/agda-native-air (SHA 664b919); identical + across agda-native-air, agda-algebras, and williamdemeo.github.io as + of 2026-08-13. This copy additionally fixes ualib/agda-algebras#293 + item 3 (calculate_file_metadata's ProcessingStage annotation was + never imported). Downstream projects re-vendor from here. +""" +from __future__ import annotations +import json +import shutil +from pathlib import Path +from typing import List, Dict + +from .pipeline_types import ( + Result, PipelineError, ErrorType, FileMetadata, ProcessingStage, +) + +def ensure_dir_exists(path: Path) -> Result[Path, PipelineError]: + """Pure function: Creates a directory if it doesn't exist.""" + try: + path.mkdir(parents=True, exist_ok=True) + return Result.ok(path) + except Exception as e: + return Result.err(PipelineError( + error_type=ErrorType.PERMISSION_DENIED, + message=f"Failed to create directory: {path}", + cause=e + )) + +def clean_dir(path: Path) -> Result[Path, PipelineError]: + """Pure function: Removes and recreates a directory.""" + try: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + return Result.ok(path) + except Exception as e: + return Result.err(PipelineError( + error_type=ErrorType.PERMISSION_DENIED, + message=f"Failed to clean directory: {path}", + cause=e + )) + +def rm_file(path: Path) -> Result[None, PipelineError]: + """Removes a single file if it exists.""" + try: + if path.exists() and path.is_file(): + path.unlink() + return Result.ok(None) + except Exception as e: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Failed to remove file: {path}", + cause=e + )) + +def rm_dir(path: Path) -> Result[None, PipelineError]: + """Removes a directory if it exists.""" + try: + if path.exists(): + shutil.rmtree(path) + return Result.ok(None) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to remove directory: {path}", cause=e)) + +def rm_artifact(path: Path) -> Result[Path, PipelineError]: + """Removes a file or directory path if it exists.""" + try: + if path.is_dir(): + shutil.rmtree(path) + elif path.is_file(): + path.unlink() + return Result.ok(path) + except Exception as e: + return Result.err(PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Failed to remove artifact: {path}", + cause=e + )) + +def cp_file(source: Path, target: Path) -> Result[Path, PipelineError]: + """Copies a single file.""" + try: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return Result.ok(target) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to copy file: {source} -> {target}", cause=e)) + +def cp_dir(source: Path, target: Path) -> Result[Path, PipelineError]: + """Copies an entire directory tree.""" + try: + if not source.is_dir(): + return Result.err(PipelineError(ErrorType.FILE_NOT_FOUND, f"Source directory not found: {source}")) + shutil.copytree(source, target, dirs_exist_ok=True) + return Result.ok(target) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to copy directory: {source} -> {target}", cause=e)) + +def ls_dir(path: Path, pattern: str = '*') -> Result[List[Path], PipelineError]: + """ + Lists all files in a directory matching a glob pattern, returning a Result. + Args: + path: The directory to list. + pattern: A glob pattern (e.g., '*.agda', '**/*.md'). + """ + try: + if not path.is_dir(): + return Result.err(PipelineError(ErrorType.FILE_NOT_FOUND, f"Directory not found: {path}")) + + files = sorted([item for item in path.glob(pattern) if item.is_file()]) + return Result.ok(files) + + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to list directory: {path}", cause=e)) + +def ls_dir_special(path: Path, recursive: bool = False) -> Result[List[Path], PipelineError]: + """Lists all files in a directory, returning a Result.""" + try: + if not path.is_dir(): + return Result.err(PipelineError(ErrorType.FILE_NOT_FOUND, f"Directory not found: {path}")) + pattern = '**/*' if recursive else '*' + files = [item for item in path.glob(pattern) if item.is_file()] + return Result.ok(files) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to list directory: {path}", cause=e)) + +def read_text(path: Path) -> Result[str, PipelineError]: + """Reads a text file, returning a Result.""" + try: + return Result.ok(path.read_text('utf-8')) + except FileNotFoundError: + return Result.err(PipelineError(ErrorType.FILE_NOT_FOUND, f"File not found: {path}")) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to read file: {path}", cause=e)) + +def write_text(path: Path, content: str) -> Result[None, PipelineError]: + """Writes text to a file, returning a Result.""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, 'utf-8') + return Result.ok(None) + except Exception as e: + return Result.err(PipelineError(ErrorType.COMMAND_FAILED, f"Failed to write file: {path}", cause=e)) + +def load_json(path: Path) -> Result[Dict, PipelineError]: + """Loads a JSON file, handling potential parsing errors.""" + def parse(content: str) -> Result[Dict, PipelineError]: + try: + return Result.ok(json.loads(content)) + except json.JSONDecodeError as e: + return Result.err(PipelineError(ErrorType.PARSING_ERROR, f"Failed to parse JSON: {path}", cause=e)) + + return read_text(path).and_then(parse) + +def write_json(path: Path, data: Dict) -> Result[None, PipelineError]: + """Serializes a dictionary to a JSON file.""" + try: + content = json.dumps(data, indent=2) + return write_text(path, content) + except TypeError as e: + return Result.err(PipelineError(ErrorType.PARSING_ERROR, f"Failed to serialize JSON for: {path}", cause=e)) + +# ============================================================================= +# Utility Functions +# ============================================================================= + +def calculate_file_metadata(file_path: Path, stage: ProcessingStage) -> FileMetadata: + """ + Calculates metadata for a processed file. This is a special case where a + try-except is pragmatic because a failure to get a file's size is not + a pipeline-halting error; we can simply default to 0. + """ + try: + file_size = file_path.stat().st_size + except FileNotFoundError: + file_size = 0 + return FileMetadata( + relative_path=Path(file_path.name), stage=stage, + processing_time=0.0, file_size=file_size, checksum="" + ) diff --git a/scripts/_utils/pipeline_types.py b/scripts/_utils/pipeline_types.py new file mode 100644 index 0000000..e0d5250 --- /dev/null +++ b/scripts/_utils/pipeline_types.py @@ -0,0 +1,500 @@ +""" +File: scripts/_utils/pipeline_types.py + +Description: Functional programming primitives and immutable data structures. + + This module provides functional error handling (Result), structured + errors (PipelineError), and immutable pipeline-state types. + +Design Principles: + - Immutable data structures everywhere + - Functional error handling with Result types + - Algebraic data types for domain modeling + - Pure transformations between states + - Composable operations + +Provenance: + This repository is the upstream of this module. Originally adapted + from formalverification/agda-native-air (SHA 664b919); identical + across agda-native-air, agda-algebras, and williamdemeo.github.io as + of 2026-08-13. This copy additionally fixes ualib/agda-algebras#293 + items 1 and 2 (unwrap/unwrap_or on Ok(None); map() no longer swallows + exceptions). Downstream projects re-vendor from here — see + scripts/VERSION and the README's upgrade section. +""" + +from __future__ import annotations +from dataclasses import dataclass, field +from pathlib import Path +from typing import ( + TypeVar, Generic, Optional, Union, Dict, List, Tuple, + NamedTuple, Callable, Any, Protocol, runtime_checkable +) +from enum import Enum, auto +import json + +# ============================================================================= +# Generic Type Variables +# ============================================================================= + +T = TypeVar('T') # Success type +E = TypeVar('E') # Error type +A = TypeVar('A') # Input type +B = TypeVar('B') # Output type + +# ============================================================================= +# Functional Error Handling: Result Type +# ============================================================================= + +@dataclass(frozen=True) +class Result(Generic[T, E]): + """ + Functional error handling type (like Haskell's Either or Rust's Result). + + Represents either a successful computation (Ok) or a failure (Err). + Supports functional composition and eliminates exception handling. + + Mathematical properties: + - Functor: supports map operations + - Monad: supports flatMap (bind) operations + - Pure: no side effects in constructors + """ + + _is_ok: bool + _value: Optional[T] = None + _error: Optional[E] = None + + @classmethod + def ok(cls, value: T) -> Result[T, E]: + """Construct a successful result.""" + return cls(_is_ok=True, _value=value, _error=None) + + @classmethod + def err(cls, error: E) -> Result[T, E]: + """Construct a failed result.""" + if error is None: + raise ValueError("Cannot create an Err result with a None value. Errors must be non-null.") + return cls(_is_ok=False, _value=None, _error=error) + + @property + def is_ok(self) -> bool: + """True if this is a successful result.""" + return self._is_ok + + @property + def is_err(self) -> bool: + """True if this is a failed result.""" + return not self._is_ok + + def unwrap(self) -> T: + """Extract the success value (throws if error). + + Keys solely off _is_ok: Result.ok(None) is a legal and common + construction (write_text returns Result[None, _]), so the + contained value must never influence the ok/err decision. + """ + if self._is_ok: + return self._value # type: ignore[return-value] + raise ValueError(f"Called unwrap() on error result: {self._error}") + + def unwrap_or(self, default: T) -> T: + """Extract the success value or return default. + + Ok(None) returns None — the contained value — not the default. + """ + return self._value if self._is_ok else default # type: ignore[return-value] + + def unwrap_err(self) -> E: + """Extract the error value (throws if success).""" + if self._is_ok: + raise ValueError(f"Called unwrap_err() on a success result: {self._value}") + if self._error is None: + # This handles the specific invalid state that caused the crash. + raise ValueError("Called unwrap_err() on an invalid Err result that contains no error value.") + return self._error + + # Functor operations + def map(self, f: Callable[[T], B]) -> Result[B, E]: + """Apply function to success value, preserve error. + + Exceptions raised by f propagate: a raising f is a bug in the + caller, not an expected failure, and catching it here would + smuggle a bare Exception into the declared error type E. + """ + if self._is_ok: + return Result.ok(f(self._value)) + return Result.err(self._error) + + def map_err(self, f: Callable[[E], B]) -> Result[T, B]: + """Apply function to error value, preserve success.""" + if self._is_ok: + return Result.ok(self._value) # type: ignore + return Result.err(f(self._error)) # type: ignore + + # Monad operations + def flat_map(self, f: Callable[[T], Result[B, E]]) -> Result[B, E]: + """Monadic bind - chain computations that might fail.""" + # The only check should be for success, not the contained value. + if self._is_ok: + return f(self._value) # It's valid to pass None to the next function. + return Result.err(self._error) + + def and_then(self, f: Callable[[T], Result[B, E]]) -> Result[B, E]: + """Alias for flat_map (more readable).""" + return self.flat_map(f) + + +# ============================================================================= +# Pipeline Error Types +# ============================================================================= + +class ErrorType(Enum): + """Categorization of pipeline errors for structured error handling.""" + + FILE_NOT_FOUND = auto() + PERMISSION_DENIED = auto() + INVALID_CONFIG = auto() + COMMAND_FAILED = auto() + PARSING_ERROR = auto() + AGDA_ERROR = auto() + PANDOC_ERROR = auto() + BIBTEX_ERROR = auto() + VALIDATION_ERROR = auto() + + +@dataclass(frozen=True) +class PipelineError: + """ + Immutable error representation with context and categorization. + + Provides structured error information for functional error handling. + """ + + error_type: ErrorType + message: str + context: Dict[str, Any] = field(default_factory=dict) + cause: Optional[Exception] = None + + def with_context(self, **kwargs) -> PipelineError: + """Add context to error (returns new error).""" + new_context = {**self.context, **kwargs} + return PipelineError( + error_type=self.error_type, + message=self.message, + context=new_context, + cause=self.cause + ) + + def __str__(self) -> str: + parts = [f"{self.error_type.name}: {self.message}"] + if self.context: + context_str = ", ".join(f"{k}={v}" for k, v in self.context.items()) + parts.append(f"Context: {context_str}") + if self.cause: + parts.append(f"Caused by: {self.cause}") + return " | ".join(parts) + + +# ============================================================================= +# File Processing Types +# ============================================================================= + +class ProcessingStage(Enum): + """Stages in the document processing pipeline.""" + + SOURCE = auto() # Original .lagda/.agda files + SNAPSHOT = auto() # Copied to build directory + PREPROCESSED = auto() # After preprocess.py + PANDOC_CONVERTED = auto() # After pandoc + lua filter + POSTPROCESSED = auto() # After postprocess.py + AGDA_HTML = auto() # After agda --html + DEPLOYED = auto() # Copied to final site + + +@dataclass(frozen=True) +class FileMetadata: + """Immutable metadata about a processed file.""" + + relative_path: Path + stage: ProcessingStage + processing_time: float + file_size: int + checksum: str = "" + + def advanced_to(self, new_stage: ProcessingStage) -> FileMetadata: + """Create new metadata for advanced processing stage.""" + return FileMetadata( + relative_path=self.relative_path, + stage=new_stage, + processing_time=self.processing_time, + file_size=self.file_size, + checksum=self.checksum + ) + + +@dataclass(frozen=True) +class ProcessedFile: + """ + Immutable representation of a file at a specific processing stage. + + Mathematical properties: + - Immutable: never changes after creation + - Pure: no side effects in operations + - Composable: can be transformed through pipeline stages + """ + + source_path: Path + current_path: Path + metadata: FileMetadata + content_hash: str = "" + + def at_stage(self, stage: ProcessingStage, new_path: Path) -> ProcessedFile: + """Create new ProcessedFile representing advancement to next stage.""" + return ProcessedFile( + source_path=self.source_path, + current_path=new_path, + metadata=self.metadata.advanced_to(stage), + content_hash=self.content_hash + ) + + @property + def is_agda_file(self) -> bool: + """True if this is an Agda source file.""" + return self.source_path.suffix in {'.agda', '.lagda', '.lagda.md'} + + @property + def module_name(self) -> str: + """Derived module name from file path.""" + # Convert path/to/Module.lagda -> Module + return self.source_path.stem.split('.')[0] + + +# ============================================================================= +# Pipeline State Types +# ============================================================================= + +@dataclass(frozen=True) +class PipelineStatistics: + """Immutable statistics about pipeline execution.""" + + files_processed: int = 0 + total_processing_time: float = 0.0 + errors_encountered: int = 0 + warnings_generated: int = 0 + agda_files_converted: int = 0 + citations_processed: int = 0 + + def add_file(self, processing_time: float = 0.0) -> PipelineStatistics: + """Add a processed file to statistics.""" + return PipelineStatistics( + files_processed=self.files_processed + 1, + total_processing_time=self.total_processing_time + processing_time, + errors_encountered=self.errors_encountered, + warnings_generated=self.warnings_generated, + agda_files_converted=self.agda_files_converted, + citations_processed=self.citations_processed + ) + + def add_error(self) -> PipelineStatistics: + """Record an error.""" + return PipelineStatistics( + files_processed=self.files_processed, + total_processing_time=self.total_processing_time, + errors_encountered=self.errors_encountered + 1, + warnings_generated=self.warnings_generated, + agda_files_converted=self.agda_files_converted, + citations_processed=self.citations_processed + ) + + +@dataclass(frozen=True) +class PipelineState: + """ + Immutable state of the entire pipeline at a point in time. + + The state that gets transformed through the pipeline. + Each processing stage takes a PipelineState and returns + a new PipelineState. + """ + + processed_files: Tuple[ProcessedFile, ...] = field(default_factory=tuple) + artifacts: Dict[str, Path] = field(default_factory=dict) + statistics: PipelineStatistics = field(default_factory=PipelineStatistics) + stage: ProcessingStage = ProcessingStage.SOURCE + + def add_file(self, file: ProcessedFile) -> PipelineState: + """Add a processed file (returns new state).""" + return PipelineState( + processed_files=self.processed_files + (file,), + artifacts=self.artifacts.copy(), + statistics=self.statistics.add_file(), + stage=self.stage + ) + + def add_artifact(self, name: str, path: Path) -> PipelineState: + """Add an artifact (returns new state).""" + new_artifacts = {**self.artifacts, name: path} + return PipelineState( + processed_files=self.processed_files, + artifacts=new_artifacts, + statistics=self.statistics, + stage=self.stage + ) + + def advance_to_stage(self, new_stage: ProcessingStage) -> PipelineState: + """Advance pipeline to next stage (returns new state).""" + return PipelineState( + processed_files=self.processed_files, + artifacts=self.artifacts, + statistics=self.statistics, + stage=new_stage + ) + + def files_at_stage(self, stage: ProcessingStage) -> List[ProcessedFile]: + """Get all files currently at a specific stage.""" + return [f for f in self.processed_files if f.metadata.stage == stage] + + +# ============================================================================= +# Command Execution Types +# ============================================================================= + +@dataclass(frozen=True) +class CommandResult: + """Immutable result of executing a shell command.""" + + command: Tuple[str, ...] + return_code: int + stdout: str = "" + stderr: str = "" + execution_time: float = 0.0 + + @property + def success(self) -> bool: + """True if command succeeded.""" + return self.return_code == 0 + + @property + def failed(self) -> bool: + """True if command failed.""" + return self.return_code != 0 + + def to_result(self) -> Result[CommandResult, PipelineError]: + """Convert to Result type for functional error handling.""" + if self.success: + return Result.ok(self) + else: + error = PipelineError( + error_type=ErrorType.COMMAND_FAILED, + message=f"Command failed with exit code {self.return_code}", + context={ + "command": " ".join(self.command), + "stderr": self.stderr, + "stdout": self.stdout + } + ) + return Result.err(error) + + +# ============================================================================= +# Processing Function Types +# ============================================================================= + +# Type aliases for pipeline functions +ProcessingFunction = Callable[[PipelineState], Result[PipelineState, PipelineError]] +FileTransformer = Callable[[ProcessedFile], Result[ProcessedFile, PipelineError]] + +@runtime_checkable +class PipelineStage(Protocol): + """Protocol for pipeline stage implementations.""" + + def process(self, state: PipelineState) -> Result[PipelineState, PipelineError]: + """Process pipeline state and return new state or error.""" + ... + + @property + def stage_name(self) -> str: + """Human-readable name of this stage.""" + ... + + + +# ============================================================================= +# Utility Functions +# ============================================================================= + +def sequence_results(results: List[Result[T, E]]) -> Result[List[T], E]: + """ + Convert List[Result[T, E]] to Result[List[T], E]. + + If all results are Ok, return Ok with list of values. + If any result is Err, return the first error. + + (Like Haskell's `sequence` for the Result monad.) + """ + values = [] + for result in results: + if result.is_err: + return Result.err(result.unwrap_err()) + values.append(result.unwrap()) + return Result.ok(values) + + +def collect_errors(results: List[Result[T, E]]) -> Tuple[List[T], List[E]]: + """ + Separate successful results from errors. + + Returns tuple of (successes, errors). + """ + successes = [] + errors = [] + + for result in results: + if result.is_ok: + successes.append(result.unwrap()) + else: + errors.append(result.unwrap_err()) + + return successes, errors + + +# ============================================================================= +# Example Usage and Testing +# ============================================================================= + +if __name__ == "__main__": + # Example: Functional error handling + def divide(x: float, y: float) -> Result[float, str]: + if y == 0: + return Result.err("Division by zero") + return Result.ok(x / y) + + # Chain operations functionally + result = (divide(10, 2) + .map(lambda x: x * 3) # 5 * 3 = 15 + .and_then(lambda x: divide(x, 3))) # 15 / 3 = 5 + + print(f"Result: {result.unwrap()}") # Should print 5.0 + + # Example: Pipeline state + initial_state = PipelineState() + file_metadata = FileMetadata( + relative_path=Path("Module.lagda"), + stage=ProcessingStage.SOURCE, + processing_time=0.1, + file_size=1024 + ) + processed_file = ProcessedFile( + source_path=Path("src/Module.lagda"), + current_path=Path("_build/Module.lagda.md"), + metadata=file_metadata + ) + + new_state = (initial_state + .add_file(processed_file) + .add_artifact("macros", Path("macros.json")) + .advance_to_stage(ProcessingStage.PREPROCESSED)) + + print(f"Files processed: {new_state.statistics.files_processed}") + print(f"Current stage: {new_state.stage}") + print(f"✅ Type system working correctly!") diff --git a/scripts/tests/test_command_runner.py b/scripts/tests/test_command_runner.py new file mode 100644 index 0000000..baee904 --- /dev/null +++ b/scripts/tests/test_command_runner.py @@ -0,0 +1,110 @@ +""" +File: scripts/tests/test_command_runner.py + +Description: + Regression tests for _utils.command_runner, pinning the fixes for + ualib/agda-algebras#293 items 4 and 5: + + 4. stream_output=True must not deadlock when the child fills the + stderr pipe buffer while stdout is still open. The unfixed code + drained stdout to completion before touching stderr; a child + writing more than the OS pipe buffer (~64 KiB) to stderr first + blocked forever. The test runs exactly that child under a + watchdog timeout. + 5. stdout_file must honor `text`. (The TypeError claimed in #293 + did not reproduce on Python 3.11+ — the child writes to the file + descriptor directly, bypassing the parent's text-mode wrapper — + so these tests pin the contract the fix makes explicit: the mode + of the opened file matches `text`, and the child's output arrives + intact either way.) +""" +from __future__ import annotations + +import sys +import tempfile +import threading +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _utils.command_runner import run_command # noqa: E402 + +# Comfortably beyond any OS pipe buffer (Linux default: 64 KiB). +STDERR_FLOOD = 200_000 + +# The deadlock child: floods stderr BEFORE writing anything to stdout, +# so a reader that starts with stdout blocks while the child blocks on +# the full stderr pipe. +DEADLOCK_CHILD = ( + "import sys; " + f"sys.stderr.write('x' * {STDERR_FLOOD}); sys.stderr.flush(); " + "sys.stdout.write('done'); sys.stdout.flush()" +) + + +class StreamOutputDoesNotDeadlock(unittest.TestCase): + """#293 item 4: concurrent draining of stdout and stderr.""" + + def _run_with_watchdog(self, argv: list[str], timeout: float = 30.0): + holder: dict = {} + + def call() -> None: + holder["result"] = run_command(argv, stream_output=True) + + t = threading.Thread(target=call, daemon=True) + t.start() + t.join(timeout=timeout) + if t.is_alive(): + self.fail( + f"run_command(stream_output=True) did not finish within " + f"{timeout}s — the stdout-before-stderr deadlock is back" + ) + return holder["result"] + + def test_stderr_flood_completes(self) -> None: + result = self._run_with_watchdog( + [sys.executable, "-c", DEADLOCK_CHILD] + ) + self.assertTrue(result.is_ok) + self.assertEqual(result.unwrap().returncode, 0) + + def test_nonzero_exit_is_err(self) -> None: + result = self._run_with_watchdog( + [sys.executable, "-c", "import sys; sys.exit(3)"] + ) + self.assertTrue(result.is_err) + + +class StdoutFileHonorsTextFlag(unittest.TestCase): + """#293 item 5: the file mode matches `text` in both directions.""" + + def test_text_false_writes_bytes(self) -> None: + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "out.bin" + result = run_command( + [sys.executable, "-c", "print('abc')"], + stdout_file=out, + text=False, + ) + self.assertTrue(result.is_ok) + self.assertEqual(out.read_bytes(), b"abc\n") + + def test_text_true_writes_text(self) -> None: + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "out.txt" + result = run_command( + [sys.executable, "-c", "print('abc')"], + stdout_file=out, + text=True, + ) + self.assertTrue(result.is_ok) + self.assertEqual(out.read_text(encoding="utf-8"), "abc\n") + + def test_missing_command_is_err(self) -> None: + result = run_command(["definitely-not-a-command-xyzzy"]) + self.assertTrue(result.is_err) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_file_ops.py b/scripts/tests/test_file_ops.py new file mode 100644 index 0000000..8bd80a5 --- /dev/null +++ b/scripts/tests/test_file_ops.py @@ -0,0 +1,66 @@ +""" +File: scripts/tests/test_file_ops.py + +Description: + Regression tests for _utils.file_ops, pinning the fix for + ualib/agda-algebras#293 item 3: calculate_file_metadata annotates + `stage: ProcessingStage` — the name must actually be imported. + Postponed evaluation (`from __future__ import annotations`) hides the + missing import at call time, so the test forces annotation resolution + with typing.get_type_hints, which raises NameError on the unfixed copy. +""" +from __future__ import annotations + +import sys +import tempfile +import typing +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _utils import file_ops # noqa: E402 +from _utils.pipeline_types import FileMetadata, ProcessingStage # noqa: E402 + + +class CalculateFileMetadataAnnotations(unittest.TestCase): + """#293 item 3: the ProcessingStage annotation must resolve.""" + + def test_annotations_resolve(self) -> None: + hints = typing.get_type_hints(file_ops.calculate_file_metadata) + self.assertIs(hints["stage"], ProcessingStage) + + def test_metadata_for_existing_file(self) -> None: + with tempfile.NamedTemporaryFile(suffix=".md") as f: + f.write(b"12345") + f.flush() + meta = file_ops.calculate_file_metadata( + Path(f.name), ProcessingStage.SOURCE + ) + self.assertIsInstance(meta, FileMetadata) + self.assertEqual(meta.stage, ProcessingStage.SOURCE) + self.assertEqual(meta.file_size, 5) + + def test_metadata_for_missing_file_defaults_size_zero(self) -> None: + meta = file_ops.calculate_file_metadata( + Path("/nonexistent/never/here.md"), ProcessingStage.SOURCE + ) + self.assertEqual(meta.file_size, 0) + + +class WriteTextReturnsOkNone(unittest.TestCase): + """write_text returns Result[None, _]; combined with the unwrap() + fix this round-trips without special-casing.""" + + def test_write_then_read(self) -> None: + with tempfile.TemporaryDirectory() as d: + target = Path(d) / "x.txt" + w = file_ops.write_text(target, "hello") + self.assertTrue(w.is_ok) + self.assertIsNone(w.unwrap()) + r = file_ops.read_text(target) + self.assertEqual(r.unwrap(), "hello") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_pipeline_types.py b/scripts/tests/test_pipeline_types.py new file mode 100644 index 0000000..8294be7 --- /dev/null +++ b/scripts/tests/test_pipeline_types.py @@ -0,0 +1,99 @@ +""" +File: scripts/tests/test_pipeline_types.py + +Description: + Regression tests for _utils.pipeline_types, pinning the fixes for + ualib/agda-algebras#293 items 1 and 2: + + 1. Result.unwrap() must accept Ok(None) — success extraction keys + solely off _is_ok, never off the contained value. (write_text + returns Result[None, _], so Ok(None) is routine, not exotic.) + 2. Result.map() must not catch exceptions raised by the mapped + function and repackage them as wrong-typed errors; a raising f + is a bug in the caller and propagates. +""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _utils.pipeline_types import ( # noqa: E402 + ErrorType, + PipelineError, + Result, + collect_errors, + sequence_results, +) + + +class UnwrapAcceptsOkNone(unittest.TestCase): + """#293 item 1: Ok(None) is a legal, common success value.""" + + def test_unwrap_ok_none_returns_none(self) -> None: + self.assertIsNone(Result.ok(None).unwrap()) + + def test_unwrap_or_ok_none_returns_none_not_default(self) -> None: + # Same defect class as unwrap(): the eliminator must key off + # _is_ok, not off the contained value. + self.assertIsNone(Result.ok(None).unwrap_or("default")) + + def test_unwrap_err_still_raises(self) -> None: + err = PipelineError(ErrorType.VALIDATION_ERROR, "boom") + with self.assertRaises(ValueError): + Result.err(err).unwrap() + + def test_unwrap_or_err_returns_default(self) -> None: + err = PipelineError(ErrorType.VALIDATION_ERROR, "boom") + self.assertEqual(Result.err(err).unwrap_or("default"), "default") + + def test_ok_none_is_ok(self) -> None: + self.assertTrue(Result.ok(None).is_ok) + self.assertFalse(Result.ok(None).is_err) + + +class MapDoesNotSwallowExceptions(unittest.TestCase): + """#293 item 2: map() previously synthesized Result.err(Exception), + violating the declared Result[_, PipelineError] error type.""" + + def test_exception_in_mapped_function_propagates(self) -> None: + with self.assertRaises(ZeroDivisionError): + Result.ok(1).map(lambda x: x / 0) + + def test_map_ok(self) -> None: + self.assertEqual(Result.ok(2).map(lambda x: x * 3).unwrap(), 6) + + def test_map_err_preserves_error(self) -> None: + err = PipelineError(ErrorType.VALIDATION_ERROR, "boom") + r = Result.err(err).map(lambda x: x * 3) + self.assertTrue(r.is_err) + self.assertIs(r.unwrap_err(), err) + + def test_map_ok_none_maps_the_none(self) -> None: + # map over Ok(None) hands None to f (it is the success value). + self.assertEqual(Result.ok(None).map(lambda v: v is None).unwrap(), True) + + def test_and_then_chains(self) -> None: + r = Result.ok(2).and_then(lambda x: Result.ok(x + 1)) + self.assertEqual(r.unwrap(), 3) + + +class TraversalsHandleOkNone(unittest.TestCase): + """The list traversals must not misclassify Ok(None) either.""" + + def test_sequence_results_with_ok_none(self) -> None: + r = sequence_results([Result.ok(None), Result.ok(1)]) + self.assertTrue(r.is_ok) + self.assertEqual(r.unwrap(), [None, 1]) + + def test_collect_errors_with_ok_none(self) -> None: + err = PipelineError(ErrorType.VALIDATION_ERROR, "boom") + oks, errs = collect_errors([Result.ok(None), Result.err(err)]) + self.assertEqual(oks, [None]) + self.assertEqual(errs, [err]) + + +if __name__ == "__main__": + unittest.main() From d964ea0bc564c8cd5bef7a6117ee406663d0e453 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Thu, 13 Aug 2026 21:56:01 -0600 Subject: [PATCH 2/9] Add _gh_project_lib: shared model, parsing, pure planners, snapshot client Ported from williamdemeo.github.io's copy (the newest lineage; verified by diff to strictly subsume agda-algebras'), restructured as functional-core / imperative-shell: - GitHubClient.snapshot() fetches labels, milestones, and issues once per run (three calls total); the write side is now plain creations with no hidden refetch. This is the structural fix for #293 item 6 (idempotency checking was O(n^2): one ListIssues per create call). - plan_labels/plan_milestones/plan_issues are pure snapshot x plan reconcilers. plan_labels never overwrites and reports near-collisions (case/whitespace, e.g. 'era: conway' vs 'era:conway') instead of creating a parallel scheme. - plan-file parsing moves here from populate so lint can share it. Issue bodies now end at the next issue heading, top-level section, or END GENERATED marker (the ancestors stopped only at '## Summary:', sweeping trailing prose into the final issue's body). - Issue headings may carry a recorded '(#N)' suffix (populate's crash-safe write-back); it parses as state, not title. record_issue_number() is the targeted, line-oriented writer. - parse_file() additionally rejects stray END and nested BEGIN markers. - Issue records carry assignees (GitHub-owned state the update script will render). Also: close the drained pipes in command_runner's streaming path and silence the drain logger inside the flood test. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 853 +++++++++++++++++++++++++++ scripts/_utils/command_runner.py | 2 + scripts/tests/test_command_runner.py | 11 + scripts/tests/test_gh_project_lib.py | 379 ++++++++++++ 4 files changed, 1245 insertions(+) create mode 100644 scripts/_gh_project_lib.py create mode 100644 scripts/tests/test_gh_project_lib.py diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py new file mode 100644 index 0000000..14b2b86 --- /dev/null +++ b/scripts/_gh_project_lib.py @@ -0,0 +1,853 @@ +""" +File: scripts/_gh_project_lib.py + +Description: + + Shared data model, plan-file parsing, pure planning logic, and GitHub + I/O layer for the gh_project_{populate, update, lint} scripts. + + The design is functional-core / imperative-shell: + + - Parsing (`parse_project_plan`, `parse_file`) and planning + (`plan_labels`, `plan_milestones`, `plan_issues`) are pure functions + over strings and immutable records. They never touch the network, + so `--dry-run` and `make lint` are exact by construction: a dry run + prints the same plan a real run executes. + - `GitHubClient` wraps the `gh` CLI in Result-returning methods. Its + read side produces one `RepoSnapshot` per run; its write side + performs plain creations with no hidden existence checks. The + existence checks live in the pure planners, which receive the + snapshot — this is what makes populate O(n) in API calls instead of + re-fetching the world per creation (ualib/agda-algebras#293 item 6). + + All GitHubClient methods return Result[T, PipelineError] for + functional composition by callers. + +Provenance: + + Ported from williamdemeo.github.io (the newest lineage, 2026-07-31), + which extracted it from agda-algebras' gh_project_populate.py during + agda-algebras#289; originally adapted from + formalverification/agda-native-air. This repository is now the + upstream; downstream projects re-vendor from here. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Optional + +from _utils import Result, PipelineError, ErrorType +from _utils.command_runner import run_command + + +# ── Issue identifier conventions ──────────────────────────────────────────── + +# Issue titles are prefixed with `[MN-k]` (or `[MN-ka]` for sub-tickets +# of a fan-out parent) for cross-script identification and within-milestone +# ordering. Examples: `[M0-1] ...`, `[M1-12] ...`, `[M2-7a] ...`. +ISSUE_ID_PATTERN = re.compile(r"^\[(M(\d+)-(\d+)([a-z]?))\]\s+(.+)$") + +# Labels of the form `milestone-N-*` map issues to milestone N. Populate +# emits these labels; update uses them to infer milestone membership when +# pulling state back from GitHub. +MILESTONE_LABEL_PATTERN = re.compile(r"^milestone-(\d+)-") + + +def parse_issue_id(title: str) -> Optional[tuple[str, int, int, str, str]]: + """Parse a `[MN-k] Title` prefix into (id, milestone, ordinal, suffix, rest). + + Returns None if the title does not begin with a recognized prefix. + Used by populate's idempotency guard and by update's title-based + ordering. The suffix is "" for plain `[MN-k]` ids and a single + lowercase letter for fan-out sub-tickets like `[M2-7a]`, `[M2-7b]`. + """ + m = ISSUE_ID_PATTERN.match(title) + if not m: + return None + issue_id = m.group(1) + ms = int(m.group(2)) + ord_ = int(m.group(3)) + suffix = m.group(4) or "" + rest = m.group(5) + return issue_id, ms, ord_, suffix, rest + + +def milestone_index_from_labels(labels: list[str]) -> Optional[int]: + """Infer milestone number from a `milestone-N-*` label, or None if absent.""" + for label in labels: + m = MILESTONE_LABEL_PATTERN.match(label) + if m: + return int(m.group(1)) + return None + + +def issue_id_gte(a: str, b: str) -> bool: + """Compare issue IDs by (milestone, ordinal, suffix). + + `M1-3 >= M1-2` is True; `M0-9 >= M1-1` is False; `M2-7a >= M2-7` is + True; `M2-7b >= M2-7a` is True. Used by populate's --start-from + logic and by update to order issues within a milestone. + """ + def parse(s: str) -> tuple[int, int, str]: + m = re.match(r"M(\d+)-(\d+)([a-z]?)", s) + return (int(m.group(1)), int(m.group(2)), m.group(3) or "") if m else (10**9, 10**9, "") + return parse(a) >= parse(b) + + +# ── Data classes ──────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class Label: + """A GitHub label with its display color and optional description.""" + name: str + color: str # six hex digits, no leading `#` + description: str = "" + + +@dataclass(frozen=True) +class Milestone: + """A GitHub milestone; `gh_number` is set after creation or lookup.""" + number: int # plan index: the leading integer of + # the GitHub title ("1. Foundation" → 1) + title: str # GitHub display title, e.g. "1. Foundation" + description: str + gh_number: Optional[int] = None # GitHub-assigned milestone number + + def with_gh_number(self, n: int) -> Milestone: + """Return a copy with `gh_number` populated (immutable update).""" + return Milestone(self.number, self.title, self.description, n) + + +@dataclass(frozen=True) +class Issue: + """A GitHub issue carrying the full information needed by all scripts.""" + id: str # e.g. "M0-1"; parsed from title prefix + title: str # full title including `[MN-k]` prefix + body: str + labels: tuple[str, ...] = () + milestone_idx: int = 0 # plan index, matching the + # `milestone-N-*` label group; 0 + # means "unclassified" + state: str = "open" # "open" or "closed" + gh_number: Optional[int] = None # GitHub-assigned issue number; + # from GitHub on the read side, + # from a recorded `(#N)` heading + # suffix on the parse side + assignees: tuple[str, ...] = () # GitHub logins; read side only + + +@dataclass(frozen=True) +class RepoSnapshot: + """Live GitHub state, fetched once per run and passed to the planners. + + One snapshot per populate run replaces the per-creation re-fetch that + made idempotency checking O(n²) in API calls (#293 item 6). + """ + labels: tuple[Label, ...] + milestones: tuple[Milestone, ...] + issues: tuple[Issue, ...] + + +# ── Plan-file parsing ─────────────────────────────────────────────────────── +# +# The plan file (docs/GITHUB_PROJECT.md) is parsed with line-anchored +# regular expressions rather than a markdown library: the format is a +# deliberate, narrow convention (see the shipped example), and the +# scripts must run on a bare Python 3.11+ with no third-party imports. + +# A `**Repository**: `owner/name`` line names the target repo so that +# `--repo` need not be repeated on every invocation. CLI wins over file. +REPOSITORY_LINE_RE = re.compile( + r"^\*\*Repository\*\*\s*:\s*`?([\w.-]+/[\w.-]+)`?\s*$", + re.MULTILINE, +) + +ISSUE_HEADING_RE = re.compile(r"^### Issue (M\d+-\d+[a-z]?): (.+?)[ \t]*$", re.MULTILINE) + +# A recorded issue number at the end of a heading: `(#123)` as written +# back by populate, or `(#123, closed)` as rendered by update. +HEADING_NUMBER_SUFFIX_RE = re.compile(r"\s*\(#(\d+)(?:,\s*[a-z]+)?\)$") + +# An issue body ends at the next issue heading, the next top-level `## ` +# section, or an END GENERATED marker — whichever comes first. (The +# ancestor scripts stopped only at `## Summary:` and at leaked +# `## Milestone N` headers, which swept trailing prose into the final +# issue's body.) +BODY_END_RE = re.compile(r"^(?:## (?!#)|[ \t]*$\n?", + re.MULTILINE, +) + + +@dataclass(frozen=True) +class ProjectPlan: + """Everything populate needs, parsed from one plan file.""" + repository: Optional[str] + milestones: tuple[Milestone, ...] + labels: tuple[Label, ...] + issues: tuple[Issue, ...] + + +def parse_repository(text: str) -> Optional[str]: + """Extract `owner/name` from the `**Repository**:` header line, if any.""" + m = REPOSITORY_LINE_RE.search(text) + return m.group(1) if m else None + + +def parse_project_plan(text: str) -> ProjectPlan: + """Parse the structured project plan markdown into a ProjectPlan.""" + return ProjectPlan( + repository=parse_repository(text), + milestones=tuple(_parse_milestones(text)), + labels=tuple(_parse_labels(text)), + issues=tuple(_parse_issues(text)), + ) + + +def _parse_milestones(text: str) -> list[Milestone]: + """Extract milestones from the ## Milestones section.""" + milestones: list[Milestone] = [] + + ms_match = re.search(r"^## Milestones\s*$", text, re.MULTILINE) + if not ms_match: + return milestones + + ms_text = text[ms_match.end():] + # Stop at the next top-level section. + next_section = re.search(r"^## (?!#)", ms_text, re.MULTILINE) + if next_section: + ms_text = ms_text[:next_section.start()] + + # Each milestone starts with `### Milestone N — Title`. + ms_blocks = re.split(r"^### Milestone (\d+) — (.+)$", ms_text, flags=re.MULTILINE) + + # ms_blocks[0] is preamble, then groups of (number, title, body). + i = 1 + while i + 2 < len(ms_blocks): + num = int(ms_blocks[i]) + title = ms_blocks[i + 1].strip() + body = ms_blocks[i + 2].strip() + + desc_match = re.search( + r"\*\*Description[:\*]*\*?\s*\n(.*?)(?=\*\*Exit criterion|---|\Z)", + body, re.DOTALL + ) + desc = desc_match.group(1).strip() if desc_match else "" + + exit_match = re.search( + r"\*\*Exit criterion[:\*]*\*?\s*(.+?)(?=\n---|\n\n###|\Z)", + body, re.DOTALL + ) + exit_crit = exit_match.group(1).strip() if exit_match else "" + + full_desc = desc + if exit_crit: + full_desc += f"\n\n**Exit criterion:** {exit_crit}" + + milestones.append(Milestone( + number=num, + title=f"{num}. {title}", + description=full_desc, + )) + i += 3 + + return milestones + + +def _parse_labels(text: str) -> list[Label]: + """Collect labels from the markdown. + + Preferred: an explicit ``## Labels`` section with entries of the form + + - `label-name` (COLORHEX) — Description + + (The separator may be em-dash, en-dash, hyphen, or colon.) + + Fallback: if no ``## Labels`` section is present, collect all label + names referenced in issues' ``**Labels:**`` lines and give them a + neutral gray color with no description. + + If neither yields anything, return no labels at all. (The ancestor + scripts shipped a hard-coded default set here; a template must not + impose labels the plan never mentioned.) + """ + explicit = _parse_explicit_labels(text) + if explicit: + return explicit + return _collect_labels_from_issues(text) + + +def _parse_explicit_labels(text: str) -> list[Label]: + """Parse the ``## Labels`` section into Label records, if it exists.""" + m = re.search(r"^## Labels\s*$", text, re.MULTILINE) + if not m: + return [] + start = m.end() + next_section = re.search(r"^## (?!#)", text[start:], re.MULTILINE) + end = start + next_section.start() if next_section else len(text) + section = text[start:end] + pattern = re.compile( + r"^\s*[-*]\s+`([^`]+)`\s*\(([0-9a-fA-F]{6})\)\s*(?:—|–|-|:)\s*(.+?)\s*$", + re.MULTILINE, + ) + return [ + Label(mm.group(1).strip(), mm.group(2).strip().lower(), mm.group(3).strip()) + for mm in pattern.finditer(section) + ] + + +def _collect_labels_from_issues(text: str) -> list[Label]: + """Collect unique label names referenced in issues' ``**Labels:**`` lines.""" + seen: dict[str, Label] = {} + for m in re.finditer(r"\*\*Labels:\*\*\s*(.+)", text): + for raw in m.group(1).split(","): + name = raw.strip().strip("`").strip() + if name and name not in seen: + seen[name] = Label(name, "cccccc", "") + return list(seen.values()) + + +def _parse_issues(text: str) -> list[Issue]: + """Extract issues from `### Issue MN-k: Title` headings anywhere in the file.""" + issues: list[Issue] = [] + matches = list(ISSUE_HEADING_RE.finditer(text)) + + for idx, match in enumerate(matches): + issue_id = match.group(1) + heading_title = match.group(2).strip() + + ms_idx = int(re.match(r"M(\d+)", issue_id).group(1)) + + # Body runs from after the heading to the next issue heading, + # top-level section, or END GENERATED marker. + start = match.end() + end = matches[idx + 1].start() if idx + 1 < len(matches) else len(text) + stop = BODY_END_RE.search(text, start, end) + if stop: + end = stop.start() + + raw_body = text[start:end].strip() + + # Remove trailing --- separators. + raw_body = re.sub(r"\n---+\s*$", "", raw_body).strip() + + # Extract labels from the **Labels:** line. + labels_match = re.search(r"\*\*Labels:\*\*\s*(.+)", raw_body) + labels: list[str] = [] + if labels_match: + label_str = labels_match.group(1).strip() + labels = [lbl.strip().strip("`") for lbl in label_str.split(",")] + + # Build the issue body: drop the Labels and Milestone metadata lines. + body_lines = [ + line for line in raw_body.split("\n") + if not line.strip().startswith("**Labels:**") + and not line.strip().startswith("**Milestone:**") + ] + body = "\n".join(body_lines).strip() + body = re.sub(r"^\n+", "", body) + + # Drop any region marker swept in from the surrounding file. + body = GENERATED_MARKER_LINE.sub("", body).strip() + + # A recorded `(#N)` / `(#N, closed)` suffix carries the GitHub + # number written back by a previous populate run (or rendered by + # update); it is state, not title text. + gh_number: Optional[int] = None + suffix = HEADING_NUMBER_SUFFIX_RE.search(heading_title) + if suffix: + gh_number = int(suffix.group(1)) + heading_title = heading_title[:suffix.start()].strip() + + # Titles carry the `[MN-k]` prefix on GitHub: it is the only handle + # gh_project_update.py has for identifying planning issues, and the + # key populate's idempotency guard compares on. Strip any prefix + # already written into the heading and re-apply it, rather than + # passing an existing one through: ISSUE_ID_PATTERN wants `\s+` + # after the bracket, so a heading reading `[M1-1]Title` would + # survive a startswith() check and still be unparseable. + bare = re.sub(rf"^\[{re.escape(issue_id)}\]\s*", "", heading_title).strip() + gh_title = f"[{issue_id}] {bare}" + + issues.append(Issue( + id=issue_id, + title=gh_title, + body=body, + labels=tuple(labels), + milestone_idx=ms_idx, + gh_number=gh_number, + )) + + return issues + + +# ── Issue-number write-back ───────────────────────────────────────────────── + +def record_issue_number(text: str, issue_id: str, gh_number: int) -> tuple[str, bool]: + """Write `(#N)` onto the `### Issue {issue_id}:` heading line. + + Returns (new_text, found). Line-oriented and targeted, so nothing + else in the file can be disturbed; idempotent (an existing suffix is + replaced). Populate persists the file after each creation using + this, which is what makes an interrupted run rerun-safe: every + number already handed out by GitHub is on disk before the next + mutation is attempted. + """ + heading_re = re.compile( + rf"^(### Issue {re.escape(issue_id)}: .+?)(\s*\(#\d+(?:,\s*[a-z]+)?\))?[ \t]*$" + ) + out: list[str] = [] + found = False + for line in text.splitlines(keepends=True): + if not found: + m = heading_re.match(line.rstrip("\n")) + if m: + newline = "\n" if line.endswith("\n") else "" + out.append(f"{m.group(1)} (#{gh_number}){newline}") + found = True + continue + out.append(line) + return "".join(out), found + + +# ── Generated-region parsing (used by update and lint) ───────────────────── + +BEGIN_RE = re.compile(r"") +END_RE = re.compile(r"") + + +@dataclass(frozen=True) +class ParsedFile: + """A file decomposed into manual prose and named generated regions. + + Invariant: len(manuals) == len(ids) + 1. The original file is + reconstructible (modulo regeneration of the marked regions) as + + manuals[0] + ... + + manuals[1] + ... + + ... + manuals[-1] + + The content between markers in the original file is discarded by + parse; update rebuilds it from GitHub state. + """ + manuals: tuple[str, ...] + ids: tuple[str, ...] + + +def parse_file(content: str) -> Result[ParsedFile, PipelineError]: + """Split file content on BEGIN/END GENERATED marker pairs.""" + manuals: list[str] = [] + ids: list[str] = [] + pos = 0 + while True: + begin = BEGIN_RE.search(content, pos) + if begin is None: + # A stray END with no matching BEGIN would otherwise vanish + # silently into a manual segment. + stray = END_RE.search(content, pos) + if stray is not None: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=( + f" " + f"has no matching BEGIN" + ), + context={"offset": stray.start()}, + )) + manuals.append(content[pos:]) + return Result.ok(ParsedFile(tuple(manuals), tuple(ids))) + marker_id = begin.group(1) + manuals.append(content[pos:begin.start()]) + end = END_RE.search(content, begin.end()) + if end is None: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"unterminated ", + context={"offset": begin.start()}, + )) + if end.group(1) != marker_id: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=( + f"mismatched markers: BEGIN GENERATED: {marker_id} " + f"closed by END GENERATED: {end.group(1)}" + ), + context={"begin_offset": begin.start(), "end_offset": end.start()}, + )) + inner_begin = BEGIN_RE.search(content, begin.end(), end.start()) + if inner_begin is not None: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=( + f"nested " + f"inside region '{marker_id}'" + ), + context={"offset": inner_begin.start()}, + )) + ids.append(marker_id) + pos = end.end() + + +# ── Pure planning (the idempotency and reconciliation logic) ──────────────── + +def normalize_label_name(name: str) -> str: + """Casefold and strip all whitespace, for near-collision detection. + + `era: conway`, `era:conway`, and `Era:Conway` all normalize to + `era:conway`: creating one of them alongside another silently forks + the label scheme (a real incident), so the planner reports the + collision instead of creating. + """ + return "".join(name.casefold().split()) + + +@dataclass(frozen=True) +class LabelPlan: + """Per-label decisions computed from plan × snapshot.""" + to_create: tuple[Label, ...] + existing: tuple[tuple[Label, Label], ...] # (desired, live) exact-name matches + collisions: tuple[tuple[Label, Label], ...] # (desired, live) near-collisions + + +def plan_labels(desired: tuple[Label, ...], existing: tuple[Label, ...]) -> LabelPlan: + """Reconcile desired labels against the repo's live labels. + + Never plans an overwrite: an exact-name match is reported as existing + even when color or description differ (the report surfaces the + difference; resolving it is a human decision), and a near-collision + (same name modulo case/whitespace) is reported and skipped rather + than silently creating a parallel scheme. + """ + by_exact = {e.name: e for e in existing} + by_norm: dict[str, Label] = {} + for e in existing: + by_norm.setdefault(normalize_label_name(e.name), e) + + to_create: list[Label] = [] + exists: list[tuple[Label, Label]] = [] + collisions: list[tuple[Label, Label]] = [] + for d in desired: + if d.name in by_exact: + exists.append((d, by_exact[d.name])) + elif normalize_label_name(d.name) in by_norm: + collisions.append((d, by_norm[normalize_label_name(d.name)])) + else: + to_create.append(d) + return LabelPlan(tuple(to_create), tuple(exists), tuple(collisions)) + + +@dataclass(frozen=True) +class MilestonePlan: + """Per-milestone decisions computed from plan × snapshot.""" + to_create: tuple[Milestone, ...] + existing: tuple[Milestone, ...] # gh_number populated from the snapshot + + +def plan_milestones( + desired: tuple[Milestone, ...], existing: tuple[Milestone, ...] +) -> MilestonePlan: + """Match desired milestones against live ones by exact title.""" + by_title = {e.title: e for e in existing} + to_create: list[Milestone] = [] + exists: list[Milestone] = [] + for d in desired: + live = by_title.get(d.title) + if live is not None: + exists.append(d.with_gh_number(live.gh_number or 0)) + else: + to_create.append(d) + return MilestonePlan(tuple(to_create), tuple(exists)) + + +@dataclass(frozen=True) +class IssuePlan: + """Per-issue decisions computed from plan × snapshot.""" + to_create: tuple[Issue, ...] + existing: tuple[tuple[Issue, Issue], ...] # (planned, live) + + +def plan_issues(desired: tuple[Issue, ...], existing: tuple[Issue, ...]) -> IssuePlan: + """Match desired issues against live ones. + + The `[MN-k]` title prefix is the primary join key (robust to + post-creation edits of the descriptive title). A `(#N)` number + recorded in the plan file is the secondary key: it keeps the guard + intact even if someone strips the prefix from the title on GitHub. + """ + by_id = {e.id: e for e in existing} + by_number = {e.gh_number: e for e in existing if e.gh_number is not None} + + to_create: list[Issue] = [] + exists: list[tuple[Issue, Issue]] = [] + for d in desired: + live = by_id.get(d.id) + if live is None and d.gh_number is not None: + live = by_number.get(d.gh_number) + if live is not None: + exists.append((d, live)) + else: + to_create.append(d) + return IssuePlan(tuple(to_create), tuple(exists)) + + +# ── GitHub client ──────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class GitHubClient: + """Thin Result-returning wrapper around the `gh` CLI. + + Read side: `snapshot()` pulls labels, milestones, and issues in three + calls total; the pure planners consume it. Write side: plain + creations with no existence checks and no dry-run logic — whether to + call at all is the caller's decision, made from the plan. (Dry-run + faithfulness is structural: the printed plan IS the execution list.) + + The `env_prefix` flag prepends `env -u GH_TOKEN -u GITHUB_TOKEN` to + every invocation, working around a long-standing `gh` quirk where + these environment variables silently override the keychain-stored + auth token. Default-on is the safer choice locally; pass + `env_prefix=False` wherever authentication comes *through* those + variables — GitHub Actions included: stripping them there leaves gh + with no credentials at all. + """ + repo: str + env_prefix: bool = True + + # — internal command construction —————————————————————————————————— + + def _gh(self, *args: str) -> list[str]: + prefix = ["env", "-u", "GH_TOKEN", "-u", "GITHUB_TOKEN"] if self.env_prefix else [] + return prefix + ["gh", *args] + + def _run(self, *args: str) -> Result[str, PipelineError]: + """Run `gh ...` and return stdout as text on success.""" + return run_command(self._gh(*args), capture_output=True, text=True).map( + lambda proc: proc.stdout + ) + + # — read side ———————————————————————————————————————————————————————— + + def snapshot(self) -> Result[RepoSnapshot, PipelineError]: + """Fetch the full live state once: labels, milestones, issues.""" + return self.list_labels().and_then( + lambda labels: self.list_milestones().and_then( + lambda milestones: self.list_issues().map( + lambda issues: RepoSnapshot( + tuple(labels), tuple(milestones), tuple(issues) + ) + ) + ) + ) + + def list_labels(self) -> Result[list[Label], PipelineError]: + """Pull every label currently defined on the repository.""" + cmd = self._run( + "label", "list", "--repo", self.repo, "--limit", "200", + "--json", "name,color,description", + ) + return cmd.and_then(_parse_labels_json) + + def list_milestones(self) -> Result[list[Milestone], PipelineError]: + """Pull every milestone (open and closed) from the repository. + + The GitHub CLI lacks first-class milestone subcommands, so we go + through the REST API. Returned `Milestone.number` is the leading + integer of the title (the plan index); `gh_number` carries + GitHub's own number for cross-reference. + """ + cmd = self._run( + "api", f"repos/{self.repo}/milestones", + "--paginate", + "-X", "GET", + "-F", "state=all", + "-F", "per_page=100", + ) + return cmd.and_then(_parse_milestones_json) + + def list_issues(self) -> Result[list[Issue], PipelineError]: + """Pull every issue (open and closed) from the repository. + + Returned `Issue.milestone_idx` is inferred from the issue's + `milestone-N-*` label rather than from GitHub's milestone + assignment; the label is the source of truth in this convention + because it survives milestone renames. + """ + cmd = self._run( + "issue", "list", + "--repo", self.repo, + "--state", "all", + "--limit", "1000", + "--json", "number,title,body,labels,milestone,state,assignees", + ) + return cmd.and_then(_parse_issues_json) + + # — write side ——————————————————————————————————————————————————————— + + def create_label(self, label: Label) -> Result[None, PipelineError]: + """Create `label` on GitHub. Never `--force`: overwriting an + existing label's color or description is a human decision.""" + return self._run( + "label", "create", label.name, + "--repo", self.repo, + "--color", label.color, + "--description", label.description, + ).map(lambda _: None) + + def create_milestone(self, ms: Milestone) -> Result[Milestone, PipelineError]: + """Create `ms` on GitHub; returns it with gh_number populated.""" + return self._run( + "api", f"repos/{self.repo}/milestones", + "-X", "POST", + "-f", f"title={ms.title}", + "-f", f"description={ms.description}", + ).and_then(_parse_milestone_create_response).map(ms.with_gh_number) + + def create_issue( + self, issue: Issue, milestone_title: Optional[str] + ) -> Result[int, PipelineError]: + """Create `issue` on GitHub; returns the new issue number.""" + args = [ + "issue", "create", + "--repo", self.repo, + "--title", issue.title, + "--body", issue.body, + ] + for label in issue.labels: + args += ["--label", label] + if milestone_title is not None: + args += ["--milestone", milestone_title] + return self._run(*args).and_then(_parse_issue_create_response) + + +# ── JSON parsers ───────────────────────────────────────────────────────────── +# +# Module-level pure functions to keep GitHubClient methods short and to +# make each parser independently testable. Each returns Result so the +# pipeline composes through `and_then` without intermediate try/except. + +def _parse_labels_json(stdout: str) -> Result[list[Label], PipelineError]: + return _parse_json(stdout, "labels").map( + lambda data: [ + Label( + name=item["name"], + color=item.get("color", "cccccc"), + description=item.get("description", "") or "", + ) + for item in data + ] + ) + + +def _parse_milestones_json(stdout: str) -> Result[list[Milestone], PipelineError]: + """Convert the GitHub REST `repos/.../milestones` response into Milestones. + + The tooling titles each milestone `N. Title`; the leading integer is + the plan index. Milestones whose title does not match this + convention are skipped (with no error): they were not created by this + tooling and do not belong to the plan. + """ + title_pattern = re.compile(r"^(\d+)\.\s+(.+)$") + + def to_milestones(data: list[dict]) -> list[Milestone]: + out: list[Milestone] = [] + for item in data: + m = title_pattern.match(item.get("title", "")) + if not m: + continue + out.append(Milestone( + number=int(m.group(1)), + title=item["title"], + description=item.get("description") or "", + gh_number=item["number"], + )) + return sorted(out, key=lambda x: x.number) + return _parse_json(stdout, "milestones").map(to_milestones) + + +def _parse_issues_json(stdout: str) -> Result[list[Issue], PipelineError]: + def to_issues(data: list[dict]) -> list[Issue]: + out: list[Issue] = [] + for item in data: + title = item.get("title", "") + parsed = parse_issue_id(title) + if parsed is None: + # Not a planning issue (e.g. an ad-hoc bug report); skip + # silently — update does not list these. + continue + issue_id, ms_idx, _ord, _suffix, _rest = parsed + label_names = tuple(lbl["name"] for lbl in item.get("labels", []) or []) + # Prefer the milestone-N-* label; fall back to the leading + # integer of GitHub's milestone title. The label is canonical + # because it survives milestone-title edits. + inferred = milestone_index_from_labels(list(label_names)) + if inferred is None: + gh_ms = item.get("milestone") or {} + gh_title = gh_ms.get("title", "") + tm = re.match(r"^(\d+)\.", gh_title) + inferred = int(tm.group(1)) if tm else ms_idx + out.append(Issue( + id=issue_id, + title=title, + body=item.get("body") or "", + labels=label_names, + milestone_idx=inferred, + state=item.get("state", "open").lower(), + gh_number=item.get("number"), + assignees=tuple( + a["login"] for a in item.get("assignees", []) or [] + ), + )) + return out + return _parse_json(stdout, "issues").map(to_issues) + + +def _parse_json(stdout: str, kind: str) -> Result[list[dict], PipelineError]: + """Decode the JSON body of a `gh` invocation, with structured errors.""" + try: + data = json.loads(stdout) + except json.JSONDecodeError as e: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"failed to decode {kind} JSON from `gh`", + cause=e, + context={"stdout_preview": stdout[:500]}, + )) + if not isinstance(data, list): + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"expected a JSON list of {kind}, got {type(data).__name__}", + )) + return Result.ok(data) + + +def _parse_milestone_create_response(stdout: str) -> Result[int, PipelineError]: + try: + data = json.loads(stdout) + return Result.ok(int(data["number"])) + except (json.JSONDecodeError, KeyError, ValueError) as e: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message="could not extract milestone number from `gh api` response", + cause=e, + )) + + +def _parse_issue_create_response(stdout: str) -> Result[int, PipelineError]: + """`gh issue create` prints the issue URL; extract the trailing number.""" + m = re.search(r"/issues/(\d+)\s*$", stdout.strip()) + if not m: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message="could not extract issue number from `gh issue create` output", + context={"stdout": stdout}, + )) + return Result.ok(int(m.group(1))) diff --git a/scripts/_utils/command_runner.py b/scripts/_utils/command_runner.py index 9c886ae..a85fec4 100644 --- a/scripts/_utils/command_runner.py +++ b/scripts/_utils/command_runner.py @@ -73,6 +73,8 @@ def _drain(pipe, emit) -> None: reader.start() for reader in readers: reader.join() + process.stdout.close() + process.stderr.close() # Wait for the process to complete and get the return code process.wait() diff --git a/scripts/tests/test_command_runner.py b/scripts/tests/test_command_runner.py index baee904..61dc5cc 100644 --- a/scripts/tests/test_command_runner.py +++ b/scripts/tests/test_command_runner.py @@ -20,6 +20,7 @@ """ from __future__ import annotations +import logging import sys import tempfile import threading @@ -30,6 +31,16 @@ from _utils.command_runner import run_command # noqa: E402 + +def setUpModule() -> None: + # The stream-output tests drain 200 KB of child stderr through the + # logger; without this the test run's output is that flood. + logging.disable(logging.CRITICAL) + + +def tearDownModule() -> None: + logging.disable(logging.NOTSET) + # Comfortably beyond any OS pipe buffer (Linux default: 64 KiB). STDERR_FLOOD = 200_000 diff --git a/scripts/tests/test_gh_project_lib.py b/scripts/tests/test_gh_project_lib.py new file mode 100644 index 0000000..cde6670 --- /dev/null +++ b/scripts/tests/test_gh_project_lib.py @@ -0,0 +1,379 @@ +""" +File: scripts/tests/test_gh_project_lib.py + +Description: + Unit tests for the pure core of _gh_project_lib: plan-file parsing, + generated-region parsing, label/milestone/issue planning, and the + crash-safe issue-number write-back. No network, no gh. +""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import _gh_project_lib as lib # noqa: E402 +from _gh_project_lib import ( # noqa: E402 + Issue, + Label, + Milestone, + normalize_label_name, + parse_file, + parse_issue_id, + parse_project_plan, + parse_repository, + plan_issues, + plan_labels, + plan_milestones, + record_issue_number, +) + +# A miniature plan file exercising every construct the parser must +# handle: repository header, labels section, two milestones, issues with +# metadata lines, a recorded issue number, a fan-out letter suffix, a +# region marker between issues, and trailing prose after the last issue. +PLAN = """\ +# Demo — GitHub Project Roadmap + +**Repository**: `octocat/demo` + +## Summary + +Two milestones, four issues. + +## Labels + +- `milestone-1-core` (0075ca) — Milestone 1: Core. +- `milestone-2-polish` (5319e7) — Milestone 2: Polish. +- `documentation` (0e8a16) — Documentation changes. + +## Milestones + +### Milestone 1 — Core + +**Description:** + +Build the core. + +**Exit criterion:** + +Core works. + +--- + +### Milestone 2 — Polish + +**Description:** + +Polish it. + +**Exit criterion:** + +It shines. + +## Issues + +## Milestone 1 — Core + + + +### Issue M1-1: Set up the build + +**Labels:** `milestone-1-core` +**Milestone:** 1. Core + +Body of M1-1. + +--- + +### Issue M1-2: [M1-2] Write the docs (#42) + +**Labels:** `milestone-1-core`, `documentation` + +Body of M1-2. + + + +## Milestone 2 — Polish + + + +### Issue M2-1: Shine it + +**Labels:** `milestone-2-polish` + +Body of M2-1. + +--- + +### Issue M2-1a: Shine it more + +**Labels:** `milestone-2-polish` + +Body of M2-1a. + + + +## How to use + +Trailing prose that must not leak into any issue body. +""" + + +class ParseIssueId(unittest.TestCase): + def test_plain(self) -> None: + self.assertEqual( + parse_issue_id("[M1-2] Title here"), + ("M1-2", 1, 2, "", "Title here"), + ) + + def test_letter_suffix(self) -> None: + self.assertEqual( + parse_issue_id("[M2-7a] Child ticket"), + ("M2-7a", 2, 7, "a", "Child ticket"), + ) + + def test_no_space_after_bracket_is_rejected(self) -> None: + self.assertIsNone(parse_issue_id("[M1-1]Title")) + + def test_unprefixed_is_rejected(self) -> None: + self.assertIsNone(parse_issue_id("Ordinary bug report")) + + +class IssueIdOrdering(unittest.TestCase): + def test_ordering(self) -> None: + self.assertTrue(lib.issue_id_gte("M1-3", "M1-2")) + self.assertFalse(lib.issue_id_gte("M0-9", "M1-1")) + self.assertTrue(lib.issue_id_gte("M2-7a", "M2-7")) + self.assertTrue(lib.issue_id_gte("M2-7b", "M2-7a")) + + +class ParseRepository(unittest.TestCase): + def test_backticked(self) -> None: + self.assertEqual(parse_repository(PLAN), "octocat/demo") + + def test_bare(self) -> None: + self.assertEqual( + parse_repository("**Repository**: octocat/demo\n"), "octocat/demo" + ) + + def test_absent(self) -> None: + self.assertIsNone(parse_repository("# No header here\n")) + + +class ParsePlan(unittest.TestCase): + def setUp(self) -> None: + self.plan = parse_project_plan(PLAN) + + def test_milestones(self) -> None: + self.assertEqual( + [(m.number, m.title) for m in self.plan.milestones], + [(1, "1. Core"), (2, "2. Polish")], + ) + self.assertIn("Build the core.", self.plan.milestones[0].description) + self.assertIn( + "**Exit criterion:** Core works.", self.plan.milestones[0].description + ) + + def test_labels_from_explicit_section(self) -> None: + self.assertEqual( + [(l.name, l.color) for l in self.plan.labels], + [ + ("milestone-1-core", "0075ca"), + ("milestone-2-polish", "5319e7"), + ("documentation", "0e8a16"), + ], + ) + + def test_labels_fallback_collects_from_issues(self) -> None: + no_section = PLAN.replace("## Labels", "## Labelless") + labels = parse_project_plan(no_section).labels + self.assertEqual( + {l.name for l in labels}, + {"milestone-1-core", "milestone-2-polish", "documentation"}, + ) + self.assertTrue(all(l.color == "cccccc" for l in labels)) + + def test_labels_no_defaults_imposed(self) -> None: + self.assertEqual(parse_project_plan("# Empty plan\n").labels, ()) + + def test_issue_ids_and_titles(self) -> None: + issues = {i.id: i for i in self.plan.issues} + self.assertEqual( + set(issues), {"M1-1", "M1-2", "M2-1", "M2-1a"} + ) + # Every pushed title carries exactly one [MN-k] prefix, whether or + # not the heading already had one. + self.assertEqual(issues["M1-1"].title, "[M1-1] Set up the build") + self.assertEqual(issues["M1-2"].title, "[M1-2] Write the docs") + + def test_recorded_number_is_state_not_title(self) -> None: + m12 = next(i for i in self.plan.issues if i.id == "M1-2") + self.assertEqual(m12.gh_number, 42) + self.assertNotIn("(#42)", m12.title) + + def test_metadata_lines_are_stripped_from_bodies(self) -> None: + for issue in self.plan.issues: + self.assertNotIn("**Labels:**", issue.body) + self.assertNotIn("**Milestone:**", issue.body) + + def test_labels_parsed_from_metadata(self) -> None: + m12 = next(i for i in self.plan.issues if i.id == "M1-2") + self.assertEqual(m12.labels, ("milestone-1-core", "documentation")) + + def test_bodies_do_not_leak(self) -> None: + bodies = {i.id: i.body for i in self.plan.issues} + self.assertEqual(bodies["M1-1"], "Body of M1-1.") + self.assertEqual(bodies["M1-2"], "Body of M1-2.") + self.assertEqual(bodies["M2-1"], "Body of M2-1.") + # The last issue's body must not swallow the trailing prose or + # the closing region marker. + self.assertEqual(bodies["M2-1a"], "Body of M2-1a.") + + def test_milestone_idx_from_id(self) -> None: + self.assertEqual( + {i.id: i.milestone_idx for i in self.plan.issues}, + {"M1-1": 1, "M1-2": 1, "M2-1": 2, "M2-1a": 2}, + ) + + +class RecordIssueNumber(unittest.TestCase): + def test_adds_suffix(self) -> None: + text, found = record_issue_number(PLAN, "M1-1", 7) + self.assertTrue(found) + self.assertIn("### Issue M1-1: Set up the build (#7)\n", text) + + def test_replaces_existing_suffix(self) -> None: + text, found = record_issue_number(PLAN, "M1-2", 99) + self.assertTrue(found) + self.assertIn("### Issue M1-2: [M1-2] Write the docs (#99)\n", text) + self.assertNotIn("(#42)", text) + + def test_only_the_heading_changes(self) -> None: + text, _ = record_issue_number(PLAN, "M1-1", 7) + self.assertEqual( + [ln for ln in PLAN.splitlines() if "M1-1" in ln and ln.startswith("###")], + ["### Issue M1-1: Set up the build"], + ) + # Remove the one changed line from both; the rest is untouched. + before = PLAN.replace("### Issue M1-1: Set up the build\n", "") + after = text.replace("### Issue M1-1: Set up the build (#7)\n", "") + self.assertEqual(before, after) + + def test_unknown_id_reports_not_found(self) -> None: + text, found = record_issue_number(PLAN, "M9-9", 1) + self.assertFalse(found) + self.assertEqual(text, PLAN) + + def test_id_match_is_exact_not_prefix(self) -> None: + # Recording M2-1 must not touch the M2-1a heading. + text, found = record_issue_number(PLAN, "M2-1", 5) + self.assertTrue(found) + self.assertIn("### Issue M2-1: Shine it (#5)\n", text) + self.assertIn("### Issue M2-1a: Shine it more\n", text) + + +class ParseFileRegions(unittest.TestCase): + def test_round_trip_structure(self) -> None: + parsed = parse_file(PLAN).unwrap() + self.assertEqual(parsed.ids, ("milestone-1", "milestone-2")) + self.assertEqual(len(parsed.manuals), 3) + self.assertTrue(parsed.manuals[-1].startswith("\n\n## How to use")) + + def test_unterminated_begin(self) -> None: + r = parse_file("\nno end\n") + self.assertTrue(r.is_err) + self.assertIn("unterminated", r.unwrap_err().message) + + def test_mismatched_ids(self) -> None: + r = parse_file( + "\n\n" + ) + self.assertTrue(r.is_err) + self.assertIn("mismatched", r.unwrap_err().message) + + def test_stray_end(self) -> None: + r = parse_file("prose\n\n") + self.assertTrue(r.is_err) + self.assertIn("no matching BEGIN", r.unwrap_err().message) + + def test_nested_begin(self) -> None: + r = parse_file( + "\n" + "\n" + "\n" + ) + self.assertTrue(r.is_err) + self.assertIn("nested", r.unwrap_err().message) + + +class PlanLabels(unittest.TestCase): + def test_reconciliation(self) -> None: + desired = ( + Label("bug", "ff0000", "A bug"), + Label("era: conway", "00ff00", "Conway era"), + Label("Docs", "0000ff", "Documentation"), + Label("brand-new", "cccccc", ""), + ) + existing = ( + Label("bug", "aa0000", "Bugs (different color)"), + Label("era:conway", "00ff00", "Conway era"), + Label("docs", "0000ff", "Documentation"), + ) + plan = plan_labels(desired, existing) + self.assertEqual([l.name for l in plan.to_create], ["brand-new"]) + self.assertEqual([(d.name, e.name) for d, e in plan.existing], [("bug", "bug")]) + self.assertEqual( + [(d.name, e.name) for d, e in plan.collisions], + [("era: conway", "era:conway"), ("Docs", "docs")], + ) + + def test_normalize(self) -> None: + self.assertEqual(normalize_label_name("Era: Conway"), "era:conway") + self.assertEqual(normalize_label_name("good first issue"), + normalize_label_name("Good First Issue")) + + +class PlanMilestones(unittest.TestCase): + def test_match_by_title(self) -> None: + desired = ( + Milestone(1, "1. Core", "d1"), + Milestone(2, "2. Polish", "d2"), + ) + existing = (Milestone(1, "1. Core", "live", gh_number=11),) + plan = plan_milestones(desired, existing) + self.assertEqual([m.title for m in plan.to_create], ["2. Polish"]) + self.assertEqual([(m.title, m.gh_number) for m in plan.existing], + [("1. Core", 11)]) + + +class PlanIssues(unittest.TestCase): + def test_match_by_id_prefix(self) -> None: + desired = (Issue("M1-1", "[M1-1] A", "b"),) + existing = (Issue("M1-1", "[M1-1] A renamed", "b", gh_number=3),) + plan = plan_issues(desired, existing) + self.assertEqual(plan.to_create, ()) + self.assertEqual(plan.existing[0][1].gh_number, 3) + + def test_match_by_recorded_number(self) -> None: + # Someone stripped the [M1-1] prefix on GitHub; the recorded (#3) + # in the plan file still prevents a duplicate... + desired = (Issue("M1-1", "[M1-1] A", "b", gh_number=3),) + # ...but note the live issue only appears in the snapshot if its + # title still parses; simulate a live issue whose ID changed. + existing = (Issue("M9-9", "[M9-9] Retitled", "b", gh_number=3),) + plan = plan_issues(desired, existing) + self.assertEqual(plan.to_create, ()) + self.assertEqual(plan.existing[0][1].gh_number, 3) + + def test_new_issue(self) -> None: + desired = (Issue("M1-2", "[M1-2] New", "b"),) + existing = (Issue("M1-1", "[M1-1] Old", "b", gh_number=3),) + plan = plan_issues(desired, existing) + self.assertEqual([i.id for i in plan.to_create], ["M1-2"]) + + +if __name__ == "__main__": + unittest.main() From 98df1597294ed1c6b849e82b05109b5c13336c6a Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Thu, 13 Aug 2026 22:02:48 -0600 Subject: [PATCH 3/9] Add populate, update, and lint CLIs with offline integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh_project_populate.py (file -> GitHub), ported from the newest lineage and extended per the design requirements: - one RepoSnapshot per run feeds the pure planners (#293 item 6 closed; regression: the fake-gh transcript shows exactly one list call per entity kind however many issues the plan holds) - crash-safe write-back: each created issue's number is recorded as a (#N) heading suffix and persisted before the next remote mutation (regression: a run that dies at issue 3 of 4 reruns to exactly 4 issues, zero duplicates) - label reconciliation: near-collisions (case/whitespace) are reported and skipped; existing labels are never overwritten (both pinned by integration tests) - --repo falls back to the plan file's **Repository**: header; --yes for non-interactive runs; --delay default 1.0s; --start-from now uses the documented >= ordering (the ancestor's loop used equality, silently skipping everything when the ID never matched) - refuses to run against a plan with lint errors gh_project_update.py (GitHub -> file) is the renamed render — 'update' names the intent (bring the file up to date), not the direction — with diff(1) exit codes kept (0 current / 1 drift / 2 failure) and assignees now rendered (GitHub-owned state the ancestor omitted). gh_project_lint.py is the no-network structural gate (markers balanced, unique, un-nested; labels section parses with no case/whitespace collisions; milestone references consistent; issue IDs unique; label references resolvable). Populate shares the same lint_plan core. scripts/tests/fake_gh/ is a recorded-transcript gh stand-in: the integration tests run the real scripts as subprocesses with it on PATH, so command construction, JSON parsing, write-back, and exit codes are exercised with zero network. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 153 +++++++++ scripts/gh_project_lint.py | 71 ++++ scripts/gh_project_populate.py | 419 ++++++++++++++++++++++++ scripts/gh_project_update.py | 246 ++++++++++++++ scripts/tests/fake_gh/gh | 134 ++++++++ scripts/tests/test_gh_project_lint.py | 129 ++++++++ scripts/tests/test_gh_project_update.py | 104 ++++++ scripts/tests/test_integration.py | 257 +++++++++++++++ 8 files changed, 1513 insertions(+) create mode 100755 scripts/gh_project_lint.py create mode 100755 scripts/gh_project_populate.py create mode 100755 scripts/gh_project_update.py create mode 100755 scripts/tests/fake_gh/gh create mode 100644 scripts/tests/test_gh_project_lint.py create mode 100644 scripts/tests/test_gh_project_update.py create mode 100644 scripts/tests/test_integration.py diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 14b2b86..6c0ad7a 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -599,6 +599,159 @@ def plan_issues(desired: tuple[Issue, ...], existing: tuple[Issue, ...]) -> Issu return IssuePlan(tuple(to_create), tuple(exists)) +# ── Structural lint (pure; no network) ────────────────────────────────────── + +@dataclass(frozen=True) +class Problem: + """One structural problem found in a plan file.""" + severity: str # "error" or "warning" + message: str + line: Optional[int] = None + + +def _line_of(text: str, offset: int) -> int: + """1-based line number of a character offset.""" + return text.count("\n", 0, offset) + 1 + + +def lint_plan(text: str) -> tuple[Problem, ...]: + """Validate a plan file's structure without touching the network. + + This is the drift gate this design admits without network access: + cheap enough to run on every PR, strict enough that populate refuses + to run against a file with errors. + + Errors (populate must not run): + - unbalanced / mismatched / nested / stray generated-region markers + - duplicate region ids + - duplicate issue IDs + - duplicate label names, or two plan labels that collide after + normalization (case/whitespace) + - an issue whose milestone number has no `### Milestone N` entry + (when a Milestones section exists) + - a `milestone-N` region id with no matching milestone + - an issue referencing a label absent from the `## Labels` section + (when that section exists — creation on GitHub would fail) + - a malformed entry line inside `## Labels` + + Warnings: + - no `**Repository**:` header (the CLI --repo can still supply it) + - duplicate milestone numbers + """ + problems: list[Problem] = [] + + region = parse_file(text) + ids: tuple[str, ...] = () + if region.is_err: + err = region.unwrap_err() + offset = err.context.get("offset") or err.context.get("begin_offset") + problems.append(Problem( + "error", err.message, + _line_of(text, offset) if offset is not None else None, + )) + else: + ids = region.unwrap().ids + seen_ids: set[str] = set() + for rid in ids: + if rid in seen_ids: + problems.append(Problem( + "error", f"duplicate generated-region id '{rid}'")) + seen_ids.add(rid) + + plan = parse_project_plan(text) + + if plan.repository is None: + problems.append(Problem( + "warning", + "no `**Repository**: owner/name` header; --repo must be " + "passed on the command line", + )) + + ms_numbers = [m.number for m in plan.milestones] + for n in {n for n in ms_numbers if ms_numbers.count(n) > 1}: + problems.append(Problem( + "warning", f"milestone number {n} is defined more than once")) + + issue_ids = [i.id for i in plan.issues] + for dup in sorted({i for i in issue_ids if issue_ids.count(i) > 1}): + problems.append(Problem( + "error", f"duplicate issue ID {dup} — populate would create " + f"one issue and orphan the other heading")) + + if plan.milestones: + known = set(ms_numbers) + for issue in plan.issues: + if issue.milestone_idx not in known: + problems.append(Problem( + "error", + f"issue {issue.id} refers to milestone " + f"{issue.milestone_idx}, which has no " + f"`### Milestone {issue.milestone_idx}` entry", + )) + for rid in ids: + m = re.match(r"^milestone-(\d+)$", rid) + if m and int(m.group(1)) not in known: + problems.append(Problem( + "error", + f"region '{rid}' has no matching milestone " + f"{int(m.group(1))} in the Milestones section", + )) + + problems.extend(_lint_labels(text, plan)) + return tuple(problems) + + +def _lint_labels(text: str, plan: ProjectPlan) -> list[Problem]: + """Label-section checks: well-formedness, duplicates, references.""" + problems: list[Problem] = [] + + section = re.search(r"^## Labels\s*$", text, re.MULTILINE) + explicit = _parse_explicit_labels(text) + + if section: + start = section.end() + nxt = re.search(r"^## (?!#)", text[start:], re.MULTILINE) + end = start + nxt.start() if nxt else len(text) + entry_starts = re.finditer(r"^\s*[-*]\s+`", text[start:end], re.MULTILINE) + parsed_count = len(explicit) + listed_count = sum(1 for _ in entry_starts) + if listed_count > parsed_count: + problems.append(Problem( + "error", + f"## Labels section: {listed_count - parsed_count} " + f"entr{'y' if listed_count - parsed_count == 1 else 'ies'} " + f"did not parse (expected `- `name` (RRGGBB) — description`)", + _line_of(text, section.start()), + )) + + names = [l.name for l in explicit] + for dup in sorted({n for n in names if names.count(n) > 1}): + problems.append(Problem("error", f"duplicate label `{dup}`")) + by_norm: dict[str, str] = {} + for n in names: + norm = normalize_label_name(n) + if norm in by_norm and by_norm[norm] != n: + problems.append(Problem( + "error", + f"labels `{by_norm[norm]}` and `{n}` collide after " + f"case/whitespace normalization", + )) + by_norm.setdefault(norm, n) + + if explicit: + known = {normalize_label_name(n) for n in names} + for issue in plan.issues: + for lbl in issue.labels: + if normalize_label_name(lbl) not in known: + problems.append(Problem( + "error", + f"issue {issue.id} references label `{lbl}`, which " + f"is not in the ## Labels section — issue creation " + f"on GitHub would fail", + )) + return problems + + # ── GitHub client ──────────────────────────────────────────────────────────── @dataclass(frozen=True) diff --git a/scripts/gh_project_lint.py b/scripts/gh_project_lint.py new file mode 100755 index 0000000..12a29bb --- /dev/null +++ b/scripts/gh_project_lint.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +File: scripts/gh_project_lint.py + +Description: + Structural validation of a project plan file — no network, no gh. + + Checks that the generated-region markers are balanced and uniquely + named, the `## Labels` section parses (no duplicates or + case/whitespace collisions), milestone references are consistent, and + issue headings are well-formed. This is the drift gate the + populate/update design admits without network access: cheap enough to + run on every PR (`make lint` / CI), and gh_project_populate.py runs + the same checks before mutating anything. + +Usage: + python3 scripts/gh_project_lint.py docs/GITHUB_PROJECT.md + +Exit codes: + 0 no errors (warnings may have been printed) + 1 at least one error + 2 the file could not be read +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +# Permit `python3 scripts/gh_project_lint.py ...` from the repo root by +# ensuring this file's directory is on sys.path before importing the +# sibling private modules. +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +from _gh_project_lib import lint_plan, parse_project_plan # noqa: E402 +from _utils.file_ops import read_text # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Validate the structure of a project plan file (no network)." + ) + parser.add_argument("markdown", type=Path, help="Path to GITHUB_PROJECT.md.") + args = parser.parse_args() + + read = read_text(args.markdown) + if read.is_err: + print(f"error: {read.unwrap_err()}", file=sys.stderr) + return 2 + text = read.unwrap() + + problems = lint_plan(text) + for p in problems: + loc = f":{p.line}" if p.line else "" + print(f"{args.markdown}{loc}: {p.severity}: {p.message}") + + errors = sum(1 for p in problems if p.severity == "error") + warnings = len(problems) - errors + plan = parse_project_plan(text) + print( + f"{args.markdown}: {len(plan.milestones)} milestones, " + f"{len(plan.labels)} labels, {len(plan.issues)} issues — " + f"{errors} error(s), {warnings} warning(s)" + ) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py new file mode 100755 index 0000000..0afa5e3 --- /dev/null +++ b/scripts/gh_project_populate.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +""" +File: scripts/gh_project_populate.py + +Description: + Create GitHub labels, milestones, and issues from a structured Markdown + project plan (docs/GITHUB_PROJECT.md), and write the new issue numbers + back into the plan file. + + This is the file → GitHub direction; the inverse (GitHub → file) is + gh_project_update.py. + +Usage: + python3 scripts/gh_project_populate.py docs/GITHUB_PROJECT.md [OPTIONS] + + The target repository comes from the plan file's `**Repository**:` + header, or from --repo (which wins when both are present). + +Behavior guarantees: + - Idempotent: one snapshot of live GitHub state is fetched per run and + everything that already exists is skipped — labels by exact name, + milestones by exact title, issues by their `[MN-k]` title prefix (or + by a `(#N)` number previously recorded in the plan file). + - Crash-safe: each created issue's number is written back into the + plan file (as a `(#N)` heading suffix) and persisted BEFORE the next + remote mutation, so an interrupted run never causes duplicates on + rerun. + - Label reconciliation, not imposition: existing labels are never + overwritten (color/description differences are reported), and + near-collisions (`era: conway` vs an existing `era:conway`) are + reported and skipped rather than silently creating a parallel + scheme. + - Dry-run-first: --dry-run prints exactly the plan a real run would + execute (reads still happen; writes never do). + +Options: + --repo OWNER/NAME Target repository (overrides the file header) + --dry-run Print the plan without mutating anything + --milestones-only Only create milestones + --labels-only Only create labels + --issues-only Only create issues (milestones must already exist) + --skip-labels Skip label creation + --start-from ID Create only issues with ID >= this (e.g. M1-3) + --delay SECONDS Pause between mutating calls (default: 1.0) + --yes Skip the interactive confirmation + --env-prefix Prefix gh with `env -u GH_TOKEN -u GITHUB_TOKEN` + (default; works around token-precedence quirks) + --no-env-prefix Disable the env prefix. Needed wherever + authentication comes *through* those variables, + GitHub Actions included. + +Exit codes: + 0 everything requested was created or already existed + 1 some mutations failed or were skipped (collisions, missing labels) + 2 the run could not proceed (unreadable file, lint errors, no repo, + snapshot failure) +""" +from __future__ import annotations + +import argparse +import sys +import textwrap +import time +from pathlib import Path + +# Permit `python3 scripts/gh_project_populate.py ...` from the repo root +# by ensuring this file's directory is on sys.path before importing the +# sibling private modules. +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +from _gh_project_lib import ( # noqa: E402 + GitHubClient, + IssuePlan, + LabelPlan, + MilestonePlan, + ProjectPlan, + RepoSnapshot, + issue_id_gte, + lint_plan, + parse_project_plan, + plan_issues, + plan_labels, + plan_milestones, + record_issue_number, +) +from _utils.file_ops import read_text, write_text # noqa: E402 + + +# ── Plan reporting (shared verbatim by dry-run and live runs) ─────────────── + +def print_label_plan(plan: LabelPlan) -> None: + for label in plan.to_create: + print(f" + create label: {label.name} (#{label.color})") + for desired, live in plan.existing: + print(f" - exists: label {live.name}") + if (desired.color.lower() != live.color.lower() + or desired.description != live.description): + print( + f" note: differs from the plan " + f"(plan #{desired.color} {desired.description!r}, " + f"live #{live.color} {live.description!r}) — not overwritten" + ) + for desired, live in plan.collisions: + print( + f" ! collision: plan label `{desired.name}` vs existing " + f"`{live.name}` (same name modulo case/whitespace) — skipped; " + f"rename one side, then re-run" + ) + + +def print_milestone_plan(plan: MilestonePlan) -> None: + for ms in plan.to_create: + print(f" + create milestone: {ms.title}") + for ms in plan.existing: + print(f" - exists: milestone #{ms.gh_number} {ms.title}") + + +def print_issue_plan(plan: IssuePlan) -> None: + for issue in plan.to_create: + print(f" + create issue: {issue.title}") + for _, live in plan.existing: + print(f" - exists: issue #{live.gh_number} {live.title}") + + +def _stage_header(title: str) -> None: + print("═" * 60) + print(title) + print("═" * 60) + + +# ── Execution (the only functions that mutate GitHub or the file) ─────────── + +def execute_labels( + client: GitHubClient, plan: LabelPlan, delay: float +) -> int: + """Create the planned labels; returns the number of failures.""" + failed = 0 + for label in plan.to_create: + result = client.create_label(label) + if result.is_ok: + print(f" created label: {label.name}") + else: + failed += 1 + print(f" ! label {label.name}: {result.unwrap_err()}") + time.sleep(delay) + return failed + + +def execute_milestones( + client: GitHubClient, plan: MilestonePlan, delay: float +) -> tuple[dict[int, str], int]: + """Create the planned milestones; returns ({plan index → title}, failures).""" + title_map = {ms.number: ms.title for ms in plan.existing} + failed = 0 + for ms in plan.to_create: + result = client.create_milestone(ms) + if result.is_ok: + created = result.unwrap() + title_map[created.number] = created.title + print(f" created milestone #{created.gh_number}: {created.title}") + else: + failed += 1 + print(f" ! milestone {ms.title}: {result.unwrap_err()}") + time.sleep(delay) + return title_map, failed + + +def execute_issues( + client: GitHubClient, + plan: IssuePlan, + ms_title_map: dict[int, str], + available_labels: set[str], + md_path: Path, + text: str, + delay: float, +) -> tuple[str, int]: + """Create the planned issues with immediate number write-back. + + Returns (current file text, failures). The plan file is persisted + after EACH creation, before the next remote mutation: a later + failure must not lose an already-assigned number, or a rerun would + file a duplicate. + """ + failed = 0 + for issue in plan.to_create: + missing = [ + lbl for lbl in issue.labels + if lbl.casefold() not in available_labels + ] + if missing: + failed += 1 + print( + f" ! issue {issue.id}: skipped — references unavailable " + f"label(s) {', '.join(f'`{m}`' for m in missing)} " + f"(not on GitHub and not created this run)" + ) + continue + + ms_title = ms_title_map.get(issue.milestone_idx) + if ms_title is None and issue.milestone_idx != 0: + print( + f" ~ issue {issue.id}: milestone {issue.milestone_idx} " + f"unavailable; creating without a milestone" + ) + + result = client.create_issue(issue, milestone_title=ms_title) + if result.is_err: + failed += 1 + print(f" ! issue {issue.id}: {result.unwrap_err()}") + time.sleep(delay) + continue + + gh_number = result.unwrap() + print(f" created issue #{gh_number}: {issue.title}") + + text, found = record_issue_number(text, issue.id, gh_number) + if not found: + print( + f" ~ could not write #{gh_number} back for {issue.id} " + f"(heading not found)", file=sys.stderr, + ) + else: + write_result = write_text(md_path, text) + if write_result.is_err: + print( + f" ~ write-back failed: {write_result.unwrap_err()}", + file=sys.stderr, + ) + time.sleep(delay) + return text, failed + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Populate a GitHub repository from a structured Markdown plan.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent("""\ + Examples: + # Dry run — see what would be created: + python3 scripts/gh_project_populate.py docs/GITHUB_PROJECT.md --dry-run + + # Create everything (prompts for confirmation): + python3 scripts/gh_project_populate.py docs/GITHUB_PROJECT.md + + # Resume from a specific issue: + python3 scripts/gh_project_populate.py docs/GITHUB_PROJECT.md \\ + --issues-only --start-from M1-3 + """), + ) + parser.add_argument("markdown", type=Path, help="Path to the project plan markdown file") + parser.add_argument("--repo", help="GitHub repo (owner/name); overrides the file header") + parser.add_argument("--dry-run", action="store_true", help="Print the plan without executing") + parser.add_argument("--milestones-only", action="store_true") + parser.add_argument("--labels-only", action="store_true") + parser.add_argument("--issues-only", action="store_true") + parser.add_argument("--skip-labels", action="store_true") + parser.add_argument("--start-from", type=str, default=None, + help="Create only issues with ID >= this (e.g. M1-3)") + parser.add_argument("--delay", type=float, default=1.0, + help="Seconds between mutating API calls (default: 1.0)") + parser.add_argument("--yes", action="store_true", + help="Skip the interactive confirmation") + parser.add_argument("--env-prefix", action="store_true", default=True, + help="Prefix gh with `env -u GH_TOKEN -u GITHUB_TOKEN` (default)") + parser.add_argument("--no-env-prefix", action="store_true", + help="Don't prefix gh commands with env") + return parser + + +def main() -> int: + args = build_parser().parse_args() + if args.no_env_prefix: + args.env_prefix = False + + do_labels = not args.issues_only and not args.milestones_only and not args.skip_labels + do_milestones = not args.issues_only and not args.labels_only + do_issues = not args.milestones_only and not args.labels_only + + # ── Read and validate ──────────────────────────────────────────────── + read = read_text(args.markdown) + if read.is_err: + print(f"error: {read.unwrap_err()}", file=sys.stderr) + return 2 + text = read.unwrap() + + problems = lint_plan(text) + for p in problems: + loc = f":{p.line}" if p.line else "" + print(f"{args.markdown}{loc}: {p.severity}: {p.message}", file=sys.stderr) + if any(p.severity == "error" for p in problems): + print("error: fix the plan file before populating", file=sys.stderr) + return 2 + + plan: ProjectPlan = parse_project_plan(text) + repo = args.repo or plan.repository + if repo is None: + print( + "error: no repository given — add a `**Repository**: owner/name` " + "header to the plan file or pass --repo", + file=sys.stderr, + ) + return 2 + + desired_issues = plan.issues + if args.start_from: + before = len(desired_issues) + desired_issues = tuple( + i for i in desired_issues if issue_id_gte(i.id, args.start_from) + ) + print(f"--start-from {args.start_from}: " + f"skipping {before - len(desired_issues)} earlier issue(s)") + + print(f"Parsed {args.markdown}: {len(plan.milestones)} milestones, " + f"{len(plan.labels)} labels, {len(plan.issues)} issues") + print(f"Target repo: {repo}{' (dry-run)' if args.dry_run else ''}") + print() + + # ── Snapshot (one fetch per run — reads happen even under --dry-run, + # so the printed plan is exactly what a live run would execute) ──── + client = GitHubClient(repo=repo, env_prefix=args.env_prefix) + snap_result = client.snapshot() + if snap_result.is_err: + print(f"error: could not fetch repository state: " + f"{snap_result.unwrap_err()}", file=sys.stderr) + return 2 + snapshot: RepoSnapshot = snap_result.unwrap() + + # ── Plan ───────────────────────────────────────────────────────────── + label_plan = plan_labels(plan.labels, snapshot.labels) + milestone_plan = plan_milestones(plan.milestones, snapshot.milestones) + issue_plan = plan_issues(desired_issues, snapshot.issues) + + if do_labels: + _stage_header("LABELS") + print_label_plan(label_plan) + print() + if do_milestones: + _stage_header("MILESTONES") + print_milestone_plan(milestone_plan) + print() + if do_issues: + _stage_header("ISSUES") + print_issue_plan(issue_plan) + print() + + collisions = len(label_plan.collisions) if do_labels else 0 + + if args.dry_run: + print("Dry run — nothing was created.") + return 1 if collisions else 0 + + # ── Confirm ────────────────────────────────────────────────────────── + to_do = [] + if do_labels: + to_do.append(f"{len(label_plan.to_create)} labels") + if do_milestones: + to_do.append(f"{len(milestone_plan.to_create)} milestones") + if do_issues: + to_do.append(f"{len(issue_plan.to_create)} issues") + print(f"This will create: {', '.join(to_do)}") + if not args.yes: + response = input("Continue? [y/N] ").strip().lower() + if response not in ("y", "yes"): + print("Aborted.") + return 0 + print() + + # ── Execute ────────────────────────────────────────────────────────── + failures = 0 + + if do_labels: + _stage_header("CREATING LABELS") + failures += execute_labels(client, label_plan, args.delay) + print(f" labels: {len(label_plan.to_create)} planned, " + f"{len(label_plan.existing)} already existed, " + f"{collisions} collisions skipped") + print() + + ms_title_map: dict[int, str] = {ms.number: ms.title + for ms in milestone_plan.existing} + if do_milestones: + _stage_header("CREATING MILESTONES") + ms_title_map, ms_failed = execute_milestones( + client, milestone_plan, args.delay + ) + failures += ms_failed + print(f" milestones: {len(milestone_plan.to_create)} planned, " + f"{len(milestone_plan.existing)} already existed") + print() + + if do_issues: + _stage_header("CREATING ISSUES") + available = ( + {lbl.name.casefold() for lbl in snapshot.labels} + | {lbl.name.casefold() for lbl in label_plan.to_create} + ) + _text, issue_failed = execute_issues( + client, issue_plan, ms_title_map, available, + args.markdown, text, args.delay, + ) + failures += issue_failed + print() + _stage_header( + f"DONE: {len(issue_plan.to_create) - issue_failed} issues created, " + f"{len(issue_plan.existing)} already existed, {issue_failed} failed" + ) + + if collisions: + print(f"note: {collisions} label collision(s) were skipped — " + f"see the LABELS report above", file=sys.stderr) + return 1 if failures or collisions else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py new file mode 100755 index 0000000..4217081 --- /dev/null +++ b/scripts/gh_project_update.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +""" +File: scripts/gh_project_update.py + +Description: + + Bring docs/GITHUB_PROJECT.md up to date: regenerate the issue listings + inside it from current GitHub state, leaving hand-edited prose + untouched. + + This is the GitHub → file direction; the inverse (file → GitHub) is + gh_project_populate.py. ("Update" names the intent — bring the file + up to date — not a push to GitHub.) + + The file is treated as a sequence of manual prose segments interleaved + with regions delimited by HTML-comment markers of the form + + + ... + + + Update preserves manual segments byte-for-byte and rebuilds each + generated region from the live GitHub API. A region with id + `milestone-N` is rebuilt from issues whose `milestone-N-*` label + identifies them as belonging to milestone N, ordered by their `[MN-k]` + ordinal. Region ids that don't match this pattern produce a + no-rendering-rule placeholder so that bad ids fail loudly rather than + silently emitting nothing. + +Usage: + + python3 scripts/gh_project_update.py docs/GITHUB_PROJECT.md [OPTIONS] + + The target repository comes from the plan file's `**Repository**:` + header, or from --repo (which wins when both are present). + + --check Verify that the file already matches the rendered + output; do not write. + +Exit codes follow diff(1), so a caller can tell "the plan is stale" from +"the check itself did not run": + + 0 the file is current (or, without --check, was written successfully) + 1 --check only: the file differs from live GitHub state + 2 the run failed — authentication, API error, unreadable file, bad markers + + --no-env-prefix Don't prefix `gh` with `env -u GH_TOKEN -u GITHUB_TOKEN`. + Needed wherever authentication comes *through* those + variables, GitHub Actions included: stripping them there + leaves gh with no credentials at all. +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +# Permit `python3 scripts/gh_project_update.py ...` from the repo root +# by ensuring this file's directory is on sys.path before importing the +# sibling private modules. +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +from _gh_project_lib import ( # noqa: E402 + GitHubClient, + Issue, + ParsedFile, + parse_file, + parse_repository, +) +from _utils import Result, PipelineError, ErrorType # noqa: E402 +from _utils.file_ops import read_text, write_text # noqa: E402 + + +# ── Issue grouping and sorting ─────────────────────────────────────────────── + +def group_by_milestone(issues: list[Issue]) -> dict[int, list[Issue]]: + out: dict[int, list[Issue]] = {} + for issue in issues: + out.setdefault(issue.milestone_idx, []).append(issue) + return out + + +def issue_sort_key(issue: Issue) -> tuple[int, int, str]: + """Sort key for in-region ordering: (milestone, ordinal, suffix). + The empty-string suffix on plain `[MN-k]` ids sorts before any + alphabetic suffix, so a parent like `M2-7` precedes its children + `M2-7a`, `M2-7b`, ... naturally. + """ + m = re.match(r"^M(\d+)-(\d+)([a-z]?)$", issue.id) + return (int(m.group(1)), int(m.group(2)), m.group(3) or "") if m else (10**9, 10**9, "") + + +# ── Region rendering ───────────────────────────────────────────────────────── + +_ID_PREFIX_RE = re.compile(r"^\[M\d+-\d+[a-z]?\]\s+(.+)$") + + +def strip_id_prefix(title: str) -> str: + """Drop the leading `[MN-k]` from an issue title; the heading reproduces + the identifier separately, so leaving it in the title text is redundant.""" + m = _ID_PREFIX_RE.match(title) + return m.group(1) if m else title + + +def render_issue(issue: Issue) -> str: + title = strip_id_prefix(issue.title) + state_suffix = ", closed" if issue.state == "closed" else "" + ref = f"#{issue.gh_number}{state_suffix}" if issue.gh_number else "(no number)" + labels = ", ".join(f"`{lbl}`" for lbl in issue.labels) + # Assignees are GitHub-owned state (like open/closed); render them so + # the file answers "who is on this?" without a browser. + assignees = ( + f"**Assignees**: {', '.join('@' + a for a in issue.assignees)}\n\n" + if issue.assignees else "" + ) + body = issue.body.strip() if issue.body else "_(no description on GitHub)_" + return ( + f"### Issue {issue.id}: {title} ({ref})\n" + f"\n" + f"**Labels**: {labels}\n" + f"\n" + f"{assignees}" + f"{body}\n" + ) + + +def render_region(region_id: str, issues_by_ms: dict[int, list[Issue]]) -> str: + m = re.match(r"^milestone-(\d+)$", region_id) + if m is None: + return f"\n\n" + n = int(m.group(1)) + issues = issues_by_ms.get(n, []) + if not issues: + return f"\n_(no open or closed issues with `milestone-{n}-*` label)_\n" + blocks = [render_issue(i) for i in sorted(issues, key=issue_sort_key)] + return "\n" + "\n---\n\n".join(blocks) + "\n" + + +def assemble_file(parsed: ParsedFile, issues_by_ms: dict[int, list[Issue]]) -> str: + parts: list[str] = [] + for i, manual in enumerate(parsed.manuals[:-1]): + rid = parsed.ids[i] + parts.append(manual) + parts.append(f"\n") + parts.append(render_region(rid, issues_by_ms)) + parts.append(f"") + parts.append(parsed.manuals[-1]) + return "".join(parts) + + +# ── Validation ─────────────────────────────────────────────────────────────── + +def warn_orphan_milestones(parsed: ParsedFile, issues_by_ms: dict[int, list[Issue]]) -> None: + """Stderr-warn for each milestone-N-* label group lacking a marker + region in the file. Such issues are silently dropped from output; + surfacing the warning lets the maintainer notice and add the region.""" + region_milestones: set[int] = set() + for rid in parsed.ids: + m = re.match(r"^milestone-(\d+)$", rid) + if m is not None: + region_milestones.add(int(m.group(1))) + for n, issues in sorted(issues_by_ms.items()): + if n not in region_milestones: + print( + f"warning: {len(issues)} issue(s) with milestone-{n}-* label " + f"have no region", + file=sys.stderr, + ) + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def _render(parsed: ParsedFile, issues: list[Issue]) -> str: + issues_by_ms = group_by_milestone(issues) + warn_orphan_milestones(parsed, issues_by_ms) + return assemble_file(parsed, issues_by_ms) + + +def run(markdown: Path, repo: str | None, env_prefix: bool) -> Result[str, PipelineError]: + """Compose read → parse → fetch → render; returns the new content.""" + def render_with_repo(text: str) -> Result[str, PipelineError]: + target = repo or parse_repository(text) + if target is None: + return Result.err(PipelineError( + error_type=ErrorType.INVALID_CONFIG, + message=( + "no repository given — add a `**Repository**: owner/name` " + "header to the plan file or pass --repo" + ), + )) + client = GitHubClient(repo=target, env_prefix=env_prefix) + return parse_file(text).and_then( + lambda parsed: client.list_issues().map( + lambda issues: _render(parsed, issues) + ) + ) + + return read_text(markdown).and_then(render_with_repo) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Regenerate the generated regions of GITHUB_PROJECT.md from GitHub.", + ) + parser.add_argument("markdown", type=Path, help="Path to GITHUB_PROJECT.md.") + parser.add_argument("--repo", help="GitHub repo (OWNER/NAME); overrides the file header.") + parser.add_argument( + "--check", + action="store_true", + help="Verify that the file already matches the rendered output; do not write.", + ) + parser.add_argument( + "--no-env-prefix", + action="store_true", + help="Don't prefix `gh` with `env -u GH_TOKEN -u GITHUB_TOKEN`.", + ) + args = parser.parse_args() + + result = run(args.markdown, args.repo, env_prefix=not args.no_env_prefix) + if result.is_err: + print(f"update failed: {result.unwrap_err()}", file=sys.stderr) + return 2 + rendered = result.unwrap() + + if args.check: + existing = args.markdown.read_text(encoding="utf-8") + if existing == rendered: + print(f"OK: {args.markdown} matches rendered output") + return 0 + print(f"FAIL: {args.markdown} differs from rendered output", file=sys.stderr) + print(" (run without --check to update in place)", file=sys.stderr) + return 1 + + write_result = write_text(args.markdown, rendered) + if write_result.is_err: + print(f"write failed: {write_result.unwrap_err()}", file=sys.stderr) + return 2 + print(f"wrote {args.markdown}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/fake_gh/gh b/scripts/tests/fake_gh/gh new file mode 100755 index 0000000..ed3b109 --- /dev/null +++ b/scripts/tests/fake_gh/gh @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +File: scripts/tests/fake_gh/gh + +Description: + A recorded-transcript stand-in for the GitHub CLI, used by the + integration tests. Prepend this directory to PATH and the scripts + under test exercise their real command-construction and JSON-parsing + paths with zero network. + + State lives under $FAKE_GH_STATE: + labels.json / milestones.json / issues.json the fake repository + calls.log one line per invocation + fail_create_issue_at (optional) 1-based index + of an issue-create call + that must fail, for + crash-safety tests + + Only the invocations the gh_project scripts actually make are + implemented; anything else exits 64 loudly so a drifting client + surfaces as a test failure, not a silent success. +""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +STATE = Path(os.environ["FAKE_GH_STATE"]) + + +def load(name: str, default): + path = STATE / name + return json.loads(path.read_text()) if path.exists() else default + + +def save(name: str, data) -> None: + (STATE / name).write_text(json.dumps(data, indent=1)) + + +def bump(name: str) -> int: + path = STATE / name + n = int(path.read_text()) + 1 if path.exists() else 1 + path.write_text(str(n)) + return n + + +def flag_value(args: list[str], flag: str) -> str | None: + return args[args.index(flag) + 1] if flag in args else None + + +def flag_values(args: list[str], flag: str) -> list[str]: + return [args[i + 1] for i, a in enumerate(args) if a == flag] + + +def kv_values(args: list[str], flag: str) -> dict[str, str]: + """Collect `-f key=value` / `-F key=value` style pairs.""" + out: dict[str, str] = {} + for raw in flag_values(args, flag): + key, _, value = raw.partition("=") + out[key] = value + return out + + +def main() -> int: + args = sys.argv[1:] + with (STATE / "calls.log").open("a") as log: + log.write(" ".join(args) + "\n") + + if args[:2] == ["label", "list"]: + print(json.dumps(load("labels.json", []))) + return 0 + + if args[:2] == ["label", "create"]: + labels = load("labels.json", []) + labels.append({ + "name": args[2], + "color": flag_value(args, "--color") or "cccccc", + "description": flag_value(args, "--description") or "", + }) + save("labels.json", labels) + return 0 + + if args[0] == "api" and "milestones" in args[1]: + if "POST" in flag_values(args, "-X"): + fields = kv_values(args, "-f") + milestones = load("milestones.json", []) + number = bump("next_milestone") + milestones.append({ + "title": fields.get("title", ""), + "description": fields.get("description", ""), + "number": number, + "state": "open", + }) + save("milestones.json", milestones) + print(json.dumps({"number": number})) + return 0 + print(json.dumps(load("milestones.json", []))) + return 0 + + if args[:2] == ["issue", "list"]: + print(json.dumps(load("issues.json", []))) + return 0 + + if args[:2] == ["issue", "create"]: + call_index = bump("issue_create_calls") + fail_at = STATE / "fail_create_issue_at" + if fail_at.exists() and call_index == int(fail_at.read_text()): + print("fake gh: injected failure", file=sys.stderr) + return 1 + repo = flag_value(args, "--repo") or "octocat/demo" + issues = load("issues.json", []) + number = bump("next_issue") + milestone_title = flag_value(args, "--milestone") + issues.append({ + "number": number, + "title": flag_value(args, "--title") or "", + "body": flag_value(args, "--body") or "", + "labels": [{"name": n} for n in flag_values(args, "--label")], + "milestone": {"title": milestone_title} if milestone_title else None, + "state": "OPEN", + "assignees": [], + }) + save("issues.json", issues) + print(f"https://github.com/{repo}/issues/{number}") + return 0 + + print(f"fake gh: unimplemented invocation: {' '.join(args)}", file=sys.stderr) + return 64 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_gh_project_lint.py b/scripts/tests/test_gh_project_lint.py new file mode 100644 index 0000000..9f66eac --- /dev/null +++ b/scripts/tests/test_gh_project_lint.py @@ -0,0 +1,129 @@ +""" +File: scripts/tests/test_gh_project_lint.py + +Description: + Tests for the pure structural lint (lint_plan) that `make lint` and + gh_project_populate.py share. No network. +""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _gh_project_lib import lint_plan # noqa: E402 +from test_gh_project_lib import PLAN # noqa: E402 + + +def errors(text: str) -> list[str]: + return [p.message for p in lint_plan(text) if p.severity == "error"] + + +def warnings(text: str) -> list[str]: + return [p.message for p in lint_plan(text) if p.severity == "warning"] + + +class CleanPlan(unittest.TestCase): + def test_fixture_is_clean(self) -> None: + self.assertEqual(errors(PLAN), []) + self.assertEqual(warnings(PLAN), []) + + +class MarkerProblems(unittest.TestCase): + def test_unterminated(self) -> None: + broken = PLAN.replace("\n", "") + self.assertTrue(any("unterminated" in e for e in errors(broken))) + + def test_mismatched(self) -> None: + broken = PLAN.replace( + "", + "", + ) + self.assertTrue(any("mismatched" in e for e in errors(broken))) + + def test_duplicate_region_id(self) -> None: + broken = PLAN.replace( + "", + "", + ).replace( + "", + "", + ) + self.assertTrue(any("duplicate generated-region id" in e + for e in errors(broken))) + + +class ReferenceProblems(unittest.TestCase): + def test_duplicate_issue_id(self) -> None: + broken = PLAN.replace( + "### Issue M2-1a: Shine it more", + "### Issue M2-1: Shine it more", + ) + self.assertTrue(any("duplicate issue ID M2-1" in e for e in errors(broken))) + + def test_issue_with_unknown_milestone(self) -> None: + broken = PLAN.replace( + "### Issue M2-1a: Shine it more", + "### Issue M9-1: Shine it more", + ) + self.assertTrue(any("milestone 9" in e for e in errors(broken))) + + def test_region_with_unknown_milestone(self) -> None: + broken = PLAN.replace( + "", + "", + ).replace( + "", + "", + ) + self.assertTrue(any("region 'milestone-9'" in e for e in errors(broken))) + + def test_issue_references_unlisted_label(self) -> None: + broken = PLAN.replace( + "**Labels:** `milestone-2-polish`\n\nBody of M2-1a.", + "**Labels:** `nonexistent-label`\n\nBody of M2-1a.", + ) + self.assertTrue(any("`nonexistent-label`" in e for e in errors(broken))) + + +class LabelSectionProblems(unittest.TestCase): + def test_malformed_entry(self) -> None: + broken = PLAN.replace( + "- `documentation` (0e8a16) — Documentation changes.", + "- `documentation` (ZZZZZZ) — Bad color.", + ) + self.assertTrue(any("did not parse" in e for e in errors(broken))) + + def test_case_collision_within_plan(self) -> None: + broken = PLAN.replace( + "- `documentation` (0e8a16) — Documentation changes.", + "- `documentation` (0e8a16) — Documentation changes.\n" + "- `Documentation` (0e8a16) — Duplicate modulo case.", + ) + self.assertTrue(any("collide after" in e for e in errors(broken))) + + def test_exact_duplicate(self) -> None: + broken = PLAN.replace( + "- `documentation` (0e8a16) — Documentation changes.", + "- `documentation` (0e8a16) — Documentation changes.\n" + "- `documentation` (0e8a16) — Same again.", + ) + self.assertTrue(any("duplicate label `documentation`" in e + for e in errors(broken))) + + +class Advisories(unittest.TestCase): + def test_missing_repository_is_warning(self) -> None: + broken = PLAN.replace("**Repository**: `octocat/demo`\n", "") + self.assertEqual(errors(broken), []) + self.assertTrue(any("no `**Repository**" in w for w in warnings(broken))) + + def test_duplicate_milestone_number_is_warning(self) -> None: + broken = PLAN.replace("### Milestone 2 — Polish", "### Milestone 1 — Polish") + self.assertTrue(any("more than once" in w for w in warnings(broken))) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_gh_project_update.py b/scripts/tests/test_gh_project_update.py new file mode 100644 index 0000000..cc25bf7 --- /dev/null +++ b/scripts/tests/test_gh_project_update.py @@ -0,0 +1,104 @@ +""" +File: scripts/tests/test_gh_project_update.py + +Description: + Tests for gh_project_update's pure rendering core: issue formatting, + region rebuilding, and manual-prose preservation. No network. +""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import gh_project_update as upd # noqa: E402 +from _gh_project_lib import Issue, parse_file # noqa: E402 +from test_gh_project_lib import PLAN # noqa: E402 + + +def issue(id_: str, title: str, **kw) -> Issue: + return Issue(id=id_, title=title, body=kw.pop("body", "Body."), **kw) + + +class RenderIssue(unittest.TestCase): + def test_open_issue(self) -> None: + out = upd.render_issue( + issue("M1-1", "[M1-1] Set up", gh_number=7, + labels=("milestone-1-core",)) + ) + self.assertIn("### Issue M1-1: Set up (#7)\n", out) + self.assertIn("**Labels**: `milestone-1-core`\n", out) + self.assertNotIn("closed", out) + self.assertNotIn("Assignees", out) + + def test_closed_issue(self) -> None: + out = upd.render_issue( + issue("M1-1", "[M1-1] Set up", gh_number=7, state="closed") + ) + self.assertIn("(#7, closed)", out) + + def test_assignees_rendered_when_present(self) -> None: + out = upd.render_issue( + issue("M1-1", "[M1-1] Set up", gh_number=7, + assignees=("octocat", "hubot")) + ) + self.assertIn("**Assignees**: @octocat, @hubot\n", out) + + def test_empty_body_placeholder(self) -> None: + out = upd.render_issue(issue("M1-1", "[M1-1] Set up", body="", gh_number=7)) + self.assertIn("_(no description on GitHub)_", out) + + +class RenderRegion(unittest.TestCase): + def test_unknown_region_id_fails_loudly(self) -> None: + out = upd.render_region("weird-id", {}) + self.assertIn("no rendering rule", out) + + def test_empty_milestone(self) -> None: + out = upd.render_region("milestone-3", {}) + self.assertIn("no open or closed issues", out) + + def test_parent_before_children(self) -> None: + issues = { + 2: [ + issue("M2-7b", "[M2-7b] Child B", gh_number=3), + issue("M2-7", "[M2-7] Parent", gh_number=1), + issue("M2-7a", "[M2-7a] Child A", gh_number=2), + ] + } + out = upd.render_region("milestone-2", issues) + self.assertLess(out.index("Parent"), out.index("Child A")) + self.assertLess(out.index("Child A"), out.index("Child B")) + + +class AssembleFile(unittest.TestCase): + def test_manuals_preserved_and_regions_rebuilt(self) -> None: + parsed = parse_file(PLAN).unwrap() + issues = { + 1: [issue("M1-1", "[M1-1] Set up the build", gh_number=1, + labels=("milestone-1-core",))], + 2: [issue("M2-1", "[M2-1] Shine it", gh_number=3, state="closed")], + } + out = upd.assemble_file(parsed, issues) + # Manual prose survives byte-for-byte. + for segment in parsed.manuals: + self.assertIn(segment, out) + # Regions were rebuilt from the fake GitHub state. + self.assertIn("### Issue M1-1: Set up the build (#1)", out) + self.assertIn("### Issue M2-1: Shine it (#3, closed)", out) + # Round trip: the output parses to the same structure. + reparsed = parse_file(out).unwrap() + self.assertEqual(reparsed.ids, parsed.ids) + self.assertEqual(reparsed.manuals, parsed.manuals) + + +class StripIdPrefix(unittest.TestCase): + def test_strip(self) -> None: + self.assertEqual(upd.strip_id_prefix("[M1-2] Title"), "Title") + self.assertEqual(upd.strip_id_prefix("No prefix"), "No prefix") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py new file mode 100644 index 0000000..4e263b0 --- /dev/null +++ b/scripts/tests/test_integration.py @@ -0,0 +1,257 @@ +""" +File: scripts/tests/test_integration.py + +Description: + End-to-end tests of gh_project_populate.py and gh_project_update.py + against the recorded fake `gh` in scripts/tests/fake_gh/ — the real + scripts run as subprocesses with the fake on PATH, so command + construction, JSON parsing, write-back, and exit codes are all + exercised with zero network. + + Pinned here, per ualib/agda-algebras#293 item 6 and the design + requirements: + - populate fetches ONE snapshot per run (one `label list`, one + milestones GET, one `issue list`) no matter how many issues the + plan holds; + - a rerun after success creates nothing (idempotency); + - a rerun after a mid-run crash creates only what is missing + (crash-safe write-back); + - label near-collisions are reported, not created; + - update rewrites only the generated regions and follows diff(1) + exit codes for --check. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from test_gh_project_lib import PLAN # noqa: E402 + +TESTS_DIR = Path(__file__).resolve().parent +SCRIPTS_DIR = TESTS_DIR.parent +FAKE_GH_DIR = TESTS_DIR / "fake_gh" + + +class FakeGhHarness(unittest.TestCase): + """Shared setup: a temp plan file and a fresh fake-gh state dir.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + tmp = Path(self._tmp.name) + self.state = tmp / "state" + self.state.mkdir() + self.plan_path = tmp / "GITHUB_PROJECT.md" + self.plan_path.write_text(PLAN, encoding="utf-8") + + def run_script(self, script: str, *args: str) -> subprocess.CompletedProcess: + env = { + **os.environ, + "PATH": f"{FAKE_GH_DIR}:{os.environ['PATH']}", + "FAKE_GH_STATE": str(self.state), + } + return subprocess.run( + [sys.executable, str(SCRIPTS_DIR / script), str(self.plan_path), *args], + env=env, capture_output=True, text=True, timeout=120, + ) + + def populate(self, *args: str) -> subprocess.CompletedProcess: + return self.run_script( + "gh_project_populate.py", "--yes", "--delay", "0", *args + ) + + def update(self, *args: str) -> subprocess.CompletedProcess: + return self.run_script("gh_project_update.py", *args) + + def calls(self, needle: str) -> int: + log = self.state / "calls.log" + if not log.exists(): + return 0 + return sum(needle in line for line in log.read_text().splitlines()) + + def issues_on_fake_github(self) -> list[dict]: + path = self.state / "issues.json" + return json.loads(path.read_text()) if path.exists() else [] + + +class FreshPopulate(FakeGhHarness): + def test_creates_everything_with_one_snapshot(self) -> None: + proc = self.populate() + self.assertEqual(proc.returncode, 0, proc.stderr + proc.stdout) + + # One snapshot per run — not one list call per created entity + # (#293 item 6: this used to be 1 ListIssues per issue create). + self.assertEqual(self.calls("label list"), 1) + self.assertEqual(self.calls("issue list"), 1) + self.assertEqual( + sum("milestones" in line and "POST" not in line + for line in (self.state / "calls.log").read_text().splitlines()), + 1, + ) + + created = self.issues_on_fake_github() + self.assertEqual(len(created), 4) + self.assertEqual( + sorted(i["title"] for i in created), + [ + "[M1-1] Set up the build", + "[M1-2] Write the docs", + "[M2-1] Shine it", + "[M2-1a] Shine it more", + ], + ) + # Milestones attached by title. + m11 = next(i for i in created if i["title"].startswith("[M1-1]")) + self.assertEqual(m11["milestone"], {"title": "1. Core"}) + + # Every heading now records its number (crash-safe write-back), + # including replacing the stale (#42) the fixture carried. + text = self.plan_path.read_text(encoding="utf-8") + for issue in created: + issue_id = issue["title"].split("]")[0][1:] + self.assertIn(f"### Issue {issue_id}: ", text) + self.assertIn(f"(#{issue['number']})", text) + self.assertNotIn("(#42)", text) + + def test_rerun_creates_nothing(self) -> None: + first = self.populate() + self.assertEqual(first.returncode, 0, first.stderr) + creates_after_first = self.calls("issue create") + + second = self.populate() + self.assertEqual(second.returncode, 0, second.stderr) + self.assertEqual(self.calls("issue create"), creates_after_first) + self.assertEqual(len(self.issues_on_fake_github()), 4) + self.assertIn("- exists: issue", second.stdout) + + def test_dry_run_mutates_nothing(self) -> None: + proc = self.populate("--dry-run") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("+ create issue: [M1-1] Set up the build", proc.stdout) + self.assertEqual(self.calls("issue create"), 0) + self.assertEqual(self.calls("label create"), 0) + self.assertEqual(self.issues_on_fake_github(), []) + self.assertEqual(self.plan_path.read_text(encoding="utf-8"), PLAN) + + +class CrashSafety(FakeGhHarness): + def test_interrupted_run_reruns_without_duplicates(self) -> None: + # The third issue-create call dies mid-run. Populate continues + # past a failed creation (a single bad issue must not abort the + # rest), so the run ends with 3 of 4 issues live and exit 1. + (self.state / "fail_create_issue_at").write_text("3") + first = self.populate() + self.assertEqual(first.returncode, 1, first.stdout + first.stderr) + self.assertEqual(len(self.issues_on_fake_github()), 3) + + # Every number GitHub handed out was persisted immediately — + # BEFORE the next mutation — so all 3 are already on disk. + text = self.plan_path.read_text(encoding="utf-8") + self.assertEqual(text.count("(#"), 3) + + # Heal the fake and rerun: exactly the one missing issue is + # created; nothing that succeeded before is duplicated. + (self.state / "fail_create_issue_at").unlink() + second = self.populate() + self.assertEqual(second.returncode, 0, second.stderr) + titles = [i["title"] for i in self.issues_on_fake_github()] + self.assertEqual(len(titles), 4) + self.assertEqual(len(set(titles)), 4, f"duplicates: {titles}") + + +class LabelReconciliation(FakeGhHarness): + def test_near_collision_is_reported_not_created(self) -> None: + (self.state / "labels.json").write_text(json.dumps([ + {"name": "Milestone-1-Core", "color": "ffffff", "description": ""}, + ])) + proc = self.populate("--dry-run") + self.assertEqual(proc.returncode, 1, proc.stdout) + self.assertIn("collision", proc.stdout) + self.assertIn("Milestone-1-Core", proc.stdout) + + live = self.populate() + self.assertEqual(live.returncode, 1) + labels = json.loads((self.state / "labels.json").read_text()) + # The plan's `milestone-1-core` was NOT created alongside the + # existing case-variant; the other two labels were. + self.assertEqual( + sorted(l["name"] for l in labels), + ["Milestone-1-Core", "documentation", "milestone-2-polish"], + ) + + def test_existing_label_never_overwritten(self) -> None: + (self.state / "labels.json").write_text(json.dumps([ + {"name": "documentation", "color": "000000", + "description": "Different color"}, + ])) + proc = self.populate() + self.assertEqual(proc.returncode, 0, proc.stderr) + docs = [l for l in json.loads((self.state / "labels.json").read_text()) + if l["name"] == "documentation"] + self.assertEqual(docs, [{"name": "documentation", "color": "000000", + "description": "Different color"}]) + self.assertIn("not overwritten", proc.stdout) + + +class UpdateRoundTrip(FakeGhHarness): + def test_update_rewrites_regions_and_check_tracks_drift(self) -> None: + self.assertEqual(self.populate().returncode, 0) + + # Freshly populated file is NOT current yet: update canonicalizes + # the regions (bodies now live on GitHub). + stale = self.update("--check") + self.assertEqual(stale.returncode, 1, stale.stderr) + + wrote = self.update() + self.assertEqual(wrote.returncode, 0, wrote.stderr) + text = self.plan_path.read_text(encoding="utf-8") + self.assertIn("## How to use", text) # manual prose survives + self.assertIn("### Issue M1-1: Set up the build (#", text) + + current = self.update("--check") + self.assertEqual(current.returncode, 0, current.stderr) + + # Close an issue on "GitHub"; --check flags drift; update heals it. + issues = self.issues_on_fake_github() + issues[0]["state"] = "CLOSED" + (self.state / "issues.json").write_text(json.dumps(issues)) + drifted = self.update("--check") + self.assertEqual(drifted.returncode, 1) + self.assertEqual(self.update().returncode, 0) + self.assertIn(", closed)", self.plan_path.read_text(encoding="utf-8")) + + def test_update_failure_is_exit_2_not_drift(self) -> None: + proc = self.run_script( + "gh_project_update.py", "--check", "--repo", "", + ) + # An empty --repo falls back to the file header, so break it harder: + # point at a plan with no header and pass no repo. + stripped = PLAN.replace("**Repository**: `octocat/demo`\n", "") + self.plan_path.write_text(stripped, encoding="utf-8") + proc = self.run_script("gh_project_update.py", "--check") + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("no repository", proc.stderr) + + +class PopulateRefusesBrokenPlans(FakeGhHarness): + def test_lint_gate(self) -> None: + self.plan_path.write_text( + PLAN.replace("### Issue M2-1a: Shine it more", + "### Issue M2-1: Shine it more"), + encoding="utf-8", + ) + proc = self.populate("--dry-run") + self.assertEqual(proc.returncode, 2, proc.stdout) + self.assertIn("duplicate issue ID", proc.stderr) + self.assertEqual(self.calls("issue list"), 0) # refused before fetching + + +if __name__ == "__main__": + unittest.main() From 2812c0b05814acdb565d3ad4814e772c55172ff6 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Thu, 13 Aug 2026 22:05:48 -0600 Subject: [PATCH 4/9] Add Makefile, worked-example plan, and scripts/VERSION The Makefile is the primary UX: make help (default), populate-dry, populate, update, update-check, lint, test. REPO defaults to the plan file's **Repository**: header; NO_ENV_PREFIX passes through for environments that authenticate via GH_TOKEN/GITHUB_TOKEN. docs/GITHUB_PROJECT.md is a deliberately small worked example (2 milestones, 4 issues) that a consumer replaces wholesale; every construct the parsers understand appears exactly once, including the load-bearing milestone-N-* labels, the hand-authored mermaid dependency graph, and the generated-region markers. It lints clean. scripts/VERSION (0.1.0) is the vendoring marker consumers use to tell which release of the scripts directory they carry. AI-assisted: Claude Fable 5 (Anthropic) --- Makefile | 53 +++++++++++++ docs/GITHUB_PROJECT.md | 174 +++++++++++++++++++++++++++++++++++++++++ scripts/VERSION | 1 + 3 files changed, 228 insertions(+) create mode 100644 Makefile create mode 100644 docs/GITHUB_PROJECT.md create mode 100644 scripts/VERSION diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..09040b4 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +# File: Makefile +# +# The primary UX for this template. `make help` (the default) lists the +# targets; `make populate` / `make update` are the two directions of the +# contract stated in the README. +# +# Variables: +# PLAN plan file (default: docs/GITHUB_PROJECT.md) +# REPO owner/name override (default: the plan file's +# **Repository**: header) +# PYTHON interpreter (default: python3) +# NO_ENV_PREFIX set (to anything) wherever gh authenticates *through* +# GH_TOKEN/GITHUB_TOKEN — GitHub Actions included; by +# default the scripts strip those variables so they +# cannot shadow a keychain-stored token. + +PYTHON ?= python3 +PLAN ?= docs/GITHUB_PROJECT.md +REPO ?= + +FLAGS := $(if $(REPO),--repo $(REPO)) $(if $(NO_ENV_PREFIX),--no-env-prefix) + +.PHONY: help populate-dry populate update update-check lint test guard-gh +.DEFAULT_GOAL := help + +help: ## Show this help + @grep -hE '^[a-zA-Z][a-zA-Z0-9_-]*:.*## ' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*## "} {printf " \033[1m%-14s\033[0m %s\n", $$1, $$2}' + +populate-dry: guard-gh ## Preview what populate would create (no mutations) + $(PYTHON) scripts/gh_project_populate.py $(PLAN) $(FLAGS) --dry-run + +populate: guard-gh ## Create labels, milestones, issues on GitHub from $(PLAN) + $(PYTHON) scripts/gh_project_populate.py $(PLAN) $(FLAGS) + +update: guard-gh ## Rewrite $(PLAN)'s generated regions from live GitHub state + $(PYTHON) scripts/gh_project_update.py $(PLAN) $(FLAGS) + +update-check: guard-gh ## Report whether $(PLAN) is stale (exit 1) without writing + $(PYTHON) scripts/gh_project_update.py $(PLAN) $(FLAGS) --check + +lint: ## Validate $(PLAN)'s structure (no network) + $(PYTHON) scripts/gh_project_lint.py $(PLAN) + +test: ## Run the test suite (no network; uses a recorded fake gh) + $(PYTHON) -m unittest discover -s scripts/tests + +guard-gh: + @command -v gh >/dev/null || { \ + echo "error: the GitHub CLI (gh) is required for this target."; \ + echo " https://cli.github.com/ (or: nix develop)"; \ + exit 1; \ + } diff --git a/docs/GITHUB_PROJECT.md b/docs/GITHUB_PROJECT.md new file mode 100644 index 0000000..45d967c --- /dev/null +++ b/docs/GITHUB_PROJECT.md @@ -0,0 +1,174 @@ + + +# Example — GitHub Project Roadmap + +**Project Title**: Timeline — a small command-line time tracker + +**Repository**: `your-github-user/your-repo` + +--- + +## Summary + +Build a tiny command-line time tracker in two milestones: a working core +(start/stop/report on local JSON storage), then polish (colored output, +shell completion, a published release). The example is deliberately +small — two milestones, four issues — so that every construct the +tooling understands appears here exactly once. + +--- + +## Labels + +The `milestone-N-*` labels are load-bearing: populate attaches them, and +update uses them to decide which generated region an issue belongs to +(they survive milestone renames, unlike the milestone title). The other +labels are ordinary topic labels; add your own. + +- `milestone-1-core` (0075ca) — Milestone 1: Working core. +- `milestone-2-polish` (5319e7) — Milestone 2: Polish and release. +- `documentation` (0e8a16) — Documentation changes. +- `good first issue` (7057ff) — Small and self-contained. + +--- + +## Milestones + +### Milestone 1 — Working core + +**Description:** + +Stand up the smallest useful tool: `timeline start `, +`timeline stop`, and `timeline report` over a local JSON file. No +configuration, no dependencies beyond the standard library. + +**Exit criterion:** + +A fresh checkout runs `timeline start x && timeline stop && timeline +report` and prints one correctly-summed entry; CI runs the test suite on +every pull request. + +--- + +### Milestone 2 — Polish and release + +**Description:** + +Make it pleasant: colored report output, shell completion, and a tagged +v1.0 release with install instructions. + +**Exit criterion:** + +`pipx install` from the tagged release works on a clean machine and +`timeline report` output is readable at a glance. + +--- + +### Milestone dependencies + +Hand-authored (machine generation would require a dependency convention +nobody has committed to); update never touches it. + +```mermaid +graph TD + M1["M1 — Working core"] + M2["M2 — Polish and release"] + M1 --> M2 +``` + +--- + +## Issues + +Each issue below carries its milestone, labels, and a body ready for +GitHub. On first populate, the `[MN-k]` ID in each heading becomes a +`[MN-k]` title prefix on GitHub — the join key that keeps re-runs +idempotent — and the new issue number is written back into the heading +as a `(#N)` suffix. + +## Milestone 1 — Working core + + + +### Issue M1-1: Storage layer — append-only JSON event log + +**Labels:** `milestone-1-core` +**Milestone:** 1. Working core + +Design and implement the on-disk format: one JSON object per line +(`{"task": ..., "start": ..., "stop": ...}`), appended atomically. + +Acceptance criteria: + +- concurrent `start` invocations cannot corrupt the file +- a malformed trailing line (crash artifact) is detected and reported + +--- + +### Issue M1-2: CLI commands — start, stop, report + +**Labels:** `milestone-1-core` +**Milestone:** 1. Working core + +`start ` opens an interval, `stop` closes the open interval, +`report` sums per-task durations for today. Refuse to `start` twice +without an intervening `stop`. + +--- + + + +## Milestone 2 — Polish and release + + + +### Issue M2-1: Colored, aligned report output + +**Labels:** `milestone-2-polish` +**Milestone:** 2. Polish and release + +Align columns, add duration bars, and colorize over-budget tasks. +Respect `NO_COLOR`. + +--- + +### Issue M2-2: Document the storage format + +**Labels:** `milestone-2-polish`, `documentation`, `good first issue` +**Milestone:** 2. Polish and release + +Write `docs/FORMAT.md` describing the event-log schema and its +compatibility guarantees. + +--- + + + +## How to use this file + +1. Replace the plan above with your project (keep the structure), set + `**Repository**:` to your `owner/name`, and run `make lint`. +2. `make populate-dry` to preview, then `make populate` to create + everything on GitHub. Re-running is safe: existing labels, + milestones, and issues are skipped. +3. From then on, work the issues on GitHub as usual and run + `make update` whenever you want this file to reflect reality. Only + the marked regions are rewritten; prose is preserved byte-for-byte. diff --git a/scripts/VERSION b/scripts/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/scripts/VERSION @@ -0,0 +1 @@ +0.1.0 From bb3b1d1a18ffc348075ba72480427a376810b28a Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Thu, 13 Aug 2026 22:07:52 -0600 Subject: [PATCH 5/9] Add repo CI and the opt-in freshness workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci.yml: make lint + make test on Python 3.11 and 3.12 (setup-python, deliberately not Nix — the flake is a convenience, never a requirement), shellcheck over scripts/ci/, and ruff (E9,F only) over scripts/. project-plan-update.yml runs Mondays and on workflow_dispatch. All logic lives in scripts/ci/project-plan-update.sh (runnable, linted, tested locally; the YAML holds only schedule, permissions, and the call). Shipped default is REPORT-ONLY: drift lands in the step summary and the job always succeeds — no bot commits. Setting the repository variable PROJECT_PLAN_AUTO_UPDATE to 'pr' or 'commit' arms the automatic update. The script keeps gh_project_update.py's diff(1) exit codes distinct so 'the plan drifted' is never conflated with 'the check could not run' (all three paths exercised locally via the UPDATE override). Also: drop unused imports and a placeholder-less f-string that the vendored pipeline_types carried from its ancestors (ruff E9,F now gates them). AI-assisted: Claude Fable 5 (Anthropic) --- .github/workflows/ci.yml | 52 +++++++++ .github/workflows/project-plan-update.yml | 45 ++++++++ scripts/_utils/pipeline_types.py | 7 +- scripts/ci/project-plan-update.sh | 131 ++++++++++++++++++++++ 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/project-plan-update.yml create mode 100755 scripts/ci/project-plan-update.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76ecaf4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: ci + +# The template's own CI: structural lint of the example plan, the test +# suite (offline; a recorded fake gh), shell lint, and a minimal Python +# lint. Deliberately not Nix-based — consumers must be able to use this +# template with nothing but python3, gh, and make (the flake is a +# convenience, never a requirement). + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + checks: + runs-on: ubuntu-latest + strategy: + matrix: + # 3.11 is the documented floor; 3.12 the current default. + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Structural lint of the example plan + run: make lint + + - name: Test suite + run: make test + + static: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Shellcheck + run: shellcheck scripts/ci/*.sh + + - name: Python lint (errors and pyflakes only) + run: | + pip install ruff + ruff check --select E9,F scripts/ diff --git a/.github/workflows/project-plan-update.yml b/.github/workflows/project-plan-update.yml new file mode 100644 index 0000000..d25ccd0 --- /dev/null +++ b/.github/workflows/project-plan-update.yml @@ -0,0 +1,45 @@ +name: project-plan-update + +# Keeps docs/GITHUB_PROJECT.md fresh. By default this only REPORTS +# drift to the run's step summary — no bot commits. Opt in to automatic +# updates by setting the repository variable PROJECT_PLAN_AUTO_UPDATE to +# `pr` (open/refresh a pull request) or `commit` (push directly); see +# scripts/ci/project-plan-update.sh, which holds all of the logic and is +# runnable and testable locally. Delete this file (or disable the +# workflow in the Actions tab) if you want no scheduled runs at all. + +on: + schedule: + - cron: "0 6 * * 1" # Mondays, 06:00 UTC + workflow_dispatch: + +# contents:write and pull-requests:write are exercised only when the +# PROJECT_PLAN_AUTO_UPDATE variable arms the commit/pr modes; the +# default report-only path reads issues and writes nothing. +permissions: + contents: write + issues: read + pull-requests: write + +jobs: + freshness: + name: Is the plan current? + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Check, and update if armed + env: + GH_TOKEN: ${{ github.token }} + PROJECT_PLAN_AUTO_UPDATE: ${{ vars.PROJECT_PLAN_AUTO_UPDATE }} + # Required here. The update script strips GH_TOKEN and + # GITHUB_TOKEN by default, which is exactly how Actions + # authenticates; without this every gh call exits with a + # message pointing at `gh auth login`, i.e. away from the + # real cause. + NO_ENV_PREFIX: "1" + run: scripts/ci/project-plan-update.sh diff --git a/scripts/_utils/pipeline_types.py b/scripts/_utils/pipeline_types.py index e0d5250..14a27b1 100644 --- a/scripts/_utils/pipeline_types.py +++ b/scripts/_utils/pipeline_types.py @@ -27,11 +27,10 @@ from dataclasses import dataclass, field from pathlib import Path from typing import ( - TypeVar, Generic, Optional, Union, Dict, List, Tuple, - NamedTuple, Callable, Any, Protocol, runtime_checkable + TypeVar, Generic, Optional, Dict, List, Tuple, + Callable, Any, Protocol, runtime_checkable ) from enum import Enum, auto -import json # ============================================================================= # Generic Type Variables @@ -497,4 +496,4 @@ def divide(x: float, y: float) -> Result[float, str]: print(f"Files processed: {new_state.statistics.files_processed}") print(f"Current stage: {new_state.stage}") - print(f"✅ Type system working correctly!") + print("✅ Type system working correctly!") diff --git a/scripts/ci/project-plan-update.sh b/scripts/ci/project-plan-update.sh new file mode 100755 index 0000000..cce12cc --- /dev/null +++ b/scripts/ci/project-plan-update.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# File: scripts/ci/project-plan-update.sh +# +# Keep docs/GITHUB_PROJECT.md fresh from GitHub Actions. +# +# This lives here rather than inside a workflow `run:` block because +# shell in YAML cannot be run locally, cannot be linted, and cannot be +# tested; the workflow holds only what genuinely must be there (when it +# runs, what it may access, and what it calls). +# +# Three modes, selected by $PROJECT_PLAN_AUTO_UPDATE (unset, "commit", +# or "pr"): +# +# (unset) Report only. Drift is written to the step summary; the +# job always succeeds. This is the shipped default: a +# template consumer gets a weekly freshness signal and no +# bot commits unless they explicitly arm one of the modes +# by setting the repository variable. +# commit Run the update and push the result straight to the +# current branch. +# pr Run the update on a fixed branch (project-plan-update) +# and open a pull request (or update the open one). +# Requires "Allow GitHub Actions to create and approve pull +# requests" in the repository's Actions settings. +# +# The distinction that matters is drift versus a check that never ran. +# gh_project_update.py follows diff(1) — 0 current, 1 differs, 2 failed +# — and reporting "the plan has drifted" when the real problem was an +# expired token sends you to fix the wrong thing. +# +# Environment: +# REPO owner/name (default: this repository) +# PLAN plan file (default: docs/GITHUB_PROJECT.md) +# PROJECT_PLAN_AUTO_UPDATE "", commit, or pr +# NO_ENV_PREFIX set to pass --no-env-prefix; required +# wherever gh authenticates through +# GH_TOKEN/GITHUB_TOKEN, GitHub Actions +# included +# UPDATE the update command, overridable so the +# reporting logic can be exercised without +# the GitHub API +# +# Writes to $GITHUB_STEP_SUMMARY when set, otherwise to stdout, so +# running it locally shows exactly what CI would report. Exit is 0 on +# every advisory path; only a failed commit/pr action exits nonzero. + +# The backticks throughout the printf strings are markdown code spans +# destined for the step summary, not command substitutions. +# shellcheck disable=SC2016 + +set -uo pipefail + +REPO="${REPO:-${GITHUB_REPOSITORY:-}}" +PLAN="${PLAN:-docs/GITHUB_PROJECT.md}" +MODE="${PROJECT_PLAN_AUTO_UPDATE:-}" +UPDATE="${UPDATE:-python3 scripts/gh_project_update.py}" +SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/stdout}" +BRANCH="project-plan-update" + +flags=() +[ -n "${REPO}" ] && flags+=(--repo "$REPO") +[ -n "${NO_ENV_PREFIX:-}" ] && flags+=(--no-env-prefix) + +# ${flags[@]+...} guards the empty-array expansion under `set -u`, which +# is an error on bash before 4.4 (still the system bash on macOS). +out="$($UPDATE "$PLAN" --check ${flags[@]+"${flags[@]}"} 2>&1)" +code=$? + +case "$code" in + 0) + printf '`%s` is current.\n' "$PLAN" >> "$SUMMARY" + exit 0 + ;; + 1) + : # drift — handled below according to $MODE + ;; + *) + { + printf '### The freshness check could not run\n\n' + printf 'This is **not** drift: the check itself failed with exit code' + printf ' %s, so whether the plan is current is unknown.\n' "$code" + printf 'Usually authentication or a GitHub API error.\n\n' + printf '```\n%s\n```\n' "$out" + } >> "$SUMMARY" + exit 0 + ;; +esac + +run_update() { + $UPDATE "$PLAN" ${flags[@]+"${flags[@]}"} +} + +case "$MODE" in + commit) + run_update || exit 1 + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "$PLAN" + git commit -m "Update $PLAN from live GitHub state" || exit 1 + git push || exit 1 + printf '### `%s` had drifted\n\nUpdated and pushed directly (mode: commit).\n' "$PLAN" >> "$SUMMARY" + ;; + pr) + run_update || exit 1 + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$BRANCH" + git add "$PLAN" + git commit -m "Update $PLAN from live GitHub state" || exit 1 + git push --force origin "$BRANCH" || exit 1 + if [ -z "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then + gh pr create \ + --head "$BRANCH" \ + --title "Update $PLAN from live GitHub state" \ + --body "Automated freshness update: the generated regions of \`$PLAN\` drifted from live GitHub state. Only content between BEGIN/END GENERATED markers changes." \ + || exit 1 + fi + printf '### `%s` had drifted\n\nOpened/updated the `%s` pull request (mode: pr).\n' "$PLAN" "$BRANCH" >> "$SUMMARY" + ;; + *) + { + printf '### `%s` has drifted\n\n' "$PLAN" + printf 'Run `make update` locally and commit the result — or set the\n' + printf 'repository variable `PROJECT_PLAN_AUTO_UPDATE` to `pr` or\n' + printf '`commit` to let this workflow do it for you.\n' + } >> "$SUMMARY" + ;; +esac + +exit 0 From 82f34eb6ec5c0dc24a120b3b89e311e54d06cef6 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Fri, 14 Aug 2026 02:50:25 -0600 Subject: [PATCH 6/9] Add flake.nix, committed Claude skills, CLAUDE.md, and the full README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit flake.nix: a devShell (python3, gh, gnumake) so a machine with only Nix can run the tooling. Strictly optional — CI uses setup-python and the README documents the plain path first. Verified: nix develop provides python3/gh/make. .claude/skills/: committed config as product, for template consumers. project-setup covers the bootstrap (author plan -> lint -> dry-run -> populate -> update); roadmap-upkeep covers steady state (update/ update-check, adding issues/milestones to a live project, arming the freshness workflow). CLAUDE.md states the structure/state contract and its editing corollaries. README: quickstart from 'Use this template' through the steady-state loop, the opt-in freshness story (PROJECT_PLAN_AUTO_UPDATE=pr|commit), the one-liner re-vendor upgrade path pinned to scripts/VERSION, the Nix path, layout, and design notes. AI-assisted: Claude Fable 5 (Anthropic) --- .claude/skills/project-setup/SKILL.md | 63 +++++++++ .claude/skills/roadmap-upkeep/SKILL.md | 61 +++++++++ CLAUDE.md | 49 +++++++ README.md | 171 ++++++++++++++++++++++--- flake.lock | 27 ++++ flake.nix | 26 ++++ 6 files changed, 379 insertions(+), 18 deletions(-) create mode 100644 .claude/skills/project-setup/SKILL.md create mode 100644 .claude/skills/roadmap-upkeep/SKILL.md create mode 100644 CLAUDE.md create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.claude/skills/project-setup/SKILL.md b/.claude/skills/project-setup/SKILL.md new file mode 100644 index 0000000..c6528bc --- /dev/null +++ b/.claude/skills/project-setup/SKILL.md @@ -0,0 +1,63 @@ +--- +name: project-setup +description: Set up a new GitHub project from docs/GITHUB_PROJECT.md — author the plan (milestones, labels, issues), validate it, and push it to GitHub with make populate. Use when starting a project from this template, when the user asks to "create the roadmap", "set up the project board", "populate GitHub", or when docs/GITHUB_PROJECT.md still contains the shipped example. +--- + +# Set up a new project from GITHUB_PROJECT.md + +`docs/GITHUB_PROJECT.md` is the single source of truth for project +STRUCTURE (milestones, labels, issue definitions, prose, dependency +graphs). GitHub owns STATE (issue numbers, open/closed, assignees). +This skill covers the one-time bootstrap: author the file, then push it +to GitHub. + +## Procedure + +1. **Author the plan** by replacing the worked example in + `docs/GITHUB_PROJECT.md`, keeping every structural convention: + - `**Repository**: \`owner/name\`` header line (this is where the + scripts learn the target repo; no `--repo` flag needed). + - `## Labels` section, one entry per line: + `` - `name` (RRGGBB) — Description ``. Include one + `milestone-N-` label per milestone — these are load-bearing + (update uses them to place issues into regions). + - `## Milestones` section with `### Milestone N — Title` blocks, each + carrying `**Description:**` and `**Exit criterion:**`. + - Per-milestone issue sections wrapped in markers: + `` ... ``, + containing `### Issue MN-k: Title` headings with `**Labels:**` + and `**Milestone:**` metadata lines, separated by `---`. + - Optional but valuable: a hand-authored mermaid dependency graph. +2. **Validate** (no network): `make lint` — fix every error it reports. + Warnings are advisories. +3. **Preview**: `make populate-dry` — read the plan it prints. `+` + lines will be created; `-` lines already exist; `!` lines are label + collisions you must resolve first (rename in the plan or on GitHub — + the tool never creates a near-duplicate label scheme). +4. **Push**: `make populate` (append `--` nothing; it prompts before + creating; use `python3 scripts/gh_project_populate.py docs/GITHUB_PROJECT.md --yes` + for non-interactive runs). Issue numbers are written back into the + headings as `(#N)` suffixes as each issue is created. +5. **Canonicalize**: `make update` — rewrites the generated regions from + GitHub so the file and GitHub agree exactly from day one. + +## Safety properties you can rely on + +- Re-running `make populate` never duplicates: existing labels (by + name), milestones (by title), and issues (by `[MN-k]` title prefix or + recorded `(#N)`) are skipped. +- An interrupted populate is rerun-safe: every created issue's number is + persisted to the file before the next API call. +- Existing labels are never overwritten, even when the plan disagrees + on color or description — differences are reported for a human. + +## Pitfalls + +- `gh` must be authenticated (`gh auth status`); the target repo must + already exist on GitHub. +- In environments that authenticate via `GH_TOKEN`/`GITHUB_TOKEN` + (GitHub Actions), set `NO_ENV_PREFIX=1` on the make invocation — by + default the scripts strip those variables to stop them shadowing a + keychain token. +- Never remove the `[MN-k]` prefix from issue titles on GitHub; it is + the join key for idempotency and for update. diff --git a/.claude/skills/roadmap-upkeep/SKILL.md b/.claude/skills/roadmap-upkeep/SKILL.md new file mode 100644 index 0000000..db6c59f --- /dev/null +++ b/.claude/skills/roadmap-upkeep/SKILL.md @@ -0,0 +1,61 @@ +--- +name: roadmap-upkeep +description: Keep docs/GITHUB_PROJECT.md current with GitHub — pull live issue state into the file (make update), check for drift, add new issues or milestones to an already-populated project, and interpret the freshness workflow. Use for "update the roadmap", "is the plan current", "sync the project file", "add an issue to the plan", or after issues were closed/edited on GitHub. +--- + +# Keep the roadmap current + +After bootstrap (see the project-setup skill), GitHub owns issue STATE; +`docs/GITHUB_PROJECT.md` mirrors it inside marked regions and owns the +surrounding prose. Two commands, two directions: + +| Command | Direction | Rewrites | +| --------------- | ------------- | --------------------------------- | +| `make update` | GitHub → file | only the BEGIN/END GENERATED regions | +| `make populate` | file → GitHub | nothing in the file except `(#N)` write-backs | + +## Routine refresh + +- `make update-check` — is the file current? Exit 0 = current, 1 = + drifted, 2 = the check itself failed (auth, network — NOT drift; fix + the cause before trusting any answer). +- `make update` — rewrite the generated regions from live GitHub state. + Hand-written prose outside the markers is preserved byte-for-byte. + Commit the result. + +## Adding work to a live project + +1. New issue in an existing milestone: add a + `### Issue MN-k: Title` block (next free k) inside that milestone's + region, with `**Labels:**` including `milestone-N-`. Fan-out + sub-tickets of issue MN-k are `MN-ka`, `MN-kb`, ... +2. New milestone: add the `### Milestone N — Title` block, a + `milestone-N-` label entry, and a new + `` region for its issues. +3. `make lint`, then `make populate-dry`, then `make populate` — + existing entities are skipped, only the additions are created. +4. `make update` to canonicalize. + +Issue bodies of already-created issues are edited ON GITHUB (the region +mirror would overwrite in-file edits on the next update). Prose outside +regions is edited in the file. + +## The scheduled freshness workflow + +`.github/workflows/project-plan-update.yml` runs Mondays (and on +manual dispatch). By default it only REPORTS drift in the run's step +summary. To let it act, set the repository variable +`PROJECT_PLAN_AUTO_UPDATE` to `pr` (opens/refreshes a pull request — +requires "Allow GitHub Actions to create and approve pull requests" in +the repo's Actions settings) or `commit` (pushes directly). Delete the +workflow file if no scheduled runs are wanted. + +## Reading update's output + +- `### Issue M1-2: Title (#42)` — open issue #42. +- `### Issue M1-2: Title (#42, closed)` — closed on GitHub. +- `**Assignees**: @login` — present only when assigned. +- `_(no open or closed issues with `milestone-N-*` label)_` — the + region exists but no issue carries its label yet. +- A `no rendering rule` comment — the region id is not `milestone-N`; + fix the marker id. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4a631c5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,49 @@ +# CLAUDE.md + +This repository is a **template** for running a GitHub project roadmap +from one markdown file. If you are reading this inside a project +created from the template, the file to care about is +`docs/GITHUB_PROJECT.md`. + +## The contract (one source of truth per concern) + +| Concern | Source of truth | Who edits it | +| --- | --- | --- | +| STRUCTURE: milestones, labels, issue definitions, prose, dependency graphs | `docs/GITHUB_PROJECT.md` | humans and Claude, in the file | +| STATE: issue numbers, open/closed, assignees | GitHub | the normal GitHub workflow | + +Two commands move information between them: + +- `make populate` — file → GitHub. Creates labels, milestones, issues; + writes each new issue number back into the file immediately. + Idempotent: re-running skips everything that already exists. +- `make update` — GitHub → file. Rewrites ONLY the regions between + `` / `` + markers; hand-written prose is preserved byte-for-byte. + +Corollaries for editing: + +- Prose outside the markers: edit in the file, freely. +- Issue bodies after bootstrap: edit ON GITHUB (in-file edits inside a + generated region are overwritten by the next `make update`). +- Never strip the `[MN-k]` prefix from issue titles on GitHub — it is + the join key for idempotency and for update. + +## Working here + +- `make help` lists every target. `make lint` (structure, no network) + and `make test` (offline suite against a recorded fake `gh`) must + pass before any PR. +- Two committed skills cover the workflows: `project-setup` (bootstrap a + new project from the plan file) and `roadmap-upkeep` (keep the file + and GitHub in sync, add issues/milestones to a live project). Prefer + them over improvising. +- Python here follows a functional house style: total functions, + `Result`-based error handling (never exceptions for control flow), + frozen dataclasses, pure parsing/planning core with `gh` calls at the + edges, a `File:` header docstring in every file, and a test for every + pure function. Match it. +- The scripts directory is self-contained on purpose (vendored `_utils`, + `scripts/VERSION`): downstream projects re-vendor it wholesale. Do + not add imports that reach outside `scripts/`, and bump + `scripts/VERSION` when its behavior changes. diff --git a/README.md b/README.md index 3b3c0a1..87d64d6 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,21 @@ # github-project -A template for creating and working on GitHub projects, especially (though -not exclusively) useful for AI-assisted projects. +A template for creating and working on GitHub projects, especially +(though not exclusively) useful for AI-assisted projects. -One markdown file — `docs/GITHUB_PROJECT.md` — holds your roadmap: milestones, -labels, and issues, written by hand (or by your AI assistant) before anything -exists on GitHub. Two commands keep that file and GitHub in sync, in opposite -directions: +One markdown file — [`docs/GITHUB_PROJECT.md`](docs/GITHUB_PROJECT.md) — +holds your roadmap: milestones, labels, and issues, written by hand (or +by your AI assistant) before anything exists on GitHub. Two commands +keep that file and GitHub in sync, in opposite directions: | Command | Direction | What moves | | --------------- | ------------- | ----------------------------------------------------------------- | | `make populate` | file → GitHub | creates labels, milestones, issues; writes issue numbers back into the file | | `make update` | GitHub → file | rewrites the *generated regions* with live issue state; hand-written prose is never touched | -("Update" names the user's intent — bring `docs/GITHUB_PROJECT.md` up to -date — not a push to GitHub. The push direction is `populate`.) +("Update" names the user's intent — bring `docs/GITHUB_PROJECT.md` up +to date — not a push to GitHub. The push direction is `populate`.) ## The contract @@ -27,15 +27,150 @@ One source of truth per concern: | **Structure**: milestones, labels, issue definitions, prose, dependency graphs | `docs/GITHUB_PROJECT.md` | humans (and Claude) | | **State**: issue numbers, open/closed, assignees | GitHub | GitHub UI / normal workflow | -`populate` pushes structure to GitHub once, at bootstrap; it is idempotent, so -re-running it never duplicates anything. `update` pulls state back into the -file for as long as the project lives, rewriting only the regions between -`` / `` markers. -After bootstrap, issue bodies live on GitHub like any other issue; the -generated regions mirror them. +`populate` pushes structure to GitHub once, at bootstrap; it is +idempotent, so re-running it never duplicates anything. `update` pulls +state back into the file for as long as the project lives, rewriting +only the regions between `` / +`` markers. After bootstrap, issue bodies +live on GitHub like any other issue; the generated regions mirror them. -## Status +## Quickstart -Under construction — the scripts, Makefile, worked example, and CI are landing -via pull request. This README will grow a quickstart once every command in it -has been executed as written. +Requirements: Python 3.11+, GNU make, and the +[GitHub CLI](https://cli.github.com/) authenticated with write access to +your repository (`gh auth status`). No third-party Python packages. +(Alternatively, [use Nix](#the-nix-path) and skip installing any of +that.) + +1. **Create your repository from this template** (the "Use this + template" button on GitHub, or): + + ```sh + gh repo create you/your-project --template williamdemeo/github-project --private --clone + cd your-project + ``` + +2. **Author your plan.** Replace the worked example in + `docs/GITHUB_PROJECT.md` with your milestones, labels, and issues — + keep the structure (the example demonstrates every construct exactly + once). Set the header line to your repository: + + ```markdown + **Repository**: `you/your-project` + ``` + +3. **Validate and preview** (lint needs no network; the dry run makes + read-only API calls): + + ```sh + make lint + make populate-dry + ``` + +4. **Push the structure to GitHub** (prompts before creating): + + ```sh + make populate + ``` + + Each created issue's number is written back into the file + immediately, as a `(#N)` suffix on its heading — commit the result. + +5. **From then on**: work the issues on GitHub as usual, and run + + ```sh + make update + ``` + + whenever you want the file to reflect reality (state flips like + `(#7)` → `(#7, closed)`, assignees, edited bodies). Commit the + result. `make update-check` tells you whether that is needed + (exit 0 current / 1 drifted / 2 the check itself failed). + +`make help` lists every target. + +## Staying fresh automatically (opt-in) + +The repository ships a scheduled workflow +(`.github/workflows/project-plan-update.yml`, Mondays + manual +dispatch) that by default only **reports** drift in the run's step +summary — it never commits. To let it act, set the repository variable +`PROJECT_PLAN_AUTO_UPDATE` (Settings → Secrets and variables → Actions +→ Variables) to: + +- `pr` — run `make update` and open (or refresh) a pull request. Also + enable "Allow GitHub Actions to create and approve pull requests" in + Settings → Actions. +- `commit` — run `make update` and push directly to the branch. + +Delete the workflow file if you want no scheduled runs at all. + +## Upgrading a project created from this template + +Template consumers fork at creation time and never see later +improvements — except that the tooling is deliberately confined to two +self-contained paths (`scripts/`, plus the workflow shell in +`scripts/ci/`), so re-vendoring is one command. From your project's +root: + +```sh +V=$(curl -fsSL https://raw.githubusercontent.com/williamdemeo/github-project/main/scripts/VERSION) \ + && curl -fsSL https://github.com/williamdemeo/github-project/archive/refs/tags/v$V.tar.gz \ + | tar -xz --strip-components=1 "github-project-$V/scripts" "github-project-$V/Makefile" +``` + +Then run `make test` and review `git diff` before committing. Your +`docs/GITHUB_PROJECT.md`, workflows, and everything else are untouched; +`scripts/VERSION` records which release you now carry. + +## The Nix path + +A machine with [Nix](https://nixos.org/) (flakes enabled) needs nothing +else installed — not even Python: + +```sh +nix develop # provides python3, gh, gnumake +make lint +``` + +The flake is a convenience, never a requirement: this repository's own +CI uses plain `setup-python`, and everything documented above works +without Nix. + +## Repository layout + +``` +docs/GITHUB_PROJECT.md the roadmap (a worked example until you replace it) +scripts/ self-contained tooling (re-vendorable as a unit) + gh_project_populate.py file → GitHub + gh_project_update.py GitHub → file + gh_project_lint.py structural validation, no network + _gh_project_lib.py shared parsing, planning, gh client + _utils/ vendored functional-Python core (Result, file_ops, ...) + tests/ offline test suite (recorded fake `gh`) + VERSION which release of the tooling this tree carries +.claude/skills/ committed Claude Code skills for the two workflows +.github/workflows/ CI + the opt-in freshness workflow +``` + +## Design notes + +- **Idempotent by snapshot**: populate fetches live state once per run + and plans against it — rerunning after any interruption is always + safe, and `--dry-run` prints exactly the plan a real run executes. +- **Crash-safe write-back**: issue numbers land in the file after each + creation, not at the end. +- **Label reconciliation, not imposition**: existing labels are never + overwritten, and near-collisions (`era: conway` vs `era:conway`) are + reported instead of silently creating a parallel scheme. +- **Staged and rate-limit-aware**: `--labels-only`, `--milestones-only`, + `--issues-only`, `--start-from M2-3`, `--delay`. +- Provenance: this template distills the gh-project tooling that grew up + in [agda-algebras](https://github.com/ualib/agda-algebras), + [williamdemeo.github.io](https://github.com/williamdemeo/williamdemeo.github.io), + and agda-native-air, and is now its upstream — fixes land here and + downstream projects re-vendor. + +## License + +[MIT](LICENSE). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..227b212 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1782847189, + "narHash": "sha256-twXPFqFsrrY5r28Zh7Homgcp2gUMBgQ6WDS98Q/3xFI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b6018f87da91d19d0ab4cf979885689b469cdd41", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..09db60b --- /dev/null +++ b/flake.nix @@ -0,0 +1,26 @@ +# File: flake.nix +# +# Strictly optional convenience: `nix develop` drops you into a shell +# with everything the Makefile needs (python3, gh, gnumake), so a +# machine without Python can still run the tooling. Nothing in this +# repository requires Nix — the plain path is documented first in the +# README, and CI uses setup-python, so consumers are never Nix-bound. +{ + description = "Dev shell for the github-project template (python3, gh, make)"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + + outputs = { self, nixpkgs }: + let + systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems + (system: f nixpkgs.legacyPackages.${system}); + in + { + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + packages = [ pkgs.python3 pkgs.gh pkgs.gnumake ]; + }; + }); + }; +} From 64cef3f1ce05b335511f936cf8a54cce324eb90c Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Fri, 14 Aug 2026 02:54:02 -0600 Subject: [PATCH 7/9] Document make update-check's real exit behavior Caught during the live E2E: GNU make maps any failing recipe to its own exit 2, so 'make update-check' cannot relay the update script's diff(1)-style exit 1 for drift. The script's 0/1/2 contract is real (verified live: drift exits 1, make wraps it to 2) and the freshness workflow's shell calls the script directly, so its drift-vs-failure branching is unaffected; only the README and the roadmap-upkeep skill promised codes the make path cannot deliver. Both now state: the make target succeeds/fails with explanatory output; call the script directly when the exit code must distinguish drift from a broken check. AI-assisted: Claude Fable 5 (Anthropic) --- .claude/skills/roadmap-upkeep/SKILL.md | 11 ++++++++--- README.md | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.claude/skills/roadmap-upkeep/SKILL.md b/.claude/skills/roadmap-upkeep/SKILL.md index db6c59f..dc9484a 100644 --- a/.claude/skills/roadmap-upkeep/SKILL.md +++ b/.claude/skills/roadmap-upkeep/SKILL.md @@ -16,9 +16,14 @@ surrounding prose. Two commands, two directions: ## Routine refresh -- `make update-check` — is the file current? Exit 0 = current, 1 = - drifted, 2 = the check itself failed (auth, network — NOT drift; fix - the cause before trusting any answer). +- `make update-check` — is the file current? Succeeds when current, + fails otherwise; the output says whether the file drifted or the + check itself failed (auth, network — NOT drift; fix the cause before + trusting any answer). When an exit code must distinguish drift from + failure, call the script directly — + `python3 scripts/gh_project_update.py docs/GITHUB_PROJECT.md --check` + follows diff(1) (0 current / 1 drifted / 2 failed); `make` collapses + any recipe failure to exit 2. - `make update` — rewrite the generated regions from live GitHub state. Hand-written prose outside the markers is preserved byte-for-byte. Commit the result. diff --git a/README.md b/README.md index 87d64d6..438cd59 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,14 @@ that.) whenever you want the file to reflect reality (state flips like `(#7)` → `(#7, closed)`, assignees, edited bodies). Commit the - result. `make update-check` tells you whether that is needed - (exit 0 current / 1 drifted / 2 the check itself failed). + result. `make update-check` tells you whether that is needed: it + succeeds when the file is current and fails otherwise, with output + saying whether the file drifted or the check itself could not run. + (When an exit code must distinguish those two — as the shipped + freshness workflow does — call the script directly: + `python3 scripts/gh_project_update.py docs/GITHUB_PROJECT.md --check` + follows diff(1): 0 current, 1 drifted, 2 failed. `make` collapses + any recipe failure to its own exit 2.) `make help` lists every target. From a5c7c9964abd9b0dce3391a1a349798d763c359d Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Fri, 14 Aug 2026 03:13:23 -0600 Subject: [PATCH 8/9] Address all seven Copilot review findings on PR #1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every finding was real; all seven are fixed, with tests: 1. The recorded-(#N) fallback in plan_issues was unreachable: the snapshot parser dropped any live issue whose [MN-k] prefix had been stripped, so the planner never saw its number and would re-create it. _parse_issues_json now keeps such issues with id=""; populate matches them by number, update filters them out of rendering. The original unit test had simulated the wrong scenario (a changed ID, which still parses) — replaced with the real one, plus an integration test: strip a live title's prefix, re-run populate, zero duplicates. 2. A failed issue-number write-back now aborts the run (counted as a failure) instead of continuing to mutate GitHub while numbers stop landing on disk — that continuation violated the stated persist-before-next-mutation contract. Integration test: chmod-444 plan file aborts after the first creation; rerun completes to four issues with no duplicates. 3. Duplicate milestone numbers escalate from lint warning to error: ms_title_map keys on the number, so a duplicate makes issue->milestone attachment ambiguous. 4. Lint now rejects any '### Issue ...' heading that does not match 'Issue MN-k: Title' — a malformed heading (e.g. M1-X) was invisible to populate and would be silently erased when update rebuilt its region. 5. The ## Labels well-formedness check now validates every bullet in the section, not just bullets that already open a backtick — a backtick-less entry was invisible to the old count. 6. list_issues and list_labels move from fixed --limit ceilings (1000 / 200) to REST endpoints with --paginate; the issues parser filters pull requests (the REST endpoint includes them). Verified against live GitHub, including PR filtering. 7. An issue whose plan-declared milestone is unavailable is skipped and counted as a failure instead of being created milestone-less — populate never revisits existing issues, so that incompleteness was permanent and silent. Plans that declare no milestones still populate milestone-free. Integration test: --issues-only on a fresh repo exits 1 and creates nothing. Suite grows 86 -> 95 tests, all green; REST switch live-verified against the E2E throwaway. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 115 +++++++++++++++++------- scripts/gh_project_populate.py | 71 ++++++++++----- scripts/gh_project_update.py | 9 ++ scripts/tests/fake_gh/gh | 4 +- scripts/tests/test_gh_project_lib.py | 50 +++++++++-- scripts/tests/test_gh_project_lint.py | 24 ++++- scripts/tests/test_gh_project_update.py | 11 +++ scripts/tests/test_integration.py | 68 +++++++++++++- 8 files changed, 288 insertions(+), 64 deletions(-) diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 6c0ad7a..64647d5 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -581,9 +581,11 @@ def plan_issues(desired: tuple[Issue, ...], existing: tuple[Issue, ...]) -> Issu The `[MN-k]` title prefix is the primary join key (robust to post-creation edits of the descriptive title). A `(#N)` number recorded in the plan file is the secondary key: it keeps the guard - intact even if someone strips the prefix from the title on GitHub. + intact even if someone strips the prefix from the title on GitHub + (such issues arrive from the snapshot with id="", so only the + number can match them). """ - by_id = {e.id: e for e in existing} + by_id = {e.id: e for e in existing if e.id} by_number = {e.gh_number: e for e in existing if e.gh_number is not None} to_create: list[Issue] = [] @@ -624,7 +626,12 @@ def lint_plan(text: str) -> tuple[Problem, ...]: Errors (populate must not run): - unbalanced / mismatched / nested / stray generated-region markers - duplicate region ids + - a `### Issue ...` heading that does not match + `### Issue MN-k: Title` (such a heading is invisible to populate + and silently erased when update rebuilds its region) - duplicate issue IDs + - duplicate milestone numbers (issue→milestone attachment keys on + the number, so a duplicate makes attachment ambiguous) - duplicate label names, or two plan labels that collide after normalization (case/whitespace) - an issue whose milestone number has no `### Milestone N` entry @@ -632,11 +639,11 @@ def lint_plan(text: str) -> tuple[Problem, ...]: - a `milestone-N` region id with no matching milestone - an issue referencing a label absent from the `## Labels` section (when that section exists — creation on GitHub would fail) - - a malformed entry line inside `## Labels` + - a bullet inside `## Labels` that does not parse as + `` - `name` (RRGGBB) — description `` Warnings: - no `**Repository**:` header (the CLI --repo can still supply it) - - duplicate milestone numbers """ problems: list[Problem] = [] @@ -668,9 +675,26 @@ def lint_plan(text: str) -> tuple[Problem, ...]: )) ms_numbers = [m.number for m in plan.milestones] - for n in {n for n in ms_numbers if ms_numbers.count(n) > 1}: + for n in sorted({n for n in ms_numbers if ms_numbers.count(n) > 1}): problems.append(Problem( - "warning", f"milestone number {n} is defined more than once")) + "error", + f"milestone number {n} is defined more than once — " + f"issue→milestone attachment keys on the number, so populate " + f"cannot tell which milestone issues M{n}-* belong to", + )) + + # A heading that says `### Issue` but does not parse is worse than a + # missing one: populate never pushes it, and the next update erases + # it when the surrounding generated region is rebuilt from GitHub. + for m in re.finditer(r"^### Issue\b[^\n]*$", text, re.MULTILINE): + if not ISSUE_HEADING_RE.match(m.group(0)): + problems.append(Problem( + "error", + f"malformed issue heading {m.group(0)!r} — expected " + f"`### Issue MN-k: Title`; as written it is invisible to " + f"populate and will be erased by the next update", + _line_of(text, m.start()), + )) issue_ids = [i.id for i in plan.issues] for dup in sorted({i for i in issue_ids if issue_ids.count(i) > 1}): @@ -712,17 +736,20 @@ def _lint_labels(text: str, plan: ProjectPlan) -> list[Problem]: start = section.end() nxt = re.search(r"^## (?!#)", text[start:], re.MULTILINE) end = start + nxt.start() if nxt else len(text) - entry_starts = re.finditer(r"^\s*[-*]\s+`", text[start:end], re.MULTILINE) - parsed_count = len(explicit) - listed_count = sum(1 for _ in entry_starts) - if listed_count > parsed_count: - problems.append(Problem( - "error", - f"## Labels section: {listed_count - parsed_count} " - f"entr{'y' if listed_count - parsed_count == 1 else 'ies'} " - f"did not parse (expected `- `name` (RRGGBB) — description`)", - _line_of(text, section.start()), - )) + # Every bullet in the section must parse — including ones missing + # the backticks entirely (`- documentation (0e8a16) — ...`), which + # would otherwise silently vanish from the label set. + entry_re = re.compile( + r"^\s*[-*]\s+`([^`]+)`\s*\(([0-9a-fA-F]{6})\)\s*(?:—|–|-|:)\s*\S", + ) + for m in re.finditer(r"^\s*[-*]\s+\S[^\n]*$", text[start:end], re.MULTILINE): + if not entry_re.match(m.group(0)): + problems.append(Problem( + "error", + f"## Labels entry {m.group(0).strip()!r} did not parse " + f"(expected `- `name` (RRGGBB) — description`)", + _line_of(text, start + m.start()), + )) names = [l.name for l in explicit] for dup in sorted({n for n in names if names.count(n) > 1}): @@ -802,10 +829,17 @@ def snapshot(self) -> Result[RepoSnapshot, PipelineError]: ) def list_labels(self) -> Result[list[Label], PipelineError]: - """Pull every label currently defined on the repository.""" + """Pull every label currently defined on the repository. + + Uses the REST endpoint with --paginate: `gh label list --limit N` + imposes a fixed ceiling, and a silently truncated snapshot would + defeat the collision detection it feeds. + """ cmd = self._run( - "label", "list", "--repo", self.repo, "--limit", "200", - "--json", "name,color,description", + "api", f"repos/{self.repo}/labels", + "--paginate", + "-X", "GET", + "-F", "per_page=100", ) return cmd.and_then(_parse_labels_json) @@ -829,17 +863,28 @@ def list_milestones(self) -> Result[list[Milestone], PipelineError]: def list_issues(self) -> Result[list[Issue], PipelineError]: """Pull every issue (open and closed) from the repository. + Uses the REST endpoint with --paginate: `gh issue list --limit N` + imposes a fixed ceiling, and a truncated snapshot would make + populate re-create issues it failed to see and update omit them + from generated regions. Pull requests (which the REST issues + endpoint includes) are filtered out by the parser. + + Issues whose title carries no parseable `[MN-k]` prefix are + returned with `id=""`: populate needs them (their number is the + fallback join key when a title was edited on GitHub), while + update filters them out of rendering. + Returned `Issue.milestone_idx` is inferred from the issue's `milestone-N-*` label rather than from GitHub's milestone assignment; the label is the source of truth in this convention because it survives milestone renames. """ cmd = self._run( - "issue", "list", - "--repo", self.repo, - "--state", "all", - "--limit", "1000", - "--json", "number,title,body,labels,milestone,state,assignees", + "api", f"repos/{self.repo}/issues", + "--paginate", + "-X", "GET", + "-F", "state=all", + "-F", "per_page=100", ) return cmd.and_then(_parse_issues_json) @@ -930,13 +975,21 @@ def _parse_issues_json(stdout: str) -> Result[list[Issue], PipelineError]: def to_issues(data: list[dict]) -> list[Issue]: out: list[Issue] = [] for item in data: + if "pull_request" in item: + # The REST issues endpoint returns pull requests too; + # they are never planning issues. + continue title = item.get("title", "") parsed = parse_issue_id(title) - if parsed is None: - # Not a planning issue (e.g. an ad-hoc bug report); skip - # silently — update does not list these. - continue - issue_id, ms_idx, _ord, _suffix, _rest = parsed + # An unparseable title (e.g. an ad-hoc bug report, or a + # planning issue whose prefix someone stripped on GitHub) is + # kept with id="" rather than dropped: populate matches such + # issues by their recorded (#N) number — dropping them here + # would make that fallback unreachable and re-create the + # issue. Update filters id="" records out of rendering. + issue_id, ms_idx = ("", None) + if parsed is not None: + issue_id, ms_idx, _ord, _suffix, _rest = parsed label_names = tuple(lbl["name"] for lbl in item.get("labels", []) or []) # Prefer the milestone-N-* label; fall back to the leading # integer of GitHub's milestone title. The label is canonical @@ -946,7 +999,7 @@ def to_issues(data: list[dict]) -> list[Issue]: gh_ms = item.get("milestone") or {} gh_title = gh_ms.get("title", "") tm = re.match(r"^(\d+)\.", gh_title) - inferred = int(tm.group(1)) if tm else ms_idx + inferred = int(tm.group(1)) if tm else (ms_idx if ms_idx is not None else 0) out.append(Issue( id=issue_id, title=title, diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index 0afa5e3..3b7d728 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -171,6 +171,7 @@ def execute_issues( client: GitHubClient, plan: IssuePlan, ms_title_map: dict[int, str], + declared_milestones: set[int], available_labels: set[str], md_path: Path, text: str, @@ -179,12 +180,22 @@ def execute_issues( """Create the planned issues with immediate number write-back. Returns (current file text, failures). The plan file is persisted - after EACH creation, before the next remote mutation: a later - failure must not lose an already-assigned number, or a rerun would - file a duplicate. + after EACH creation, before the next remote mutation — that is the + crash-safety contract, so a write-back that cannot be persisted + ABORTS the run (counted as a failure) rather than continuing to + mutate GitHub while numbers silently stop landing on disk. Nothing + is lost by aborting: created issues are matched by the snapshot on + the next run. + + An issue whose plan-declared milestone is unavailable (creation + failed, or --issues-only before milestones exist) is skipped and + counted as a failure: creating it milestone-less would leave GitHub + state permanently incomplete, since populate never revisits existing + issues. Issues referencing a milestone the plan never declared are + created without one (a plan with no Milestones section is legal). """ failed = 0 - for issue in plan.to_create: + for position, issue in enumerate(plan.to_create): missing = [ lbl for lbl in issue.labels if lbl.casefold() not in available_labels @@ -199,11 +210,22 @@ def execute_issues( continue ms_title = ms_title_map.get(issue.milestone_idx) - if ms_title is None and issue.milestone_idx != 0: - print( - f" ~ issue {issue.id}: milestone {issue.milestone_idx} " - f"unavailable; creating without a milestone" - ) + if ms_title is None: + if issue.milestone_idx in declared_milestones: + failed += 1 + print( + f" ! issue {issue.id}: skipped — milestone " + f"{issue.milestone_idx} is declared in the plan but " + f"not available on GitHub (create milestones first, " + f"then re-run)" + ) + continue + if issue.milestone_idx != 0: + print( + f" ~ issue {issue.id}: milestone {issue.milestone_idx} " + f"is not declared in the plan; creating without a " + f"milestone" + ) result = client.create_issue(issue, milestone_title=ms_title) if result.is_err: @@ -215,19 +237,25 @@ def execute_issues( gh_number = result.unwrap() print(f" created issue #{gh_number}: {issue.title}") - text, found = record_issue_number(text, issue.id, gh_number) - if not found: + new_text, found = record_issue_number(text, issue.id, gh_number) + write_result = write_text(md_path, new_text) if found else None + if not found or write_result.is_err: + reason = ( + f"heading for {issue.id} not found" + if not found else str(write_result.unwrap_err()) + ) + remaining = len(plan.to_create) - position - 1 + failed += 1 print( - f" ~ could not write #{gh_number} back for {issue.id} " - f"(heading not found)", file=sys.stderr, + f" ! could not persist #{gh_number} for {issue.id} " + f"({reason}); aborting before further mutations " + f"({remaining} issue(s) not attempted). Issue " + f"#{gh_number} IS live on GitHub; fix the problem and " + f"re-run — the snapshot match makes the rerun safe.", + file=sys.stderr, ) - else: - write_result = write_text(md_path, text) - if write_result.is_err: - print( - f" ~ write-back failed: {write_result.unwrap_err()}", - file=sys.stderr, - ) + return text, failed + text = new_text time.sleep(delay) return text, failed @@ -399,7 +427,8 @@ def main() -> int: | {lbl.name.casefold() for lbl in label_plan.to_create} ) _text, issue_failed = execute_issues( - client, issue_plan, ms_title_map, available, + client, issue_plan, ms_title_map, + {m.number for m in plan.milestones}, available, args.markdown, text, args.delay, ) failures += issue_failed diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py index 4217081..4940afb 100755 --- a/scripts/gh_project_update.py +++ b/scripts/gh_project_update.py @@ -77,8 +77,17 @@ # ── Issue grouping and sorting ─────────────────────────────────────────────── def group_by_milestone(issues: list[Issue]) -> dict[int, list[Issue]]: + """Group planning issues by milestone for region rendering. + + Issues with id="" (no parseable `[MN-k]` prefix — ad-hoc bug + reports, or planning issues whose prefix was stripped on GitHub) are + excluded: the snapshot carries them for populate's number-based + matching, but they cannot be rendered under a stable identifier. + """ out: dict[int, list[Issue]] = {} for issue in issues: + if not issue.id: + continue out.setdefault(issue.milestone_idx, []).append(issue) return out diff --git a/scripts/tests/fake_gh/gh b/scripts/tests/fake_gh/gh index ed3b109..b7a5ce5 100755 --- a/scripts/tests/fake_gh/gh +++ b/scripts/tests/fake_gh/gh @@ -68,7 +68,7 @@ def main() -> int: with (STATE / "calls.log").open("a") as log: log.write(" ".join(args) + "\n") - if args[:2] == ["label", "list"]: + if args[0] == "api" and args[1].endswith("/labels"): print(json.dumps(load("labels.json", []))) return 0 @@ -99,7 +99,7 @@ def main() -> int: print(json.dumps(load("milestones.json", []))) return 0 - if args[:2] == ["issue", "list"]: + if args[0] == "api" and args[1].endswith("/issues"): print(json.dumps(load("issues.json", []))) return 0 diff --git a/scripts/tests/test_gh_project_lib.py b/scripts/tests/test_gh_project_lib.py index cde6670..4f1b550 100644 --- a/scripts/tests/test_gh_project_lib.py +++ b/scripts/tests/test_gh_project_lib.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import json import sys import unittest from pathlib import Path @@ -358,16 +359,27 @@ def test_match_by_id_prefix(self) -> None: self.assertEqual(plan.existing[0][1].gh_number, 3) def test_match_by_recorded_number(self) -> None: - # Someone stripped the [M1-1] prefix on GitHub; the recorded (#3) - # in the plan file still prevents a duplicate... + # Someone stripped the [M1-1] prefix on GitHub. The snapshot + # carries such issues with id="" (the parser must not drop them, + # or this fallback would be unreachable); the recorded (#3) in + # the plan file prevents a duplicate. desired = (Issue("M1-1", "[M1-1] A", "b", gh_number=3),) - # ...but note the live issue only appears in the snapshot if its - # title still parses; simulate a live issue whose ID changed. - existing = (Issue("M9-9", "[M9-9] Retitled", "b", gh_number=3),) + existing = (Issue("", "A, prefix stripped", "b", gh_number=3),) plan = plan_issues(desired, existing) self.assertEqual(plan.to_create, ()) self.assertEqual(plan.existing[0][1].gh_number, 3) + def test_blank_ids_never_match_each_other(self) -> None: + # Two different prefix-less live issues must not collapse into + # one by their empty id. + desired = (Issue("M1-1", "[M1-1] A", "b"),) + existing = ( + Issue("", "Ad-hoc bug", "b", gh_number=8), + Issue("", "Another ad-hoc", "b", gh_number=9), + ) + plan = plan_issues(desired, existing) + self.assertEqual([i.id for i in plan.to_create], ["M1-1"]) + def test_new_issue(self) -> None: desired = (Issue("M1-2", "[M1-2] New", "b"),) existing = (Issue("M1-1", "[M1-1] Old", "b", gh_number=3),) @@ -375,5 +387,33 @@ def test_new_issue(self) -> None: self.assertEqual([i.id for i in plan.to_create], ["M1-2"]) +class ParseIssuesJson(unittest.TestCase): + """The snapshot parser must keep prefix-less issues and drop PRs.""" + + def test_prefixless_issue_kept_with_blank_id(self) -> None: + payload = json.dumps([ + {"number": 7, "title": "No prefix here", "body": "b", + "labels": [{"name": "milestone-1-core"}], "milestone": None, + "state": "open", "assignees": []}, + ]) + issues = lib._parse_issues_json(payload).unwrap() + self.assertEqual(len(issues), 1) + self.assertEqual(issues[0].id, "") + self.assertEqual(issues[0].gh_number, 7) + self.assertEqual(issues[0].milestone_idx, 1) # from the label + + def test_pull_requests_are_filtered(self) -> None: + payload = json.dumps([ + {"number": 5, "title": "[M1-1] Real issue", "body": "", + "labels": [], "milestone": None, "state": "open", + "assignees": []}, + {"number": 6, "title": "[M1-2] Actually a PR", "body": "", + "labels": [], "milestone": None, "state": "open", + "assignees": [], "pull_request": {"url": "..."}}, + ]) + issues = lib._parse_issues_json(payload).unwrap() + self.assertEqual([i.gh_number for i in issues], [5]) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_gh_project_lint.py b/scripts/tests/test_gh_project_lint.py index 9f66eac..77b6645 100644 --- a/scripts/tests/test_gh_project_lint.py +++ b/scripts/tests/test_gh_project_lint.py @@ -120,9 +120,29 @@ def test_missing_repository_is_warning(self) -> None: self.assertEqual(errors(broken), []) self.assertTrue(any("no `**Repository**" in w for w in warnings(broken))) - def test_duplicate_milestone_number_is_warning(self) -> None: + +class StructuralAmbiguities(unittest.TestCase): + def test_duplicate_milestone_number_is_error(self) -> None: + # ms_title_map keys on the number, so a duplicate makes + # issue→milestone attachment ambiguous — populate must refuse. broken = PLAN.replace("### Milestone 2 — Polish", "### Milestone 1 — Polish") - self.assertTrue(any("more than once" in w for w in warnings(broken))) + self.assertTrue(any("more than once" in e for e in errors(broken))) + + def test_malformed_issue_heading_is_error(self) -> None: + # `M1-X` never parses, so the heading is invisible to populate + # and would be erased when update rebuilds the region. + broken = PLAN.replace( + "### Issue M1-1: Set up the build", + "### Issue M1-X: Set up the build", + ) + self.assertTrue(any("malformed issue heading" in e for e in errors(broken))) + + def test_backtickless_label_bullet_is_error(self) -> None: + broken = PLAN.replace( + "- `documentation` (0e8a16) — Documentation changes.", + "- documentation (0e8a16) — Documentation changes.", + ) + self.assertTrue(any("did not parse" in e for e in errors(broken))) if __name__ == "__main__": diff --git a/scripts/tests/test_gh_project_update.py b/scripts/tests/test_gh_project_update.py index cc25bf7..1fa66de 100644 --- a/scripts/tests/test_gh_project_update.py +++ b/scripts/tests/test_gh_project_update.py @@ -94,6 +94,17 @@ def test_manuals_preserved_and_regions_rebuilt(self) -> None: self.assertEqual(reparsed.manuals, parsed.manuals) +class GroupByMilestone(unittest.TestCase): + def test_non_planning_issues_are_excluded(self) -> None: + # The snapshot carries id="" records (prefix-less titles) for + # populate's number matching; rendering must not list them. + groups = upd.group_by_milestone([ + issue("M1-1", "[M1-1] Real", gh_number=1, milestone_idx=1), + issue("", "Ad-hoc bug report", gh_number=9, milestone_idx=1), + ]) + self.assertEqual([i.id for i in groups[1]], ["M1-1"]) + + class StripIdPrefix(unittest.TestCase): def test_strip(self) -> None: self.assertEqual(upd.strip_id_prefix("[M1-2] Title"), "Title") diff --git a/scripts/tests/test_integration.py b/scripts/tests/test_integration.py index 4e263b0..a5aff72 100644 --- a/scripts/tests/test_integration.py +++ b/scripts/tests/test_integration.py @@ -88,8 +88,8 @@ def test_creates_everything_with_one_snapshot(self) -> None: # One snapshot per run — not one list call per created entity # (#293 item 6: this used to be 1 ListIssues per issue create). - self.assertEqual(self.calls("label list"), 1) - self.assertEqual(self.calls("issue list"), 1) + self.assertEqual(self.calls("octocat/demo/labels"), 1) + self.assertEqual(self.calls("octocat/demo/issues"), 1) self.assertEqual( sum("milestones" in line and "POST" not in line for line in (self.state / "calls.log").read_text().splitlines()), @@ -166,6 +166,68 @@ def test_interrupted_run_reruns_without_duplicates(self) -> None: self.assertEqual(len(set(titles)), 4, f"duplicates: {titles}") +class TitleEditResilience(FakeGhHarness): + def test_stripped_prefix_does_not_duplicate(self) -> None: + # After a successful populate, someone edits an issue title on + # GitHub and removes its [MN-k] prefix. The recorded (#N) in + # the plan file is the fallback join key: a re-run must match + # the renamed issue by number, not re-create it. + self.assertEqual(self.populate().returncode, 0) + issues = self.issues_on_fake_github() + victim = next(i for i in issues if i["title"].startswith("[M1-1]")) + victim["title"] = "Storage layer, renamed with no prefix" + (self.state / "issues.json").write_text(json.dumps(issues)) + + rerun = self.populate() + self.assertEqual(rerun.returncode, 0, rerun.stderr + rerun.stdout) + self.assertEqual(len(self.issues_on_fake_github()), 4) + self.assertIn(f"- exists: issue #{victim['number']}", rerun.stdout) + + # Update still works: the prefix-less issue simply leaves the + # region (it has no stable identifier to render under). + upd = self.update() + self.assertEqual(upd.returncode, 0, upd.stderr) + text = self.plan_path.read_text(encoding="utf-8") + self.assertNotIn("renamed with no prefix", text) + + +class MilestoneAvailability(FakeGhHarness): + def test_issues_only_without_milestones_fails_and_creates_nothing(self) -> None: + # --issues-only documents that milestones must already exist. + # On a repo with none, every issue whose plan-declared milestone + # is missing is skipped and counted as a failure — creating it + # milestone-less would leave GitHub state permanently incomplete + # (populate never revisits existing issues). + proc = self.populate("--issues-only") + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertEqual(self.issues_on_fake_github(), []) + self.assertIn("not available on GitHub", proc.stdout) + + +class WriteBackAbort(FakeGhHarness): + @unittest.skipIf(os.geteuid() == 0, "root ignores file permissions") + def test_unwritable_plan_aborts_after_first_creation(self) -> None: + # The crash-safety contract is persist-before-next-mutation, so + # when persisting fails the run must stop mutating GitHub, not + # continue with numbers silently lost. + self.plan_path.chmod(0o444) + try: + proc = self.populate() + finally: + self.plan_path.chmod(0o644) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertEqual(len(self.issues_on_fake_github()), 1) + self.assertIn("aborting before further mutations", proc.stderr) + + # The rerun is safe: the snapshot matches the orphaned issue by + # its [MN-k] prefix, and only the remaining three are created. + rerun = self.populate() + self.assertEqual(rerun.returncode, 0, rerun.stderr) + titles = [i["title"] for i in self.issues_on_fake_github()] + self.assertEqual(len(titles), 4) + self.assertEqual(len(set(titles)), 4, f"duplicates: {titles}") + + class LabelReconciliation(FakeGhHarness): def test_near_collision_is_reported_not_created(self) -> None: (self.state / "labels.json").write_text(json.dumps([ @@ -250,7 +312,7 @@ def test_lint_gate(self) -> None: proc = self.populate("--dry-run") self.assertEqual(proc.returncode, 2, proc.stdout) self.assertIn("duplicate issue ID", proc.stderr) - self.assertEqual(self.calls("issue list"), 0) # refused before fetching + self.assertEqual(self.calls("octocat/demo/issues"), 0) # refused before fetching if __name__ == "__main__": From 634ca0e2a047dae0186e00a143a20862471c1f37 Mon Sep 17 00:00:00 2001 From: William DeMeo Date: Fri, 14 Aug 2026 23:36:01 -0600 Subject: [PATCH 9/9] Address Copilot round 2: the pagination bug and all ten suppressed findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explicit comment was a real bug introduced by the REST switch: gh api --paginate concatenates pages as separate JSON documents, so a single json.loads fails with 'Extra data' past 100 items. _parse_json now decodes document-by-document with raw_decode and concatenates — chosen over gh's --slurp, which requires gh >= 2.47 and would raise the documented 'any gh' floor. (The ancestors' paginated milestones call carried the same latent bug.) All ten suppressed comments were also legitimate; each is fixed: - Render/parse asymmetry: update now emits canonical **Labels:** and **Assignees:** (the spelling the plan parser recognizes), populate strips Assignees metadata from bodies, and a round-trip test pins that a rendered block reparses with labels intact. Consequence handled: the label-reference lint no longer applies to issues with a recorded (#N) — their labels line is rendered state and may carry GitHub-added labels. - BODY_END_RE had regressed the exemplar format: real plans put '## Tasks' / '## Acceptance criteria' sections INSIDE issue bodies. Bodies are now bounded by region markers and the two structural headers the ancestors special-cased ('## Milestone N', '## Summary'), not by generic '## ' headings; the fixture now includes a ##-section body. - Dry run now predicts execution exactly: the shared issue_blocker computes label/milestone availability from the mode-appropriate prospective sets, prints '! blocked' lines, and drives the exit code. - Milestone-consistency lint guards on the section's PRESENCE, and numbered-but-malformed '### Milestone 1 - Core' headings (hyphen for em-dash) are their own error — an all-malformed section can no longer bypass validation. Prose headings like '### Milestone dependencies' (used by the exemplars) stay legal. - Live bodies containing region-marker text are defanged on render ('END GENERATED (escaped):'), so a hostile or accidental marker in a GitHub issue body cannot corrupt the next parse. - parse_file catches a stray END even when a valid region follows it. - The three --*-only flags are mutually exclusive (any pair silently deselected every stage). - execute_issues returns explicit created/failed counts; the DONE summary no longer claims 'planned - failed' were created after an abort, and reports the not-attempted remainder. - Vendored pipeline_types.is_agda_file matches the filename, not Path.suffix, so 'Module.lagda.md' is recognized (this repo is the utils upstream; downstreams inherit the fix on re-vendor). Suite grows 95 -> 111, all green; canonical rendering and pagination verified live against the E2E throwaway. AI-assisted: Claude Fable 5 (Anthropic) --- scripts/_gh_project_lib.py | 122 ++++++++++++------ scripts/_utils/pipeline_types.py | 8 +- scripts/gh_project_populate.py | 165 ++++++++++++++++-------- scripts/gh_project_update.py | 22 +++- scripts/tests/test_gh_project_lib.py | 50 ++++++- scripts/tests/test_gh_project_lint.py | 30 +++++ scripts/tests/test_gh_project_update.py | 34 ++++- scripts/tests/test_integration.py | 30 +++++ scripts/tests/test_pipeline_types.py | 26 ++++ 9 files changed, 390 insertions(+), 97 deletions(-) diff --git a/scripts/_gh_project_lib.py b/scripts/_gh_project_lib.py index 64647d5..ab3b740 100644 --- a/scripts/_gh_project_lib.py +++ b/scripts/_gh_project_lib.py @@ -170,12 +170,18 @@ class RepoSnapshot: # back by populate, or `(#123, closed)` as rendered by update. HEADING_NUMBER_SUFFIX_RE = re.compile(r"\s*\(#(\d+)(?:,\s*[a-z]+)?\)$") -# An issue body ends at the next issue heading, the next top-level `## ` -# section, or an END GENERATED marker — whichever comes first. (The -# ancestor scripts stopped only at `## Summary:` and at leaked -# `## Milestone N` headers, which swept trailing prose into the final -# issue's body.) -BODY_END_RE = re.compile(r"^(?:## (?!#)| " + f"has no matching BEGIN" + ), + context={"offset": stray.start()}, + )) if begin is None: - # A stray END with no matching BEGIN would otherwise vanish - # silently into a manual segment. - stray = END_RE.search(content, pos) - if stray is not None: - return Result.err(PipelineError( - error_type=ErrorType.PARSING_ERROR, - message=( - f" " - f"has no matching BEGIN" - ), - context={"offset": stray.start()}, - )) manuals.append(content[pos:]) return Result.ok(ParsedFile(tuple(manuals), tuple(ids))) marker_id = begin.group(1) @@ -696,13 +705,31 @@ def lint_plan(text: str) -> tuple[Problem, ...]: _line_of(text, m.start()), )) + # Same hazard one level up: `### Milestone 1 - Core` (hyphen, not + # em-dash) silently parses to nothing, which would otherwise also + # disable every milestone-consistency check below. Only NUMBERED + # headings are scanned — prose headings like `### Milestone + # dependencies` are legitimate (the exemplar plans use them). + milestone_heading_re = re.compile(r"^### Milestone (\d+) — (.+)$") + for m in re.finditer(r"^### Milestone\s+\d+\b[^\n]*$", text, re.MULTILINE): + if not milestone_heading_re.match(m.group(0)): + problems.append(Problem( + "error", + f"malformed milestone heading {m.group(0)!r} — expected " + f"`### Milestone N — Title` (em-dash)", + _line_of(text, m.start()), + )) + issue_ids = [i.id for i in plan.issues] for dup in sorted({i for i in issue_ids if issue_ids.count(i) > 1}): problems.append(Problem( "error", f"duplicate issue ID {dup} — populate would create " f"one issue and orphan the other heading")) - if plan.milestones: + # Guard on the SECTION's presence, not on whether anything parsed + # from it: a Milestones section whose every heading is malformed + # must not silently disable these consistency checks. + if re.search(r"^## Milestones\s*$", text, re.MULTILINE): known = set(ms_numbers) for issue in plan.issues: if issue.milestone_idx not in known: @@ -768,6 +795,11 @@ def _lint_labels(text: str, plan: ProjectPlan) -> list[Problem]: if explicit: known = {normalize_label_name(n) for n in names} for issue in plan.issues: + if issue.gh_number is not None: + # Already created: its labels line is state rendered + # back by update (and may legitimately carry labels + # added on GitHub), not creation input. + continue for lbl in issue.labels: if normalize_label_name(lbl) not in known: problems.append(Problem( @@ -1017,22 +1049,40 @@ def to_issues(data: list[dict]) -> list[Issue]: def _parse_json(stdout: str, kind: str) -> Result[list[dict], PipelineError]: - """Decode the JSON body of a `gh` invocation, with structured errors.""" - try: - data = json.loads(stdout) - except json.JSONDecodeError as e: - return Result.err(PipelineError( - error_type=ErrorType.PARSING_ERROR, - message=f"failed to decode {kind} JSON from `gh`", - cause=e, - context={"stdout_preview": stdout[:500]}, - )) - if not isinstance(data, list): - return Result.err(PipelineError( - error_type=ErrorType.PARSING_ERROR, - message=f"expected a JSON list of {kind}, got {type(data).__name__}", - )) - return Result.ok(data) + """Decode the JSON body of a `gh` invocation, with structured errors. + + `gh api --paginate` concatenates each page as a separate JSON + document (`[...][...]`), so this decodes document-by-document with + raw_decode and concatenates the lists — a single json.loads would + fail with "Extra data" on the 101st item. (gh's own --slurp flag + would do this server-side, but it requires gh >= 2.47; decoding here + keeps the documented floor at "any gh".) + """ + decoder = json.JSONDecoder() + items: list[dict] = [] + pos = 0 + end = len(stdout) + while True: + while pos < end and stdout[pos].isspace(): + pos += 1 + if pos >= end: + break + try: + data, pos = decoder.raw_decode(stdout, pos) + except json.JSONDecodeError as e: + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"failed to decode {kind} JSON from `gh`", + cause=e, + context={"stdout_preview": stdout[:500]}, + )) + if not isinstance(data, list): + return Result.err(PipelineError( + error_type=ErrorType.PARSING_ERROR, + message=f"expected a JSON list of {kind}, got {type(data).__name__}", + )) + items.extend(data) + return Result.ok(items) def _parse_milestone_create_response(stdout: str) -> Result[int, PipelineError]: diff --git a/scripts/_utils/pipeline_types.py b/scripts/_utils/pipeline_types.py index 14a27b1..52b57af 100644 --- a/scripts/_utils/pipeline_types.py +++ b/scripts/_utils/pipeline_types.py @@ -258,8 +258,12 @@ def at_stage(self, stage: ProcessingStage, new_path: Path) -> ProcessedFile: @property def is_agda_file(self) -> bool: - """True if this is an Agda source file.""" - return self.source_path.suffix in {'.agda', '.lagda', '.lagda.md'} + """True if this is an Agda source file. + + Matched against the filename, not Path.suffix: for the literate + form `Module.lagda.md`, suffix is only `.md`. + """ + return self.source_path.name.endswith(('.agda', '.lagda', '.lagda.md')) @property def module_name(self) -> str: diff --git a/scripts/gh_project_populate.py b/scripts/gh_project_populate.py index 3b7d728..0598507 100755 --- a/scripts/gh_project_populate.py +++ b/scripts/gh_project_populate.py @@ -117,11 +117,55 @@ def print_milestone_plan(plan: MilestonePlan) -> None: print(f" - exists: milestone #{ms.gh_number} {ms.title}") -def print_issue_plan(plan: IssuePlan) -> None: +def issue_blocker( + issue, + available_milestones: set[int], + declared_milestones: set[int], + available_labels: set[str], +) -> str | None: + """Why this issue cannot be created, or None if it can. + + Shared by the dry-run report and the live execution loop, so the + dry run prints — and exits with — exactly what a real run would do. + """ + missing = [ + lbl for lbl in issue.labels if lbl.casefold() not in available_labels + ] + if missing: + return ( + f"references unavailable label(s) " + f"{', '.join(f'`{m}`' for m in missing)} " + f"(not on GitHub and not created this run)" + ) + if (issue.milestone_idx not in available_milestones + and issue.milestone_idx in declared_milestones): + return ( + f"milestone {issue.milestone_idx} is declared in the plan but " + f"not available on GitHub (create milestones first, then re-run)" + ) + return None + + +def print_issue_plan( + plan: IssuePlan, + available_milestones: set[int], + declared_milestones: set[int], + available_labels: set[str], +) -> int: + """Print the issue plan; returns the number of blocked issues.""" + blocked = 0 for issue in plan.to_create: - print(f" + create issue: {issue.title}") + reason = issue_blocker( + issue, available_milestones, declared_milestones, available_labels + ) + if reason is None: + print(f" + create issue: {issue.title}") + else: + blocked += 1 + print(f" ! blocked: issue {issue.id} — {reason}") for _, live in plan.existing: print(f" - exists: issue #{live.gh_number} {live.title}") + return blocked def _stage_header(title: str) -> None: @@ -176,16 +220,19 @@ def execute_issues( md_path: Path, text: str, delay: float, -) -> tuple[str, int]: +) -> tuple[str, int, int]: """Create the planned issues with immediate number write-back. - Returns (current file text, failures). The plan file is persisted - after EACH creation, before the next remote mutation — that is the - crash-safety contract, so a write-back that cannot be persisted - ABORTS the run (counted as a failure) rather than continuing to - mutate GitHub while numbers silently stop landing on disk. Nothing - is lost by aborting: created issues are matched by the snapshot on - the next run. + Returns (current file text, created, failures). The counts are + explicit so the caller's summary stays honest when the loop aborts + early — `planned - failed` is NOT the number created in that case. + + The plan file is persisted after EACH creation, before the next + remote mutation — that is the crash-safety contract, so a write-back + that cannot be persisted ABORTS the run (counted as a failure) + rather than continuing to mutate GitHub while numbers silently stop + landing on disk. Nothing is lost by aborting: created issues are + matched by the snapshot on the next run. An issue whose plan-declared milestone is unavailable (creation failed, or --issues-only before milestones exist) is skipped and @@ -194,38 +241,24 @@ def execute_issues( issues. Issues referencing a milestone the plan never declared are created without one (a plan with no Milestones section is legal). """ + created = 0 failed = 0 - for position, issue in enumerate(plan.to_create): - missing = [ - lbl for lbl in issue.labels - if lbl.casefold() not in available_labels - ] - if missing: + for issue in plan.to_create: + reason = issue_blocker( + issue, set(ms_title_map), declared_milestones, available_labels + ) + if reason is not None: failed += 1 - print( - f" ! issue {issue.id}: skipped — references unavailable " - f"label(s) {', '.join(f'`{m}`' for m in missing)} " - f"(not on GitHub and not created this run)" - ) + print(f" ! issue {issue.id}: skipped — {reason}") continue ms_title = ms_title_map.get(issue.milestone_idx) - if ms_title is None: - if issue.milestone_idx in declared_milestones: - failed += 1 - print( - f" ! issue {issue.id}: skipped — milestone " - f"{issue.milestone_idx} is declared in the plan but " - f"not available on GitHub (create milestones first, " - f"then re-run)" - ) - continue - if issue.milestone_idx != 0: - print( - f" ~ issue {issue.id}: milestone {issue.milestone_idx} " - f"is not declared in the plan; creating without a " - f"milestone" - ) + if ms_title is None and issue.milestone_idx != 0: + print( + f" ~ issue {issue.id}: milestone {issue.milestone_idx} " + f"is not declared in the plan; creating without a " + f"milestone" + ) result = client.create_issue(issue, milestone_title=ms_title) if result.is_err: @@ -235,6 +268,7 @@ def execute_issues( continue gh_number = result.unwrap() + created += 1 print(f" created issue #{gh_number}: {issue.title}") new_text, found = record_issue_number(text, issue.id, gh_number) @@ -244,7 +278,7 @@ def execute_issues( f"heading for {issue.id} not found" if not found else str(write_result.unwrap_err()) ) - remaining = len(plan.to_create) - position - 1 + remaining = len(plan.to_create) - created - failed failed += 1 print( f" ! could not persist #{gh_number} for {issue.id} " @@ -254,10 +288,10 @@ def execute_issues( f"re-run — the snapshot match makes the rerun safe.", file=sys.stderr, ) - return text, failed + return text, created, failed text = new_text time.sleep(delay) - return text, failed + return text, created, failed # ── Main ───────────────────────────────────────────────────────────────────── @@ -282,9 +316,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("markdown", type=Path, help="Path to the project plan markdown file") parser.add_argument("--repo", help="GitHub repo (owner/name); overrides the file header") parser.add_argument("--dry-run", action="store_true", help="Print the plan without executing") - parser.add_argument("--milestones-only", action="store_true") - parser.add_argument("--labels-only", action="store_true") - parser.add_argument("--issues-only", action="store_true") + # Mutually exclusive: any pair of these would deselect every stage + # and silently do nothing. + stage = parser.add_mutually_exclusive_group() + stage.add_argument("--milestones-only", action="store_true") + stage.add_argument("--labels-only", action="store_true") + stage.add_argument("--issues-only", action="store_true") parser.add_argument("--skip-labels", action="store_true") parser.add_argument("--start-from", type=str, default=None, help="Create only issues with ID >= this (e.g. M1-3)") @@ -362,6 +399,21 @@ def main() -> int: milestone_plan = plan_milestones(plan.milestones, snapshot.milestones) issue_plan = plan_issues(desired_issues, snapshot.issues) + # Availability as it WILL stand when the issues stage runs, given + # the selected stages: entities already on GitHub, plus those this + # run will create first. Feeding these to the shared issue_blocker + # makes the printed plan — and the dry-run exit code — match what + # execution would actually do (e.g. `--dry-run --issues-only` on a + # repo with no milestones reports the blocks a real run would hit). + declared_milestones = {m.number for m in plan.milestones} + prospective_milestones = {m.number for m in milestone_plan.existing} + if do_milestones: + prospective_milestones |= {m.number for m in milestone_plan.to_create} + prospective_labels = {lbl.name.casefold() for lbl in snapshot.labels} + if do_labels: + prospective_labels |= {lbl.name.casefold() for lbl in label_plan.to_create} + + blocked = 0 if do_labels: _stage_header("LABELS") print_label_plan(label_plan) @@ -372,14 +424,17 @@ def main() -> int: print() if do_issues: _stage_header("ISSUES") - print_issue_plan(issue_plan) + blocked = print_issue_plan( + issue_plan, prospective_milestones, declared_milestones, + prospective_labels, + ) print() collisions = len(label_plan.collisions) if do_labels else 0 if args.dry_run: print("Dry run — nothing was created.") - return 1 if collisions else 0 + return 1 if collisions or blocked else 0 # ── Confirm ────────────────────────────────────────────────────────── to_do = [] @@ -422,20 +477,24 @@ def main() -> int: if do_issues: _stage_header("CREATING ISSUES") - available = ( - {lbl.name.casefold() for lbl in snapshot.labels} - | {lbl.name.casefold() for lbl in label_plan.to_create} - ) - _text, issue_failed = execute_issues( + # Labels planned for creation are available only if the labels + # stage actually ran this invocation. + available = {lbl.name.casefold() for lbl in snapshot.labels} + if do_labels: + available |= {lbl.name.casefold() for lbl in label_plan.to_create} + _text, issues_created, issue_failed = execute_issues( client, issue_plan, ms_title_map, - {m.number for m in plan.milestones}, available, + declared_milestones, available, args.markdown, text, args.delay, ) failures += issue_failed + not_attempted = len(issue_plan.to_create) - issues_created - issue_failed print() _stage_header( - f"DONE: {len(issue_plan.to_create) - issue_failed} issues created, " - f"{len(issue_plan.existing)} already existed, {issue_failed} failed" + f"DONE: {issues_created} issues created, " + f"{len(issue_plan.existing)} already existed, " + f"{issue_failed} failed or skipped, " + f"{not_attempted} not attempted" ) if collisions: diff --git a/scripts/gh_project_update.py b/scripts/gh_project_update.py index 4940afb..bd0d0f4 100755 --- a/scripts/gh_project_update.py +++ b/scripts/gh_project_update.py @@ -106,6 +106,18 @@ def issue_sort_key(issue: Issue) -> tuple[int, int, str]: _ID_PREFIX_RE = re.compile(r"^\[M\d+-\d+[a-z]?\]\s+(.+)$") +# A live GitHub body may contain anything — including text shaped like +# this file's own region markers. Rendered verbatim, such a line would +# close (or open) a region on the next parse and corrupt the file, so +# the comment-opening form is defanged before insertion. The escape is +# deterministic, keeping `--check` stable across repeated renders. +_MARKER_IN_BODY_RE = re.compile(r"\n" + "\n\n" + ) + self.assertTrue(r.is_err) + self.assertIn("no matching BEGIN", r.unwrap_err().message) + self.assertIn("stray", r.unwrap_err().message) + def test_nested_begin(self) -> None: r = parse_file( "\n" @@ -387,6 +409,32 @@ def test_new_issue(self) -> None: self.assertEqual([i.id for i in plan.to_create], ["M1-2"]) +class ParseJsonPagination(unittest.TestCase): + """`gh api --paginate` concatenates pages as separate JSON documents; + the decoder must consume them all, not fail with 'Extra data'.""" + + def test_multiple_documents_are_concatenated(self) -> None: + r = lib._parse_json('[{"a": 1}, {"a": 2}][{"a": 3}]', "things") + self.assertEqual(r.unwrap(), [{"a": 1}, {"a": 2}, {"a": 3}]) + + def test_newline_separated_documents(self) -> None: + r = lib._parse_json('[{"a": 1}]\n[{"a": 2}]\n', "things") + self.assertEqual(r.unwrap(), [{"a": 1}, {"a": 2}]) + + def test_single_document_still_works(self) -> None: + self.assertEqual(lib._parse_json('[{"a": 1}]', "things").unwrap(), + [{"a": 1}]) + + def test_empty_input_is_empty_list(self) -> None: + self.assertEqual(lib._parse_json("", "things").unwrap(), []) + + def test_non_list_document_is_err(self) -> None: + self.assertTrue(lib._parse_json('{"a": 1}', "things").is_err) + + def test_garbage_is_err(self) -> None: + self.assertTrue(lib._parse_json('[1] not json', "things").is_err) + + class ParseIssuesJson(unittest.TestCase): """The snapshot parser must keep prefix-less issues and drop PRs.""" diff --git a/scripts/tests/test_gh_project_lint.py b/scripts/tests/test_gh_project_lint.py index 77b6645..403eb38 100644 --- a/scripts/tests/test_gh_project_lint.py +++ b/scripts/tests/test_gh_project_lint.py @@ -87,6 +87,18 @@ def test_issue_references_unlisted_label(self) -> None: ) self.assertTrue(any("`nonexistent-label`" in e for e in errors(broken))) + def test_created_issues_may_carry_github_side_labels(self) -> None: + # An issue with a recorded (#N) already exists; its labels line + # is state rendered back by update and may carry labels added on + # GitHub (e.g. `wontfix`) — that must not block populate. + updated = PLAN.replace( + "### Issue M1-2: [M1-2] Write the docs (#42)\n\n" + "**Labels:** `milestone-1-core`, `documentation`", + "### Issue M1-2: [M1-2] Write the docs (#42)\n\n" + "**Labels:** `milestone-1-core`, `documentation`, `wontfix`", + ) + self.assertEqual(errors(updated), []) + class LabelSectionProblems(unittest.TestCase): def test_malformed_entry(self) -> None: @@ -137,6 +149,24 @@ def test_malformed_issue_heading_is_error(self) -> None: ) self.assertTrue(any("malformed issue heading" in e for e in errors(broken))) + def test_malformed_milestone_heading_is_error(self) -> None: + # A hyphen instead of the em-dash parses to nothing; without + # this check the plan would populate with no milestones at all. + broken = PLAN.replace("### Milestone 1 — Core", "### Milestone 1 - Core") + self.assertTrue(any("malformed milestone heading" in e + for e in errors(broken))) + + def test_milestone_checks_fire_even_when_no_heading_parses(self) -> None: + # The consistency checks guard on the SECTION's presence: a + # Milestones section whose every heading is malformed must not + # disable them. + broken = (PLAN + .replace("### Milestone 1 — Core", "### Milestone 1 - Core") + .replace("### Milestone 2 — Polish", "### Milestone 2 - Polish")) + errs = errors(broken) + self.assertTrue(any("refers to milestone" in e for e in errs)) + self.assertTrue(any("region 'milestone-1'" in e for e in errs)) + def test_backtickless_label_bullet_is_error(self) -> None: broken = PLAN.replace( "- `documentation` (0e8a16) — Documentation changes.", diff --git a/scripts/tests/test_gh_project_update.py b/scripts/tests/test_gh_project_update.py index 1fa66de..c47a4e4 100644 --- a/scripts/tests/test_gh_project_update.py +++ b/scripts/tests/test_gh_project_update.py @@ -15,6 +15,7 @@ import gh_project_update as upd # noqa: E402 from _gh_project_lib import Issue, parse_file # noqa: E402 +from _gh_project_lib import _parse_issues as lib_parse_issues # noqa: E402 from test_gh_project_lib import PLAN # noqa: E402 @@ -29,7 +30,9 @@ def test_open_issue(self) -> None: labels=("milestone-1-core",)) ) self.assertIn("### Issue M1-1: Set up (#7)\n", out) - self.assertIn("**Labels**: `milestone-1-core`\n", out) + # Canonical `**Labels:**` — the same spelling the plan parser + # recognizes, so rendered blocks are valid plan input. + self.assertIn("**Labels:** `milestone-1-core`\n", out) self.assertNotIn("closed", out) self.assertNotIn("Assignees", out) @@ -44,12 +47,39 @@ def test_assignees_rendered_when_present(self) -> None: issue("M1-1", "[M1-1] Set up", gh_number=7, assignees=("octocat", "hubot")) ) - self.assertIn("**Assignees**: @octocat, @hubot\n", out) + self.assertIn("**Assignees:** @octocat, @hubot\n", out) def test_empty_body_placeholder(self) -> None: out = upd.render_issue(issue("M1-1", "[M1-1] Set up", body="", gh_number=7)) self.assertIn("_(no description on GitHub)_", out) + def test_rendered_block_reparses_with_labels_intact(self) -> None: + # Round trip: what update renders, the plan parser must read + # back — labels recognized as metadata, body free of them. + out = upd.render_issue( + issue("M1-1", "[M1-1] Set up", gh_number=7, + labels=("milestone-1-core", "documentation"), + assignees=("octocat",)) + ) + reparsed = lib_parse_issues(out) + self.assertEqual(len(reparsed), 1) + self.assertEqual(reparsed[0].labels, + ("milestone-1-core", "documentation")) + self.assertEqual(reparsed[0].gh_number, 7) + self.assertNotIn("**Labels:**", reparsed[0].body) + self.assertNotIn("**Assignees:**", reparsed[0].body) + + def test_marker_in_live_body_is_neutralized(self) -> None: + # A GitHub body containing marker-shaped text must not be able + # to close (or open) a region on the next parse. + hostile = issue( + "M1-1", "[M1-1] Set up", gh_number=7, + body="Discussing markers:\n\nend.", + ) + out = upd.render_issue(hostile) + self.assertNotRegex(out, r"