From 81193ebd17a588df6b2a9dbc600b8a91f3652c5f Mon Sep 17 00:00:00 2001 From: Carlos Penaforte Date: Mon, 5 Oct 2026 14:40:03 -0300 Subject: [PATCH 1/4] fix(engine): emit PR read degradation, stop guessing merge strategy The read now knows which steps degraded; this carries that into the metrics. ReportMetrics gains pr_enrichment_degraded, omitted when nothing degraded so a clean run's metrics.json is unchanged, and both entry points pass the read's degradation to aggregate(). Merge strategy changes behavior on purpose. With enrichment degraded, merge PRs have no parent count and no commit refs and fall through to unknown, while squash-stamped ones still classify - so the fallback could only ever answer squash. A mixed repo read as 100% squash with its per-commit metrics flagged unreliable. The detector now reports unknown, with the reason. Refs #236 Assisted-by: Claude Code Co-Authored-By: Claude Haiku 4.5 --- iris/analysis/merge_strategy_detector.py | 19 ++++ iris/cli.py | 1 + iris/metrics/aggregator.py | 14 ++- iris/models/metrics.py | 7 ++ iris/org_runner.py | 7 +- tests/test_merge_strategy_detector.py | 22 +++++ tests/test_pr_fetch_degradation.py | 114 +++++++++++++++++++++++ 7 files changed, 181 insertions(+), 3 deletions(-) diff --git a/iris/analysis/merge_strategy_detector.py b/iris/analysis/merge_strategy_detector.py index 3e5ce9a..3cc47b0 100644 --- a/iris/analysis/merge_strategy_detector.py +++ b/iris/analysis/merge_strategy_detector.py @@ -37,6 +37,11 @@ 3. Anything ambiguous (partial presence, no commit_refs, no signal) → ``unknown`` for that PR; excluded from the dominant computation. +When the PR read's enrichment step degraded, the repo is ``unknown`` without +classifying: parent counts and commit refs are missing, so merge PRs fall +through to ``unknown`` while squash-stamped ones still classify, and the +result could only ever be ``squash``. + Aggregation ----------- The dominant strategy over the *classified* merged PRs. @@ -98,6 +103,7 @@ def detect_merge_strategy( commits: list[Commit], *, min_classified: int = MIN_CLASSIFIED_PRS, + enrichment_degraded: bool = False, ) -> MergeStrategyResult: """Classify a repo's dominant merge strategy from its merged PRs. @@ -108,6 +114,9 @@ def detect_merge_strategy( commit_refs landed verbatim and to read squash subject stamps. min_classified: minimum classifiable merged PRs before a dominant strategy is trusted; below it the repo is ``unknown``. + enrichment_degraded: the PR read's enrichment step failed this run, + so parent counts and commit refs are missing or partial. The + classification is skipped and the repo is ``unknown``. Returns: ``MergeStrategyResult`` — always non-None (``unknown`` when there @@ -126,6 +135,16 @@ def detect_merge_strategy( reason="no merged PRs in window", ) + if enrichment_degraded: + return MergeStrategyResult( + merge_strategy="unknown", + dominant_share=None, + commit_metrics_reliable=True, + classified_pr_count=0, + distribution={}, + reason="PR enrichment degraded: parent counts and commit refs incomplete", + ) + main_hashes = {c.hash for c in commits} squash_pr_numbers = _squash_pr_numbers(commits) diff --git a/iris/cli.py b/iris/cli.py index 9416140..fd96c7f 100644 --- a/iris/cli.py +++ b/iris/cli.py @@ -520,6 +520,7 @@ def _tick(label: str) -> None: churn_days=args.churn_days, prs=prs or None, external_data=external_data, + pr_fetch_degraded=pr_fetch.degraded, ) print(s["cli_classified"].format(count=len(commits))) _tick("aggregate analysis done") diff --git a/iris/metrics/aggregator.py b/iris/metrics/aggregator.py index 7d0569a..79cae9d 100644 --- a/iris/metrics/aggregator.py +++ b/iris/metrics/aggregator.py @@ -42,7 +42,7 @@ from iris.models.commit import Commit from iris.models.external import ExternalDORAData from iris.models.metrics import ReportMetrics -from iris.models.pull_request import PullRequest +from iris.models.pull_request import DEGRADED_ENRICHMENT, PullRequest logger = logging.getLogger(__name__) @@ -52,6 +52,7 @@ def aggregate( churn_days: int, prs: list[PullRequest] | None = None, external_data: ExternalDORAData | None = None, + pr_fetch_degraded: tuple[str, ...] = (), ) -> ReportMetrics: """Run all analyses on commits and return the combined ReportMetrics. @@ -64,6 +65,10 @@ def aggregate( provided, populates the ``dora_*`` fields on ReportMetrics; when None, those fields stay None and the report renders the DORA section as "not available". + pr_fetch_degraded: Steps of the PR read that failed this run + (`PullRequestFetch.degraded`). Reported as + ``pr_enrichment_degraded``; when it includes ``enrichment`` the + merge strategy is not classified. Returns: ReportMetrics with all fields populated. PR fields are None @@ -390,7 +395,11 @@ def aggregate( # local main history. Strictly per-repository — no author axis. merge_strategy_kwargs: dict = {} if prs: - merge_strategy_result = detect_merge_strategy(prs, commits) + merge_strategy_result = detect_merge_strategy( + prs, + commits, + enrichment_degraded=DEGRADED_ENRICHMENT in pr_fetch_degraded, + ) merge_strategy_kwargs["merge_strategy"] = merge_strategy_result.merge_strategy merge_strategy_kwargs["commit_metrics_reliable"] = ( merge_strategy_result.commit_metrics_reliable @@ -516,6 +525,7 @@ def aggregate( **human_review_coverage_kwargs, **open_pr_aging_kwargs, **merge_strategy_kwargs, + pr_enrichment_degraded=list(pr_fetch_degraded) or None, **_dora_real_kwargs(external_data, origin_map), ) diff --git a/iris/models/metrics.py b/iris/models/metrics.py index fc4d2b8..57a700a 100644 --- a/iris/models/metrics.py +++ b/iris/models/metrics.py @@ -165,6 +165,13 @@ class ReportMetrics: merge_strategy_dominant_share: float | None = None # 0.0–1.0 commit_metrics_reliable: bool | None = None + # PR read degradation — the steps of this run's GitHub PR read that failed + # and fell back to partial data (``basic``, ``enrichment``, ``reviews``, + # ``fetch``). None when every step succeeded, or when PRs were absent by + # design. When present, the PR-derived fields above may be missing or + # understated, and ``merge_strategy`` is ``unknown`` if enrichment failed. + pr_enrichment_degraded: list[str] | None = None + # Repository kind — CODE when a project manifest is tracked, NON_CODE for # documentation / issue-board repos where nothing is built or deployed. # Every metric below is still computed for a NON_CODE repo; the flag says diff --git a/iris/org_runner.py b/iris/org_runner.py index 96ebcd1..7046157 100644 --- a/iris/org_runner.py +++ b/iris/org_runner.py @@ -85,7 +85,12 @@ def analyze_single_repo( prs = pr_fetch.prs # Step 3+4: Aggregate metrics - metrics = aggregate(commits, churn_days=churn_days, prs=prs or None) + metrics = aggregate( + commits, + churn_days=churn_days, + prs=prs or None, + pr_fetch_degraded=pr_fetch.degraded, + ) # Step 4a: Repository kind (keeps docs/board repos out of org comparisons) from iris.cli import _merge_repo_kind diff --git a/tests/test_merge_strategy_detector.py b/tests/test_merge_strategy_detector.py index a3c62c1..011e64f 100644 --- a/tests/test_merge_strategy_detector.py +++ b/tests/test_merge_strategy_detector.py @@ -165,3 +165,25 @@ def test_squash_pr_numbers_extracts_trailing_stamp_only(): _commit("d", "refs #999 mid-message, not trailing"), ] assert _squash_pr_numbers(commits) == {71, 56} + + +# --------------------------------------------------------------------------- +# Degraded PR enrichment (issue #236) +# --------------------------------------------------------------------------- + + +def test_degraded_enrichment_reports_unknown_instead_of_squash(): + # What a degraded read leaves behind: merged PRs with no commit refs and + # no parent counts, and squash-stamped subjects on main. Only the stamps + # classify, so the heuristic can only ever answer "squash". + prs = [_merged_pr(i) for i in range(1, 7)] + commits = [_commit(f"c{i}", f"change {i} (#{i})") for i in range(1, 7)] + + assert detect_merge_strategy(prs, commits).merge_strategy == "squash" + + result = detect_merge_strategy(prs, commits, enrichment_degraded=True) + + assert result.merge_strategy == "unknown" + assert result.dominant_share is None + assert result.commit_metrics_reliable is True + assert "enrichment" in result.reason diff --git a/tests/test_pr_fetch_degradation.py b/tests/test_pr_fetch_degradation.py index bcc49a9..f48ee70 100644 --- a/tests/test_pr_fetch_degradation.py +++ b/tests/test_pr_fetch_degradation.py @@ -257,3 +257,117 @@ def test_degraded_steps_from_every_state_are_merged_and_sorted(monkeypatch): fetch = github_reader.read_pull_requests("/any/repo", days=30) assert fetch.degraded == ("enrichment", "reviews") + + +# --- the signal in the metrics --------------------------------------------- + +import argparse +import glob +from datetime import datetime, timedelta, timezone + +from iris.metrics.aggregator import aggregate +from iris.models.commit import Commit +from iris.models.pull_request import PullRequest + +_NOW = datetime(2026, 9, 1, tzinfo=timezone.utc) + + +def _stamped_commits() -> list[Commit]: + # Squash-stamped subjects: under degraded enrichment they are all the + # fallback heuristic can see. + return [ + Commit(hash=f"c{n}", author="dev", date=_NOW + timedelta(hours=n), + message=f"change {n} (#{n})") + for n in range(1, 7) + ] + + +def _merged_prs() -> list[PullRequest]: + return [ + PullRequest(number=n, title=f"PR {n}", author="dev", created_at=_NOW, + additions=1, deletions=0, changed_files=1, + merged_at=_NOW + timedelta(hours=n), + closed_at=_NOW + timedelta(hours=n), state="merged") + for n in range(1, 7) + ] + + +def test_degraded_fetch_reaches_the_metrics_payload(): + metrics = aggregate( + _stamped_commits(), churn_days=14, prs=_merged_prs(), + pr_fetch_degraded=("enrichment", "reviews"), + ) + payload = metrics.to_dict() + + assert payload["pr_enrichment_degraded"] == ["enrichment", "reviews"] + assert payload["merge_strategy"] == "unknown" + assert "merge_strategy_dominant_share" not in payload + + +def test_clean_fetch_leaves_the_payload_unchanged(): + metrics = aggregate(_stamped_commits(), churn_days=14, prs=_merged_prs()) + payload = metrics.to_dict() + + # No key at all, so a clean run's metrics.json is what it was before. + assert "pr_enrichment_degraded" not in payload + assert payload["merge_strategy"] == "squash" + + +# --- both entry points carry it into metrics.json --------------------------- + + +def _build_repo(path: Path) -> None: + """A small git repo, hermetic against the machine's global git config.""" + def git(*args): + subprocess.run(["git", *args], cwd=path, check=True, capture_output=True) + + git("init", "-q", "-b", "main", ".") + git("config", "user.email", "test@example.com") + git("config", "user.name", "Test") + git("config", "commit.gpgsign", "false") + git("config", "core.hooksPath", str(path / "no-hooks")) + for n in range(1, 4): + (path / "a.py").write_text(f"x = {n}\n", encoding="utf-8") + git("add", "-A") + git("commit", "-q", "-m", f"feat: change {n}") + + +def _degraded_read(_repo_path, days): + return PullRequestFetch(prs=[], degraded=("enrichment",)) + + +def _metrics_json(out_dir: Path) -> dict: + [path] = glob.glob(str(out_dir / "**" / "*-metrics.json"), recursive=True) + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def test_single_repo_cli_writes_the_degradation_to_metrics_json(tmp_path, monkeypatch): + from iris import cli + + repo = tmp_path / "repo" + repo.mkdir() + _build_repo(repo) + monkeypatch.setattr(cli, "read_pull_requests_with_fallback", _degraded_read) + + args = argparse.Namespace( + repo_path=str(repo), days=30, churn_days=14, lang="en", recent_days=30, + verbose=False, trend=False, out=str(tmp_path / "out"), no_push=True, + ) + cli._run_single_repo(args) + + assert _metrics_json(tmp_path / "out")["pr_enrichment_degraded"] == ["enrichment"] + + +def test_org_runner_writes_the_degradation_to_metrics_json(tmp_path, monkeypatch): + from iris import org_runner + + repo = tmp_path / "repo" + repo.mkdir() + _build_repo(repo) + monkeypatch.setattr(org_runner, "read_pull_requests_with_fallback", _degraded_read) + + org_runner.analyze_single_repo( + str(repo), days=30, churn_days=14, out_dir=str(tmp_path / "out"), + ) + + assert _metrics_json(tmp_path / "out")["pr_enrichment_degraded"] == ["enrichment"] From 16fdcfaa1d303f62cd8121456dc1279f52ee81f4 Mon Sep 17 00:00:00 2001 From: Carlos Penaforte Date: Mon, 5 Oct 2026 14:45:45 -0300 Subject: [PATCH 2/4] fix(engine): pass PR read degradation to trend and adoption splits The recent-window and pre/post-adoption metrics are aggregated from slices of the same PR list, but were not told the read degraded. The adoption split is written into metrics.json, so a degraded run still carried a guessed merge strategy there, nested under the top-level unknown. Every aggregate() over this run's PRs now receives the read's degradation. Refs #236 Assisted-by: Claude Code Co-Authored-By: Claude Haiku 4.5 --- iris/cli.py | 11 +++++- iris/org_runner.py | 11 +++++- tests/test_pr_fetch_degradation.py | 61 ++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/iris/cli.py b/iris/cli.py index fd96c7f..e4e1fcc 100644 --- a/iris/cli.py +++ b/iris/cli.py @@ -550,6 +550,7 @@ def _tick(label: str) -> None: recent_metrics = aggregate( recent_commits, churn_days=args.churn_days, prs=recent_prs, + pr_fetch_degraded=pr_fetch.degraded, ) trend = compute_trend_delta( baseline=metrics, @@ -680,8 +681,14 @@ def _tick(label: str) -> None: pre_prs = [p for p in (prs or []) if p.created_at < event.adoption_ramp_start] or None post_prs = [p for p in (prs or []) if p.created_at >= event.adoption_ramp_start] or None - pre_metrics = aggregate(pre_commits, churn_days=args.churn_days, prs=pre_prs) - post_metrics = aggregate(post_commits, churn_days=args.churn_days, prs=post_prs) + pre_metrics = aggregate( + pre_commits, churn_days=args.churn_days, prs=pre_prs, + pr_fetch_degraded=pr_fetch.degraded, + ) + post_metrics = aggregate( + post_commits, churn_days=args.churn_days, prs=post_prs, + pr_fetch_degraded=pr_fetch.degraded, + ) # Compute day spans from commit date ranges pre_days = max(1, (pre_commits[-1].date - pre_commits[0].date).days) diff --git a/iris/org_runner.py b/iris/org_runner.py index 7046157..5ae2434 100644 --- a/iris/org_runner.py +++ b/iris/org_runner.py @@ -108,6 +108,7 @@ def analyze_single_repo( recent_metrics = aggregate( recent_commits, churn_days=churn_days, prs=recent_prs, + pr_fetch_degraded=pr_fetch.degraded, ) trend = compute_trend_delta( baseline=metrics, @@ -143,8 +144,14 @@ def analyze_single_repo( pre_prs = [p for p in (prs or []) if p.created_at < event.adoption_ramp_start] or None post_prs = [p for p in (prs or []) if p.created_at >= event.adoption_ramp_start] or None - pre_metrics = aggregate(pre_commits, churn_days=churn_days, prs=pre_prs) - post_metrics = aggregate(post_commits, churn_days=churn_days, prs=post_prs) + pre_metrics = aggregate( + pre_commits, churn_days=churn_days, prs=pre_prs, + pr_fetch_degraded=pr_fetch.degraded, + ) + post_metrics = aggregate( + post_commits, churn_days=churn_days, prs=post_prs, + pr_fetch_degraded=pr_fetch.degraded, + ) pre_days = max(1, (pre_commits[-1].date - pre_commits[0].date).days) post_days = max(1, (post_commits[-1].date - post_commits[0].date).days) diff --git a/tests/test_pr_fetch_degradation.py b/tests/test_pr_fetch_degradation.py index f48ee70..efdc348 100644 --- a/tests/test_pr_fetch_degradation.py +++ b/tests/test_pr_fetch_degradation.py @@ -371,3 +371,64 @@ def test_org_runner_writes_the_degradation_to_metrics_json(tmp_path, monkeypatch ) assert _metrics_json(tmp_path / "out")["pr_enrichment_degraded"] == ["enrichment"] + + +# --- the adoption split inherits it ----------------------------------------- + + +def _adoption_after_first_commit(monkeypatch): + """Make adoption detection split the history after its first commit.""" + from iris.analysis import adoption_detector + from iris.models.adoption import AdoptionEvent + + def detect(commits): + ordered = sorted(commits, key=lambda c: c.date) + start = ordered[1].date + event = AdoptionEvent( + first_ai_commit_date=start, adoption_ramp_start=start, + adoption_ramp_end=ordered[-1].date, adoption_confidence="clear", + total_ai_commits=2, + ) + return event, ordered[:1], ordered[1:] + + monkeypatch.setattr(adoption_detector, "detect_adoption", detect) + + +def _assert_adoption_split_is_degraded(out_dir: Path) -> None: + timeline = _metrics_json(out_dir)["adoption_timeline"] + assert timeline["pre_adoption"]["pr_enrichment_degraded"] == ["enrichment"] + assert timeline["post_adoption"]["pr_enrichment_degraded"] == ["enrichment"] + + +def test_single_repo_cli_adoption_split_inherits_the_degradation(tmp_path, monkeypatch): + from iris import cli + + repo = tmp_path / "repo" + repo.mkdir() + _build_repo(repo) + monkeypatch.setattr(cli, "read_pull_requests_with_fallback", _degraded_read) + _adoption_after_first_commit(monkeypatch) + + args = argparse.Namespace( + repo_path=str(repo), days=30, churn_days=14, lang="en", recent_days=30, + verbose=False, trend=False, out=str(tmp_path / "out"), no_push=True, + ) + cli._run_single_repo(args) + + _assert_adoption_split_is_degraded(tmp_path / "out") + + +def test_org_runner_adoption_split_inherits_the_degradation(tmp_path, monkeypatch): + from iris import org_runner + + repo = tmp_path / "repo" + repo.mkdir() + _build_repo(repo) + monkeypatch.setattr(org_runner, "read_pull_requests_with_fallback", _degraded_read) + _adoption_after_first_commit(monkeypatch) + + org_runner.analyze_single_repo( + str(repo), days=30, churn_days=14, out_dir=str(tmp_path / "out"), + ) + + _assert_adoption_split_is_degraded(tmp_path / "out") From eecce8b7e90ae462cb55cff75a30157a13ea1c45 Mon Sep 17 00:00:00 2001 From: Carlos Penaforte Date: Mon, 5 Oct 2026 14:48:04 -0300 Subject: [PATCH 3/4] feat(engine): warn atop the report when PR data is incomplete The metrics now carry the degradation, but the report still presented partial PR numbers with the same confidence as complete ones. It states the failed steps at the top, next to the non-code-repository caveat - not in the PR section, which a read that lost every PR does not render. English and Portuguese. docs/METRICS.md documents the field in section 15 and the merge-strategy rule in section 28. Refs #236 Assisted-by: Claude Code Co-Authored-By: Claude Haiku 4.5 --- docs/METRICS.md | 22 ++++++++++ iris/i18n.py | 13 ++++++ iris/reports/writer.py | 8 ++++ tests/test_pr_enrichment_caveat.py | 70 ++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 tests/test_pr_enrichment_caveat.py diff --git a/docs/METRICS.md b/docs/METRICS.md index d09a569..6b00261 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -362,6 +362,7 @@ All fields require GitHub PR data. | `pr_median_size_lines` | int ≥ 0 | same | same | | `pr_review_rounds_median` | float ≥ 0 | same | same | | `pr_single_pass_rate` | float `0.0–1.0` | same | same | +| `pr_enrichment_degraded` | list of `basic\|enrichment\|reviews\|fetch` | `ingestion/github_reader.py` | every read step succeeded, or PRs are absent by design | `pr_review_rounds_median` — median count of `CHANGES_REQUESTED` reviews per PR. `pr_single_pass_rate` — fraction of PRs with zero @@ -375,6 +376,19 @@ percentile snapshots survive ingestion drops). The platform aggregates these counts across repos to render the org-level Cycle Time view without persisting individual PR durations. +`pr_enrichment_degraded` — the steps of the GitHub PR read that failed this +run and fell back to partial data: `basic` (`gh pr list` for one state failed, +so that state's PRs are missing), `enrichment` (the GraphQL pass for commit +refs and merge-commit parents failed, after one retry), `reviews` (the reviews +pass failed), `fetch` (the whole read raised). Sorted, without repeats. Unlike +the fields above it does not require PR data: a read that lost every PR still +reports why. Absent when every step succeeded, and when PRs are absent by +design — no `gh`, no GitHub remote — because Iris works without PRs. When +present, this run's PR-derived fields may be missing or understated (flow, +review coverage, `commits_in_prs`), and `merge_strategy` is `unknown` if +`enrichment` failed (§28). It is the first field to read when two runs on the +same commit disagree. + --- ## 16. Duplicate block detection @@ -907,6 +921,14 @@ Platform: indexed columns `merge_strategy` + `commit_metrics_reliable` on `metrics` (migration `019`) feed the compare table; the full payload carries `merge_strategy_dominant_share` for the repo-detail badge. +**Degraded enrichment.** When the PR read's `enrichment` step failed +(`pr_enrichment_degraded` contains `enrichment`, §15), the repo is `unknown` +without classifying: `merge_strategy_dominant_share` is absent and +`commit_metrics_reliable` is `true`. Without parent counts and commit refs, a +merge PR falls through to `unknown` while a squash-stamped one still +classifies, so the fallback could only ever answer `squash` — a mixed repo +would read as 100% squash, with its per-commit metrics flagged unreliable. + --- ## 29. Repository kind diff --git a/iris/i18n.py b/iris/i18n.py index 20fb183..62d0862 100644 --- a/iris/i18n.py +++ b/iris/i18n.py @@ -33,6 +33,12 @@ "below describe documentation and specification changes, not software " "delivery. They are not comparable to a service's." ), + "pr_enrichment_degraded_caveat": ( + "PR data is incomplete for this run — failed steps: {steps}. Merge " + "strategy is reported as unknown when enrichment failed, and flow, " + "review and in-PR metrics may be missing or understated. A re-run " + "usually recovers them." + ), "system_disclaimer": ( "Engineering outcomes emerge from system dynamics, domain complexity, " "and organizational context. Metrics describe observable patterns and " @@ -911,6 +917,13 @@ "especificação, não entrega de software. Não são comparáveis às de " "um serviço." ), + "pr_enrichment_degraded_caveat": ( + "Os dados de PR estão incompletos nesta execução — passos com falha: " + "{steps}. A estratégia de merge sai como desconhecida quando o " + "enriquecimento falha, e métricas de flow, de review e de commits em " + "PR podem faltar ou sair subestimadas. Uma nova execução costuma " + "recuperá-las." + ), "system_disclaimer": ( "Resultados de engenharia emergem de dinâmicas sistêmicas, complexidade do domínio " "e contexto organizacional. As métricas descrevem padrões observáveis e não devem " diff --git a/iris/reports/writer.py b/iris/reports/writer.py index b91bdb4..a93b839 100644 --- a/iris/reports/writer.py +++ b/iris/reports/writer.py @@ -172,6 +172,14 @@ def write_report_md( f"", ]) + # At the top, not in the PR section: a failed read can remove that section. + if metrics.pr_enrichment_degraded: + steps = ", ".join(metrics.pr_enrichment_degraded) + lines.extend([ + f"> ⚠️ {s['pr_enrichment_degraded_caveat'].format(steps=steps)}", + f"", + ]) + lines.extend([ f"---", f"", diff --git a/tests/test_pr_enrichment_caveat.py b/tests/test_pr_enrichment_caveat.py new file mode 100644 index 0000000..d35ffc2 --- /dev/null +++ b/tests/test_pr_enrichment_caveat.py @@ -0,0 +1,70 @@ +"""Tests the report caveat for runs whose PR read degraded (issue #236). + +A degraded PR read leaves the merge strategy unknown and flow, review and +in-PR metrics missing or understated. The report has to say so at the top, +where a reader meets it before any number — not inside the PR section, which a +failed read can remove entirely. + +Runnable as: `python -m pytest tests/test_pr_enrichment_caveat.py -v` +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from iris.i18n import get_strings +from iris.models.context import AnalysisContext +from iris.models.metrics import ReportMetrics +from iris.reports.writer import write_report_md + +_CAVEAT_EN = "PR data is incomplete for this run" +_CAVEAT_PT = "Os dados de PR estão incompletos nesta execução" + + +def _report_text(out_dir: Path, degraded: list[str] | None, lang: str = "en") -> str: + ctx = AnalysisContext( + repo_path=str(out_dir), + repo_name="widgets", + days=90, + churn_days=14, + out_dir=str(out_dir), + lang=lang, + ) + metrics = ReportMetrics( + commits_total=100, + commits_revert=10, + revert_rate=0.1, + churn_events=5, + churn_lines_affected=500, + files_touched=50, + files_stabilized=45, + stabilization_ratio=0.90, + pr_enrichment_degraded=degraded, + ) + path = write_report_md(ctx, metrics, out_dir=str(out_dir)) + return Path(path).read_text(encoding="utf-8") + + +def test_degraded_run_states_the_caveat_with_the_failed_steps(tmp_path): + text = _report_text(tmp_path, ["enrichment", "reviews"]) + + assert _CAVEAT_EN in text + assert "enrichment, reviews" in text + + +def test_caveat_does_not_depend_on_the_pr_section(tmp_path): + # A basic pass that failed in every state leaves no PR data, so the PR + # section is not rendered at all — the caveat must still be there. + text = _report_text(tmp_path, ["basic"]) + + assert get_strings("en")["section_pr_lifecycle"] not in text + assert _CAVEAT_EN in text + + +def test_caveat_is_translated(tmp_path): + assert _CAVEAT_PT in _report_text(tmp_path, ["enrichment"], lang="pt-br") + + +def test_clean_run_has_no_caveat(tmp_path): + assert _CAVEAT_EN not in _report_text(tmp_path, None) From 10441c46847255f6574a110a191b84a26e1a6a9a Mon Sep 17 00:00:00 2001 From: Carlos Penaforte Date: Mon, 5 Oct 2026 15:07:24 -0300 Subject: [PATCH 4/4] fix(engine): make the PR read caveat accurate when gh is unusable The caveat promised that a re-run usually recovers the missing PR data, which is false when gh is installed but not authenticated: then every run fails the basic step. It also called the affected metrics understated, but with the reviews step failed, review coverage reads 0% and the single-pass rate 100% - skewed, not understated. The caveat and the METRICS.md entry now say so, and point to gh auth status when every run fails. The console no longer reports a failed read as skipped. Refs #236 Assisted-by: Claude Code Co-Authored-By: Claude Sonnet 5.5 --- docs/METRICS.md | 7 ++++--- iris/cli.py | 2 ++ iris/i18n.py | 12 ++++++++---- tests/test_pr_fetch_degradation.py | 25 +++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/METRICS.md b/docs/METRICS.md index 6b00261..413f0af 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -384,8 +384,9 @@ pass failed), `fetch` (the whole read raised). Sorted, without repeats. Unlike the fields above it does not require PR data: a read that lost every PR still reports why. Absent when every step succeeded, and when PRs are absent by design — no `gh`, no GitHub remote — because Iris works without PRs. When -present, this run's PR-derived fields may be missing or understated (flow, -review coverage, `commits_in_prs`), and `merge_strategy` is `unknown` if +present, this run's PR-derived fields may be missing or skewed — flow and +`commits_in_prs` go missing, and with `reviews` failed review coverage reads +0% and the single-pass rate 100% — and `merge_strategy` is `unknown` if `enrichment` failed (§28). It is the first field to read when two runs on the same commit disagree. @@ -922,7 +923,7 @@ Platform: indexed columns `merge_strategy` + `commit_metrics_reliable` on carries `merge_strategy_dominant_share` for the repo-detail badge. **Degraded enrichment.** When the PR read's `enrichment` step failed -(`pr_enrichment_degraded` contains `enrichment`, §15), the repo is `unknown` +(`pr_enrichment_degraded` contains `enrichment`, from any PR state, §15), the repo is `unknown` without classifying: `merge_strategy_dominant_share` is absent and `commit_metrics_reliable` is `true`. Without parent counts and commit refs, a merge PR falls through to `unknown` while a squash-stamped one still diff --git a/iris/cli.py b/iris/cli.py index e4e1fcc..755eb41 100644 --- a/iris/cli.py +++ b/iris/cli.py @@ -500,6 +500,8 @@ def _tick(label: str) -> None: prs = pr_fetch.prs if prs: print(s["cli_prs_found"].format(count=len(prs))) + elif pr_fetch.degraded: + print(s["cli_prs_failed"].format(steps=", ".join(pr_fetch.degraded))) else: print(s["cli_prs_skipped"]) _tick("pull requests fetched") diff --git a/iris/i18n.py b/iris/i18n.py index 62d0862..4522bb8 100644 --- a/iris/i18n.py +++ b/iris/i18n.py @@ -36,8 +36,9 @@ "pr_enrichment_degraded_caveat": ( "PR data is incomplete for this run — failed steps: {steps}. Merge " "strategy is reported as unknown when enrichment failed, and flow, " - "review and in-PR metrics may be missing or understated. A re-run " - "usually recovers them." + "review and in-PR metrics may be missing or skewed. A transient " + "GitHub error usually clears on the next run; if every run fails, " + "check that gh is authenticated (gh auth status)." ), "system_disclaimer": ( "Engineering outcomes emerge from system dynamics, domain complexity, " @@ -339,6 +340,7 @@ "cli_reading_prs": "Fetching pull requests...", "cli_prs_found": "{count} merged PRs found.", "cli_prs_skipped": "skipped (no GitHub remote or gh CLI).", + "cli_prs_failed": "failed ({steps}) — continuing without PR data.", "cli_classifying": "Classifying commits...", "cli_classified": "{count} commits classified.", "cli_non_code_repo": ( @@ -921,8 +923,9 @@ "Os dados de PR estão incompletos nesta execução — passos com falha: " "{steps}. A estratégia de merge sai como desconhecida quando o " "enriquecimento falha, e métricas de flow, de review e de commits em " - "PR podem faltar ou sair subestimadas. Uma nova execução costuma " - "recuperá-las." + "PR podem faltar ou sair distorcidas. Um erro transitório do GitHub " + "costuma sumir na próxima execução; se toda execução falhar, " + "verifique se o gh está autenticado (gh auth status)." ), "system_disclaimer": ( "Resultados de engenharia emergem de dinâmicas sistêmicas, complexidade do domínio " @@ -1226,6 +1229,7 @@ "cli_reading_prs": "Buscando pull requests...", "cli_prs_found": "{count} PRs merged encontrados.", "cli_prs_skipped": "pulado (sem remote GitHub ou gh CLI).", + "cli_prs_failed": "falhou ({steps}) — seguindo sem dados de PR.", "cli_classifying": "Classificando commits...", "cli_classified": "{count} commits classificados.", "cli_non_code_repo": ( diff --git a/tests/test_pr_fetch_degradation.py b/tests/test_pr_fetch_degradation.py index efdc348..fd7c83e 100644 --- a/tests/test_pr_fetch_degradation.py +++ b/tests/test_pr_fetch_degradation.py @@ -432,3 +432,28 @@ def test_org_runner_adoption_split_inherits_the_degradation(tmp_path, monkeypatc ) _assert_adoption_split_is_degraded(tmp_path / "out") + + +def test_single_repo_cli_says_the_pr_read_failed_instead_of_skipped( + tmp_path, monkeypatch, capsys +): + from iris import cli + + repo = tmp_path / "repo" + repo.mkdir() + _build_repo(repo) + monkeypatch.setattr( + cli, + "read_pull_requests_with_fallback", + lambda *a, **k: PullRequestFetch(prs=[], degraded=("basic",)), + ) + + args = argparse.Namespace( + repo_path=str(repo), days=30, churn_days=14, lang="en", recent_days=30, + verbose=False, trend=False, out=str(tmp_path / "out"), no_push=True, + ) + cli._run_single_repo(args) + + out = capsys.readouterr().out + assert "failed (basic)" in out + assert "skipped (no GitHub remote or gh CLI)" not in out