Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions docs/METRICS.md
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ All fields require GitHub PR data.
| `pr_median_size_lines` | int ≥ 0 | same | same |
| `pr_review_rounds_median` | float ≥ 0 | same | same |
| `pr_single_pass_rate` | float `0.0–1.0` | same | same |
| `pr_enrichment_degraded` | list of `basic\|enrichment\|reviews\|fetch` | `ingestion/github_reader.py` | every read step succeeded, or PRs are absent by design |

`pr_review_rounds_median` — median count of `CHANGES_REQUESTED` reviews
per PR. `pr_single_pass_rate` — fraction of PRs with zero
Expand All @@ -375,6 +376,20 @@ percentile snapshots survive ingestion drops). The platform aggregates
these counts across repos to render the org-level Cycle Time view
without persisting individual PR durations.

`pr_enrichment_degraded` — the steps of the GitHub PR read that failed this
run and fell back to partial data: `basic` (`gh pr list` for one state failed,
so that state's PRs are missing), `enrichment` (the GraphQL pass for commit
refs and merge-commit parents failed, after one retry), `reviews` (the reviews
pass failed), `fetch` (the whole read raised). Sorted, without repeats. Unlike
the fields above it does not require PR data: a read that lost every PR still
reports why. Absent when every step succeeded, and when PRs are absent by
design — no `gh`, no GitHub remote — because Iris works without PRs. When
present, this run's PR-derived fields may be missing or skewed — flow and
`commits_in_prs` go missing, and with `reviews` failed review coverage reads
0% and the single-pass rate 100% — and `merge_strategy` is `unknown` if
`enrichment` failed (§28). It is the first field to read when two runs on the
same commit disagree.

---

## 16. Duplicate block detection
Expand Down Expand Up @@ -907,6 +922,14 @@ Platform: indexed columns `merge_strategy` + `commit_metrics_reliable` on
`metrics` (migration `019`) feed the compare table; the full payload
carries `merge_strategy_dominant_share` for the repo-detail badge.

**Degraded enrichment.** When the PR read's `enrichment` step failed
(`pr_enrichment_degraded` contains `enrichment`, from any PR state, §15), the repo is `unknown`
without classifying: `merge_strategy_dominant_share` is absent and
`commit_metrics_reliable` is `true`. Without parent counts and commit refs, a
merge PR falls through to `unknown` while a squash-stamped one still
classifies, so the fallback could only ever answer `squash` — a mixed repo
would read as 100% squash, with its per-commit metrics flagged unreliable.

---

## 29. Repository kind
Expand Down
19 changes: 19 additions & 0 deletions iris/analysis/merge_strategy_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@
3. Anything ambiguous (partial presence, no commit_refs, no signal) →
``unknown`` for that PR; excluded from the dominant computation.

When the PR read's enrichment step degraded, the repo is ``unknown`` without
classifying: parent counts and commit refs are missing, so merge PRs fall
through to ``unknown`` while squash-stamped ones still classify, and the
result could only ever be ``squash``.

Aggregation
-----------
The dominant strategy over the *classified* merged PRs.
Expand Down Expand Up @@ -98,6 +103,7 @@ def detect_merge_strategy(
commits: list[Commit],
*,
min_classified: int = MIN_CLASSIFIED_PRS,
enrichment_degraded: bool = False,
) -> MergeStrategyResult:
"""Classify a repo's dominant merge strategy from its merged PRs.

Expand All @@ -108,6 +114,9 @@ def detect_merge_strategy(
commit_refs landed verbatim and to read squash subject stamps.
min_classified: minimum classifiable merged PRs before a dominant
strategy is trusted; below it the repo is ``unknown``.
enrichment_degraded: the PR read's enrichment step failed this run,
so parent counts and commit refs are missing or partial. The
classification is skipped and the repo is ``unknown``.

Returns:
``MergeStrategyResult`` — always non-None (``unknown`` when there
Expand All @@ -126,6 +135,16 @@ def detect_merge_strategy(
reason="no merged PRs in window",
)

if enrichment_degraded:
return MergeStrategyResult(
merge_strategy="unknown",
dominant_share=None,
commit_metrics_reliable=True,
classified_pr_count=0,
distribution={},
reason="PR enrichment degraded: parent counts and commit refs incomplete",
)

main_hashes = {c.hash for c in commits}
squash_pr_numbers = _squash_pr_numbers(commits)

Expand Down
14 changes: 12 additions & 2 deletions iris/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,8 @@ def _tick(label: str) -> None:
prs = pr_fetch.prs
if prs:
print(s["cli_prs_found"].format(count=len(prs)))
elif pr_fetch.degraded:
print(s["cli_prs_failed"].format(steps=", ".join(pr_fetch.degraded)))
else:
print(s["cli_prs_skipped"])
_tick("pull requests fetched")
Expand All @@ -520,6 +522,7 @@ def _tick(label: str) -> None:
churn_days=args.churn_days,
prs=prs or None,
external_data=external_data,
pr_fetch_degraded=pr_fetch.degraded,
)
print(s["cli_classified"].format(count=len(commits)))
_tick("aggregate analysis done")
Expand Down Expand Up @@ -549,6 +552,7 @@ def _tick(label: str) -> None:

recent_metrics = aggregate(
recent_commits, churn_days=args.churn_days, prs=recent_prs,
pr_fetch_degraded=pr_fetch.degraded,
)
trend = compute_trend_delta(
baseline=metrics,
Expand Down Expand Up @@ -679,8 +683,14 @@ def _tick(label: str) -> None:
pre_prs = [p for p in (prs or []) if p.created_at < event.adoption_ramp_start] or None
post_prs = [p for p in (prs or []) if p.created_at >= event.adoption_ramp_start] or None

pre_metrics = aggregate(pre_commits, churn_days=args.churn_days, prs=pre_prs)
post_metrics = aggregate(post_commits, churn_days=args.churn_days, prs=post_prs)
pre_metrics = aggregate(
pre_commits, churn_days=args.churn_days, prs=pre_prs,
pr_fetch_degraded=pr_fetch.degraded,
)
post_metrics = aggregate(
post_commits, churn_days=args.churn_days, prs=post_prs,
pr_fetch_degraded=pr_fetch.degraded,
)

# Compute day spans from commit date ranges
pre_days = max(1, (pre_commits[-1].date - pre_commits[0].date).days)
Expand Down
17 changes: 17 additions & 0 deletions iris/i18n.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@
"below describe documentation and specification changes, not software "
"delivery. They are not comparable to a service's."
),
"pr_enrichment_degraded_caveat": (
"PR data is incomplete for this run — failed steps: {steps}. Merge "
"strategy is reported as unknown when enrichment failed, and flow, "
"review and in-PR metrics may be missing or skewed. A transient "
"GitHub error usually clears on the next run; if every run fails, "
"check that gh is authenticated (gh auth status)."
),
"system_disclaimer": (
"Engineering outcomes emerge from system dynamics, domain complexity, "
"and organizational context. Metrics describe observable patterns and "
Expand Down Expand Up @@ -333,6 +340,7 @@
"cli_reading_prs": "Fetching pull requests...",
"cli_prs_found": "{count} merged PRs found.",
"cli_prs_skipped": "skipped (no GitHub remote or gh CLI).",
"cli_prs_failed": "failed ({steps}) — continuing without PR data.",
"cli_classifying": "Classifying commits...",
"cli_classified": "{count} commits classified.",
"cli_non_code_repo": (
Expand Down Expand Up @@ -911,6 +919,14 @@
"especificação, não entrega de software. Não são comparáveis às de "
"um serviço."
),
"pr_enrichment_degraded_caveat": (
"Os dados de PR estão incompletos nesta execução — passos com falha: "
"{steps}. A estratégia de merge sai como desconhecida quando o "
"enriquecimento falha, e métricas de flow, de review e de commits em "
"PR podem faltar ou sair distorcidas. Um erro transitório do GitHub "
"costuma sumir na próxima execução; se toda execução falhar, "
"verifique se o gh está autenticado (gh auth status)."
),
"system_disclaimer": (
"Resultados de engenharia emergem de dinâmicas sistêmicas, complexidade do domínio "
"e contexto organizacional. As métricas descrevem padrões observáveis e não devem "
Expand Down Expand Up @@ -1213,6 +1229,7 @@
"cli_reading_prs": "Buscando pull requests...",
"cli_prs_found": "{count} PRs merged encontrados.",
"cli_prs_skipped": "pulado (sem remote GitHub ou gh CLI).",
"cli_prs_failed": "falhou ({steps}) — seguindo sem dados de PR.",
"cli_classifying": "Classificando commits...",
"cli_classified": "{count} commits classificados.",
"cli_non_code_repo": (
Expand Down
14 changes: 12 additions & 2 deletions iris/metrics/aggregator.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from iris.models.commit import Commit
from iris.models.external import ExternalDORAData
from iris.models.metrics import ReportMetrics
from iris.models.pull_request import PullRequest
from iris.models.pull_request import DEGRADED_ENRICHMENT, PullRequest

logger = logging.getLogger(__name__)

Expand All @@ -52,6 +52,7 @@ def aggregate(
churn_days: int,
prs: list[PullRequest] | None = None,
external_data: ExternalDORAData | None = None,
pr_fetch_degraded: tuple[str, ...] = (),
) -> ReportMetrics:
"""Run all analyses on commits and return the combined ReportMetrics.

Expand All @@ -64,6 +65,10 @@ def aggregate(
provided, populates the ``dora_*`` fields on ReportMetrics;
when None, those fields stay None and the report renders the
DORA section as "not available".
pr_fetch_degraded: Steps of the PR read that failed this run
(`PullRequestFetch.degraded`). Reported as
``pr_enrichment_degraded``; when it includes ``enrichment`` the
merge strategy is not classified.

Returns:
ReportMetrics with all fields populated. PR fields are None
Expand Down Expand Up @@ -390,7 +395,11 @@ def aggregate(
# local main history. Strictly per-repository — no author axis.
merge_strategy_kwargs: dict = {}
if prs:
merge_strategy_result = detect_merge_strategy(prs, commits)
merge_strategy_result = detect_merge_strategy(
prs,
commits,
enrichment_degraded=DEGRADED_ENRICHMENT in pr_fetch_degraded,
)
merge_strategy_kwargs["merge_strategy"] = merge_strategy_result.merge_strategy
merge_strategy_kwargs["commit_metrics_reliable"] = (
merge_strategy_result.commit_metrics_reliable
Expand Down Expand Up @@ -516,6 +525,7 @@ def aggregate(
**human_review_coverage_kwargs,
**open_pr_aging_kwargs,
**merge_strategy_kwargs,
pr_enrichment_degraded=list(pr_fetch_degraded) or None,
**_dora_real_kwargs(external_data, origin_map),
)

Expand Down
7 changes: 7 additions & 0 deletions iris/models/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,13 @@ class ReportMetrics:
merge_strategy_dominant_share: float | None = None # 0.0–1.0
commit_metrics_reliable: bool | None = None

# PR read degradation — the steps of this run's GitHub PR read that failed
# and fell back to partial data (``basic``, ``enrichment``, ``reviews``,
# ``fetch``). None when every step succeeded, or when PRs were absent by
# design. When present, the PR-derived fields above may be missing or
# understated, and ``merge_strategy`` is ``unknown`` if enrichment failed.
pr_enrichment_degraded: list[str] | None = None

# Repository kind — CODE when a project manifest is tracked, NON_CODE for
# documentation / issue-board repos where nothing is built or deployed.
# Every metric below is still computed for a NON_CODE repo; the flag says
Expand Down
18 changes: 15 additions & 3 deletions iris/org_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,12 @@ def analyze_single_repo(
prs = pr_fetch.prs

# Step 3+4: Aggregate metrics
metrics = aggregate(commits, churn_days=churn_days, prs=prs or None)
metrics = aggregate(
commits,
churn_days=churn_days,
prs=prs or None,
pr_fetch_degraded=pr_fetch.degraded,
)

# Step 4a: Repository kind (keeps docs/board repos out of org comparisons)
from iris.cli import _merge_repo_kind
Expand All @@ -103,6 +108,7 @@ def analyze_single_repo(

recent_metrics = aggregate(
recent_commits, churn_days=churn_days, prs=recent_prs,
pr_fetch_degraded=pr_fetch.degraded,
)
trend = compute_trend_delta(
baseline=metrics,
Expand Down Expand Up @@ -138,8 +144,14 @@ def analyze_single_repo(
pre_prs = [p for p in (prs or []) if p.created_at < event.adoption_ramp_start] or None
post_prs = [p for p in (prs or []) if p.created_at >= event.adoption_ramp_start] or None

pre_metrics = aggregate(pre_commits, churn_days=churn_days, prs=pre_prs)
post_metrics = aggregate(post_commits, churn_days=churn_days, prs=post_prs)
pre_metrics = aggregate(
pre_commits, churn_days=churn_days, prs=pre_prs,
pr_fetch_degraded=pr_fetch.degraded,
)
post_metrics = aggregate(
post_commits, churn_days=churn_days, prs=post_prs,
pr_fetch_degraded=pr_fetch.degraded,
)

pre_days = max(1, (pre_commits[-1].date - pre_commits[0].date).days)
post_days = max(1, (post_commits[-1].date - post_commits[0].date).days)
Expand Down
8 changes: 8 additions & 0 deletions iris/reports/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,14 @@ def write_report_md(
f"",
])

# At the top, not in the PR section: a failed read can remove that section.
if metrics.pr_enrichment_degraded:
steps = ", ".join(metrics.pr_enrichment_degraded)
lines.extend([
f"> ⚠️ {s['pr_enrichment_degraded_caveat'].format(steps=steps)}",
f"",
])

lines.extend([
f"---",
f"",
Expand Down
22 changes: 22 additions & 0 deletions tests/test_merge_strategy_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,3 +165,25 @@ def test_squash_pr_numbers_extracts_trailing_stamp_only():
_commit("d", "refs #999 mid-message, not trailing"),
]
assert _squash_pr_numbers(commits) == {71, 56}


# ---------------------------------------------------------------------------
# Degraded PR enrichment (issue #236)
# ---------------------------------------------------------------------------


def test_degraded_enrichment_reports_unknown_instead_of_squash():
# What a degraded read leaves behind: merged PRs with no commit refs and
# no parent counts, and squash-stamped subjects on main. Only the stamps
# classify, so the heuristic can only ever answer "squash".
prs = [_merged_pr(i) for i in range(1, 7)]
commits = [_commit(f"c{i}", f"change {i} (#{i})") for i in range(1, 7)]

assert detect_merge_strategy(prs, commits).merge_strategy == "squash"

result = detect_merge_strategy(prs, commits, enrichment_degraded=True)

assert result.merge_strategy == "unknown"
assert result.dominant_share is None
assert result.commit_metrics_reliable is True
assert "enrichment" in result.reason
70 changes: 70 additions & 0 deletions tests/test_pr_enrichment_caveat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
"""Tests the report caveat for runs whose PR read degraded (issue #236).

A degraded PR read leaves the merge strategy unknown and flow, review and
in-PR metrics missing or understated. The report has to say so at the top,
where a reader meets it before any number — not inside the PR section, which a
failed read can remove entirely.

Runnable as: `python -m pytest tests/test_pr_enrichment_caveat.py -v`
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from iris.i18n import get_strings
from iris.models.context import AnalysisContext
from iris.models.metrics import ReportMetrics
from iris.reports.writer import write_report_md

_CAVEAT_EN = "PR data is incomplete for this run"
_CAVEAT_PT = "Os dados de PR estão incompletos nesta execução"


def _report_text(out_dir: Path, degraded: list[str] | None, lang: str = "en") -> str:
ctx = AnalysisContext(
repo_path=str(out_dir),
repo_name="widgets",
days=90,
churn_days=14,
out_dir=str(out_dir),
lang=lang,
)
metrics = ReportMetrics(
commits_total=100,
commits_revert=10,
revert_rate=0.1,
churn_events=5,
churn_lines_affected=500,
files_touched=50,
files_stabilized=45,
stabilization_ratio=0.90,
pr_enrichment_degraded=degraded,
)
path = write_report_md(ctx, metrics, out_dir=str(out_dir))
return Path(path).read_text(encoding="utf-8")


def test_degraded_run_states_the_caveat_with_the_failed_steps(tmp_path):
text = _report_text(tmp_path, ["enrichment", "reviews"])

assert _CAVEAT_EN in text
assert "enrichment, reviews" in text


def test_caveat_does_not_depend_on_the_pr_section(tmp_path):
# A basic pass that failed in every state leaves no PR data, so the PR
# section is not rendered at all — the caveat must still be there.
text = _report_text(tmp_path, ["basic"])

assert get_strings("en")["section_pr_lifecycle"] not in text
assert _CAVEAT_EN in text


def test_caveat_is_translated(tmp_path):
assert _CAVEAT_PT in _report_text(tmp_path, ["enrichment"], lang="pt-br")


def test_clean_run_has_no_caveat(tmp_path):
assert _CAVEAT_EN not in _report_text(tmp_path, None)
Loading
Loading