From 97a14ac6087db3c9205e66bcfbcc890dc23a7ca7 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 17:18:36 +0800 Subject: [PATCH 1/6] ci: complete pre-visibility validation --- .github/workflows/ci.yml | 34 +- .github/workflows/live-smoke.yml | 4 +- .github/workflows/publish.yml | 18 +- pyproject.toml | 2 + scripts/check_version.py | 5 + scripts/check_workflows.py | 1354 +++++++++++++++++++++++------- tests/test_release_documents.py | 8 + tests/test_release_workflow.py | 483 ++++++++++- uv.lock | 77 ++ 9 files changed, 1671 insertions(+), 314 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66edbce..704200c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,9 +25,9 @@ jobs: timeout-minutes: 20 steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend @@ -65,9 +65,9 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Install the pinned uv frontend @@ -83,9 +83,9 @@ jobs: timeout-minutes: 20 steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" - name: Install the pinned uv frontend @@ -104,9 +104,9 @@ jobs: timeout-minutes: 20 steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend @@ -125,9 +125,9 @@ jobs: timeout-minutes: 25 steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend @@ -145,7 +145,7 @@ jobs: - name: Record immutable artifact digests run: sha256sum dist/* > artifact-sha256.txt - name: Retain verified artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: python-distributions path: | @@ -161,12 +161,20 @@ jobs: timeout-minutes: 35 steps: - name: Check out the candidate - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend run: python -m pip install --disable-pip-version-check "uv==$UV_VERSION" - name: Verify from a copied standalone repository run: python scripts/check_repository_independence.py + - name: Download the verified package artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-distributions + path: verified-artifacts + - name: Recheck retained artifact digests + working-directory: verified-artifacts + run: sha256sum --check artifact-sha256.txt diff --git a/.github/workflows/live-smoke.yml b/.github/workflows/live-smoke.yml index 4026d25..f51a1f7 100644 --- a/.github/workflows/live-smoke.yml +++ b/.github/workflows/live-smoke.yml @@ -33,12 +33,12 @@ jobs: environment: live-smoke steps: - name: Check out the trusted default branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.repository.default_branch }} persist-credentials: false - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f0a1be5..fa5bb86 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,7 +27,7 @@ jobs: contents: read steps: - name: Check out the published release tag - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: refs/tags/${{ github.event.release.tag_name }} fetch-depth: 0 @@ -40,7 +40,7 @@ jobs: RELEASE_TAG: ${{ github.event.release.tag_name }} run: bash scripts/verify_release_trust.sh - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend @@ -73,7 +73,7 @@ jobs: - name: Record immutable artifact digests run: sha256sum dist/* > artifact-sha256.txt - name: Retain only the verified release bundle - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-${{ steps.version.outputs.version }} path: | @@ -106,7 +106,7 @@ jobs: COMETAPI_LIVE_STOP_ON_FAILURE: "1" steps: - name: Check out the verified release commit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.build.outputs.release-commit }} persist-credentials: false @@ -115,7 +115,7 @@ jobs: RELEASE_COMMIT: ${{ needs.build.outputs.release-commit }} run: test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT" - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned uv frontend @@ -142,7 +142,7 @@ jobs: id-token: write steps: - name: Download the verified release bundle - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-${{ needs.build.outputs.version }} path: release-bundle @@ -167,19 +167,19 @@ jobs: contents: read steps: - name: Check out the registry verification source - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.build.outputs.release-commit }} persist-credentials: false - name: Download the verified release bundle after checkout - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-${{ needs.build.outputs.version }} path: release-bundle - name: Require retained pre-publication digest evidence run: test -f release-bundle/artifact-sha256.txt - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" - name: Install the pinned provenance verifier diff --git a/pyproject.toml b/pyproject.toml index be4e28e..61969af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,11 @@ dev = [ "pyright>=1.1.408,<2.0.0", "pytest>=8.3.5,<10.0.0", "pytest-asyncio>=0.24.0,<2.0.0", + "pyyaml>=6.0.2,<7.0.0", "ruff>=0.12.0,<1.0.0", "tomli>=2.2.1,<3.0.0", "twine>=6.1.0,<7.0.0", + "types-pyyaml>=6.0.12,<7.0.0", ] [build-system] diff --git a/scripts/check_version.py b/scripts/check_version.py index 30db779..239171d 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -160,6 +160,11 @@ def require_public_preview_docs() -> None: documents = _read_public_documents(violations) _check_project_identity(violations) + if Path(".github/CODEOWNERS").exists(): + violations.append( + ".github/CODEOWNERS: must remain absent until a real multi-maintainer model exists" + ) + try: license_text = Path("LICENSE").read_text(encoding="utf-8") except (OSError, UnicodeError) as exc: diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index 48ad027..d607f3e 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -5,7 +5,11 @@ import argparse import re +from collections.abc import Iterator from pathlib import Path +from typing import cast + +import yaml try: from ._checks import PROJECT_ROOT, CheckError @@ -13,342 +17,1122 @@ from _checks import PROJECT_ROOT, CheckError -def _require(text: str, needle: str, message: str) -> None: - if needle not in text: - raise CheckError(message) +def _mapping(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + raise CheckError(f"{label} must be a mapping") + raw = cast(dict[object, object], value) + if not all(isinstance(key, str) for key in raw): + raise CheckError(f"{label} must use scalar string keys") + mapping = {cast(str, key): item for key, item in raw.items()} + if "<<" in mapping: + raise CheckError(f"{label} must not use YAML merge keys") + return mapping -def _require_pattern(text: str, pattern: str, message: str) -> None: - if re.search(pattern, text) is None: - raise CheckError(message) +def _sequence(value: object, label: str) -> list[object]: + if not isinstance(value, list): + raise CheckError(f"{label} must be a sequence") + return cast(list[object], value) -def _job(text: str, name: str, *, source: str = "publish workflow") -> str: - match = re.search( - rf"(?ms)^ {re.escape(name)}:\n(?P.*?)(?=^ [a-zA-Z0-9_-]+:\n|\Z)", - text, - ) - if match is None: +def _scalar(value: object, label: str) -> str: + if not isinstance(value, str): + raise CheckError(f"{label} must be a scalar string") + return value + + +def _load_workflow(text: str, source: str) -> dict[str, object]: + try: + loaded: object = yaml.load(text, Loader=yaml.BaseLoader) + except yaml.YAMLError as error: + raise CheckError(f"{source} is not valid YAML: {error}") from error + return _mapping(loaded, source) + + +def _workflow_job(workflow: dict[str, object], name: str, source: str) -> dict[str, object]: + jobs = _mapping(workflow.get("jobs"), f"{source} jobs") + if name not in jobs: raise CheckError(f"{source} has no {name!r} job") - return match.group(0) + return _mapping(jobs[name], f"{source} {name!r} job") -def _step(job: str, name: str) -> str: - match = re.search( - rf"(?ms)^ - name: {re.escape(name)}\n(?P.*?)(?=^ - name: |\Z)", - job, - ) - if match is None: - raise CheckError(f"publish workflow has no {name!r} step") - return match.group(0) +def _workflow_steps(job: dict[str, object], label: str) -> list[dict[str, object]]: + return [ + _mapping(item, f"{label} step {index}") + for index, item in enumerate(_sequence(job.get("steps"), f"{label} steps")) + ] + + +def _walk_mappings(value: object, label: str) -> Iterator[dict[str, object]]: + if isinstance(value, dict): + mapping = _mapping(cast(dict[object, object], value), label) + yield mapping + for key, child in mapping.items(): + yield from _walk_mappings(child, f"{label}.{key}") + elif isinstance(value, list): + for index, child in enumerate(cast(list[object], value)): + yield from _walk_mappings(child, f"{label}[{index}]") + + +def _walk_scalars(value: object) -> Iterator[str]: + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for child in cast(dict[object, object], value).values(): + yield from _walk_scalars(child) + elif isinstance(value, list): + for child in cast(list[object], value): + yield from _walk_scalars(child) + + +def _secret_references(value: object) -> list[str]: + pattern = re.compile(r"\$\{\{[^}]*\bsecrets\b[^}]*\}\}", flags=re.IGNORECASE) + return [scalar for scalar in _walk_scalars(value) if pattern.search(scalar)] + + +def _require_unconditional(mapping: dict[str, object], label: str) -> None: + if "if" in mapping: + raise CheckError(f"{label} must not be conditional") + if "continue-on-error" in mapping: + raise CheckError(f"{label} must not allow failure") + if "defaults" in mapping: + raise CheckError(f"{label} must not override command defaults") + if "shell" in mapping: + raise CheckError(f"{label} must not override the command shell") + + +def _require_blocking_job(job: dict[str, object], label: str) -> None: + _require_unconditional(job, label) + if "permissions" in job: + raise CheckError(f"{label} must not override credential-free workflow permissions") + if "env" in job: + raise CheckError(f"{label} must not override the reviewed CI environment") + for index, step in enumerate(_workflow_steps(job, label)): + _require_unconditional(step, f"{label} step {index}") + if "env" in step: + raise CheckError(f"{label} step {index} must not override the CI environment") + + +def _run_step(job: dict[str, object], command: str, label: str) -> tuple[int, dict[str, object]]: + matches = [ + (index, step) + for index, step in enumerate(_workflow_steps(job, label)) + if step.get("run") == command + ] + if len(matches) != 1: + raise CheckError(f"{label} must contain exactly one active run step: {command}") + index, step = matches[0] + _require_unconditional(step, f"{label} run step {command!r}") + return index, step + + +def _action_step(job: dict[str, object], action: str, label: str) -> tuple[int, dict[str, object]]: + matches: list[tuple[int, dict[str, object]]] = [] + for index, step in enumerate(_workflow_steps(job, label)): + uses = step.get("uses") + if isinstance(uses, str) and uses.rpartition("@")[0] == action: + matches.append((index, step)) + if len(matches) != 1: + raise CheckError(f"{label} must contain exactly one {action} step") + index, step = matches[0] + _require_unconditional(step, f"{label} {action} step") + return index, step + + +def _named_step(job: dict[str, object], name: str, label: str) -> tuple[int, dict[str, object]]: + matches = [ + (index, step) + for index, step in enumerate(_workflow_steps(job, label)) + if step.get("name") == name + ] + if len(matches) != 1: + raise CheckError(f"{label} must contain exactly one {name!r} step") + index, step = matches[0] + _require_unconditional(step, f"{label} {name!r} step") + return index, step + + +def _named_run_step( + job: dict[str, object], name: str, command: str, label: str +) -> tuple[int, dict[str, object]]: + index, step = _named_step(job, name, label) + if step.get("run") != command or "uses" in step: + raise CheckError(f"{label} {name!r} step must run exactly: {command}") + return index, step + + +def _named_action_step( + job: dict[str, object], name: str, action: str, label: str +) -> tuple[int, dict[str, object]]: + index, step = _action_step(job, action, label) + if step.get("name") != name or "run" in step: + raise CheckError(f"{label} must use {action} in its {name!r} step") + return index, step + + +def _require_step_names(job: dict[str, object], expected: list[str], label: str) -> None: + names = [step.get("name") for step in _workflow_steps(job, label)] + if names != expected: + raise CheckError(f"{label} steps must match the reviewed sequence") + + +def _require_needs(job: dict[str, object], expected: list[str], label: str) -> None: + value = job.get("needs") + if isinstance(value, str): + actual = [value] + else: + actual = [ + _scalar(item, f"{label} dependency") + for item in _sequence(value, f"{label} dependencies") + ] + if actual != expected: + raise CheckError(f"{label} must depend on {', '.join(expected)}") + + +def _require_permissions(mapping: dict[str, object], expected: dict[str, str], label: str) -> None: + permissions = _mapping(mapping.get("permissions"), f"{label} permissions") + if permissions != expected: + raise CheckError(f"{label} permissions do not match the reviewed least-privilege map") + + +def _require_options(step: dict[str, object], expected: dict[str, str], label: str) -> None: + options = _mapping(step.get("with"), f"{label} options") + if options != expected: + raise CheckError(f"{label} options do not match the reviewed contract") + + +def _require_step_environments( + job: dict[str, object], expected: dict[str, dict[str, str]], label: str +) -> None: + for index, step in enumerate(_workflow_steps(job, label)): + name = _scalar(step.get("name"), f"{label} step {index} name") + if name in expected: + environment = _mapping(step.get("env"), f"{label} {name!r} environment") + if environment != expected[name]: + raise CheckError(f"{label} {name!r} environment does not match the contract") + elif "env" in step: + raise CheckError(f"{label} {name!r} step must not override the environment") + + +def _action_references(workflow: dict[str, object], source: str) -> Iterator[tuple[str, str]]: + jobs = _mapping(workflow.get("jobs"), f"{source} jobs") + for job_name, value in jobs.items(): + job = _mapping(value, f"{source} {job_name!r} job") + if "uses" in job: + yield ( + f"{source} {job_name!r} job", + _scalar(job["uses"], f"{source} {job_name!r} job uses"), + ) + if "steps" not in job: + continue + for index, step in enumerate(_workflow_steps(job, f"{source} {job_name!r} job")): + if "uses" in step: + yield ( + f"{source} {job_name!r} step {index}", + _scalar(step["uses"], f"{source} {job_name!r} step {index} uses"), + ) def check_action_pins(text: str, source: str) -> None: - for match in re.finditer(r"(?m)^\s*uses:\s*([^@\s]+)@([^\s#]+)", text): - action, reference = match.groups() + workflow = _load_workflow(text, source) + for label, value in _action_references(workflow, source): + if value.startswith("./"): + continue + if value.startswith("docker://"): + raise CheckError(f"{label}: Docker action references are not permitted") + action, separator, reference = value.rpartition("@") + if not separator or not action: + raise CheckError(f"{label}: external action reference {value!r} has no ref") if re.fullmatch(r"[0-9a-f]{40}", reference) is None: - raise CheckError(f"{source}: {action} must be pinned to a full commit SHA") + raise CheckError(f"{label}: {action} must be pinned to a full commit SHA") def check_ci_workflow(text: str) -> None: """Require credential-free CI to cover every private-validation evidence layer.""" - _prefix, separator, _jobs = text.partition("\njobs:\n") - if not separator: - raise CheckError("CI workflow has no jobs mapping") - for needle, message in ( - ("pull_request:", "CI must run for pull requests"), - ("push:\n branches:\n - main", "CI must run for default-branch pushes"), - ("run: uv lock --check", "CI must verify lock consistency"), - ( - "run: uv run python scripts/check_version.py --require-public-preview-docs", - "CI must enforce canonical public content and identity", + workflow = _load_workflow(text, "CI workflow") + _require_permissions(workflow, {"contents": "read"}, "credential-free CI") + if "defaults" in workflow: + raise CheckError("credential-free CI must not override command defaults") + if _mapping(workflow.get("env"), "CI workflow environment") != {"UV_VERSION": "0.11.8"}: + raise CheckError("credential-free CI must retain only its pinned uv frontend version") + triggers = _mapping(workflow.get("on"), "CI workflow triggers") + if set(triggers) != {"pull_request", "push", "schedule"}: + raise CheckError("CI triggers must equal pull requests, main pushes, and weekly schedule") + if triggers["pull_request"] != "": + raise CheckError("CI pull-request validation must not use activity or path filters") + push = _mapping(triggers["push"], "CI push trigger") + if set(push) != {"branches"}: + raise CheckError("CI main-push validation must not use path, tag, or activity filters") + branches = [ + _scalar(item, "CI push branch") + for item in _sequence(push.get("branches"), "CI push branches") + ] + if branches != ["main"]: + raise CheckError("CI must run only for default-branch pushes") + schedule = _sequence(triggers["schedule"], "CI schedule trigger") + if schedule != [{"cron": "23 4 * * 1"}]: + raise CheckError("CI latest-OpenAI canary must run on the reviewed weekly schedule") + + quality = _workflow_job(workflow, "quality", "CI workflow") + locked_runtime = _workflow_job(workflow, "locked-runtime", "CI workflow") + minimum_openai = _workflow_job(workflow, "minimum-openai", "CI workflow") + latest_openai = _workflow_job(workflow, "latest-openai", "CI workflow") + package = _workflow_job(workflow, "package", "CI workflow") + standalone = _workflow_job(workflow, "standalone", "CI workflow") + jobs = _mapping(workflow.get("jobs"), "CI workflow jobs") + for name, value in jobs.items(): + job = _mapping(value, f"CI {name!r} job") + if "permissions" in job: + raise CheckError("CI jobs must not override credential-free workflow permissions") + for name, job in ( + ("quality", quality), + ("locked-runtime", locked_runtime), + ("minimum-openai", minimum_openai), + ("package", package), + ("standalone", standalone), + ): + _require_blocking_job(job, f"CI {name!r} job") + if latest_openai.get("if") != ( + "github.event_name == 'schedule' || github.actor == 'dependabot[bot]'" + ): + raise CheckError( + "CI latest-OpenAI canary must run only for the weekly schedule or Dependabot" + ) + if any(key in latest_openai for key in ("continue-on-error", "defaults", "env", "shell")): + raise CheckError("CI latest-OpenAI canary must retain blocking command execution") + for index, step in enumerate(_workflow_steps(latest_openai, "CI latest-openai job")): + _require_unconditional(step, f"CI latest-openai step {index}") + if "env" in step: + raise CheckError("CI latest-OpenAI steps must not override the CI environment") + + _require_needs( + package, + ["quality", "locked-runtime", "minimum-openai"], + "CI package job", + ) + _require_needs(standalone, ["package"], "CI standalone job") + + required_commands = { + "quality": ( + "uv lock --check", + "uv sync --locked", + "uv run ruff check src tests scripts", + "uv run ruff format --check src tests scripts", + "uv run pyright", + 'uv run pytest -m "not live"', + "uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog", + "uv run python scripts/check_version.py --require-public-preview-docs", + "uv run python scripts/check_secrets.py", + "uv run python scripts/run_actionlint.py", + "uv run python scripts/check_workflows.py", ), - ( - "run: python scripts/check_repository_independence.py", - "CI must run standalone copied-checkout verification", + "locked-runtime": ("uv sync --locked", 'uv run pytest -m "not live"'), + "minimum-openai": ( + "uv sync --locked", + 'uv pip install --python .venv/bin/python "openai==2.45.0"', + 'uv run --no-sync pytest -m "not live"', ), - ( - 'python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]', - "CI must block on every supported Python runtime", + "latest-openai": ( + "uv sync --locked", + 'uv pip install --python .venv/bin/python --upgrade "openai>=2.45.0,<3.0.0"', + 'uv run --no-sync pytest -m "not live"', ), - ( - 'run: uv pip install --python .venv/bin/python "openai==2.45.0"', - "CI must cover the minimum supported OpenAI version", + "package": ( + "uv sync --locked", + "uv build", + "uv run twine check dist/*", + "uv run python scripts/check_artifacts.py dist/*", + "uv run python scripts/check_clean_install.py dist/*", + "sha256sum dist/* > artifact-sha256.txt", ), - ): - _require(text, needle, message) - if re.search(r"\$\{\{\s*secrets\.", text, flags=re.IGNORECASE): + "standalone": ( + "python scripts/check_repository_independence.py", + "sha256sum --check artifact-sha256.txt", + ), + } + required_jobs = { + "quality": quality, + "locked-runtime": locked_runtime, + "minimum-openai": minimum_openai, + "latest-openai": latest_openai, + "package": package, + "standalone": standalone, + } + expected_timeouts = { + "quality": "20", + "locked-runtime": "20", + "minimum-openai": "20", + "latest-openai": "20", + "package": "25", + "standalone": "35", + } + expected_python = { + "quality": "3.14", + "locked-runtime": "${{ matrix.python-version }}", + "minimum-openai": "3.10", + "latest-openai": "3.14", + "package": "3.14", + "standalone": "3.14", + } + for name, job in required_jobs.items(): + if job.get("runs-on") != "ubuntu-latest": + raise CheckError(f"CI {name} job must use the reviewed GitHub-hosted runner") + if job.get("timeout-minutes") != expected_timeouts[name]: + raise CheckError(f"CI {name} job must retain its reviewed timeout") + _, setup_step = _named_action_step( + job, "Set up Python", "actions/setup-python", f"CI {name} job" + ) + _require_options( + setup_step, + {"python-version": expected_python[name]}, + f"CI {name} Python setup", + ) + for name, commands in required_commands.items(): + for command in commands: + _run_step(required_jobs[name], command, f"CI {name} job") + + strategy = _mapping(locked_runtime.get("strategy"), "CI locked-runtime strategy") + if set(strategy) != {"fail-fast", "matrix"} or strategy["fail-fast"] != "false": + raise CheckError("CI runtime matrix must retain fail-fast: false") + matrix = _mapping(strategy.get("matrix"), "CI locked-runtime matrix") + if set(matrix) != {"python-version"}: + raise CheckError("CI runtime matrix must vary only the supported Python version") + python_versions = [ + _scalar(item, "CI locked-runtime Python version") + for item in _sequence(matrix.get("python-version"), "CI locked-runtime Python versions") + ] + if python_versions != ["3.10", "3.11", "3.12", "3.13", "3.14"]: + raise CheckError("CI must block on every supported Python runtime") + package_digest, _ = _run_step( + package, "sha256sum dist/* > artifact-sha256.txt", "CI package job" + ) + package_upload, upload_step = _named_action_step( + package, "Retain verified artifacts", "actions/upload-artifact", "CI package job" + ) + _require_options( + upload_step, + { + "name": "python-distributions", + "path": "dist/*\nartifact-sha256.txt\n", + "if-no-files-found": "error", + "retention-days": "7", + }, + "CI package artifact upload", + ) + if package_digest >= package_upload: + raise CheckError("CI package job must digest artifacts before retaining them") + copied_checkout, _ = _run_step( + standalone, + "python scripts/check_repository_independence.py", + "CI standalone job", + ) + artifact_download, download_step = _action_step( + standalone, "actions/download-artifact", "CI standalone job" + ) + _require_options( + download_step, + {"name": "python-distributions", "path": "verified-artifacts"}, + "CI artifact download", + ) + digest_check, digest_step = _run_step( + standalone, + "sha256sum --check artifact-sha256.txt", + "CI standalone job", + ) + if digest_step.get("working-directory") != "verified-artifacts": + raise CheckError("CI must recheck retained artifact digests after download") + if not copied_checkout < artifact_download < digest_check: + raise CheckError( + "CI must finish copied-checkout verification before downloading and " + "rechecking retained artifacts" + ) + if _secret_references(workflow): raise CheckError("credential-free CI must not reference repository secrets") def check_release_please_workflow(text: str) -> None: """Require Release Please to remain explicitly disabled by default.""" - prefix, separator, jobs = text.partition("\njobs:\n") - if not separator: - raise CheckError("release-please workflow has no jobs mapping") - _require( - prefix, - "on:\n push:\n branches:\n - main", - "Release Please must run only for default-branch pushes", - ) - if "workflow_dispatch:" in prefix or "release:" in prefix: - raise CheckError("Release Please must not accept manual or release events") - _require( - jobs, - "if: vars.RELEASE_PLEASE_ENABLED == 'true'", - "Release Please must require RELEASE_PLEASE_ENABLED=true", - ) - if "secrets." in text: + workflow = _load_workflow(text, "Release Please workflow") + _require_permissions(workflow, {"contents": "read"}, "Release Please workflow") + if "env" in workflow: + raise CheckError("Release Please workflow must not override the action environment") + triggers = _mapping(workflow.get("on"), "Release Please triggers") + if set(triggers) != {"push"}: + raise CheckError("Release Please must run only for default-branch pushes") + push = _mapping(triggers["push"], "Release Please push trigger") + if set(push) != {"branches"}: + raise CheckError("Release Please push trigger must not use path or tag filters") + branches = [ + _scalar(item, "Release Please push branch") + for item in _sequence(push.get("branches"), "Release Please push branches") + ] + if branches != ["main"]: + raise CheckError("Release Please must run only for default-branch pushes") + concurrency = _mapping(workflow.get("concurrency"), "Release Please concurrency") + if concurrency != { + "group": "release-please-${{ github.ref }}", + "cancel-in-progress": "false", + }: + raise CheckError("Release Please must serialize updates per ref without cancellation") + jobs = _mapping(workflow.get("jobs"), "Release Please jobs") + if set(jobs) != {"release-please"}: + raise CheckError("Release Please must contain only its gated release-please job") + release_job = _workflow_job(workflow, "release-please", "Release Please workflow") + if release_job.get("if") != "vars.RELEASE_PLEASE_ENABLED == 'true'": + raise CheckError("Release Please must require RELEASE_PLEASE_ENABLED=true") + if release_job.get("runs-on") != "ubuntu-latest": + raise CheckError("Release Please must use the reviewed GitHub-hosted runner") + if release_job.get("timeout-minutes") != "10": + raise CheckError("Release Please must retain its ten-minute timeout") + if "continue-on-error" in release_job: + raise CheckError("Release Please must not allow its job to fail") + if "env" in release_job: + raise CheckError("Release Please job must not override the action environment") + _require_permissions( + release_job, + {"contents": "write", "pull-requests": "write"}, + "Release Please job", + ) + if "defaults" in workflow or "defaults" in release_job: + raise CheckError("Release Please must not override command defaults") + for index, step in enumerate(_workflow_steps(release_job, "Release Please job")): + _require_unconditional(step, f"Release Please step {index}") + _require_step_names( + release_job, + ["Open or update the release PR, or create its approved release"], + "Release Please job", + ) + _require_step_environments(release_job, {}, "Release Please job") + _, release_step = _named_action_step( + release_job, + "Open or update the release PR, or create its approved release", + "googleapis/release-please-action", + "Release Please job", + ) + _require_options( + release_step, + { + "config-file": "release-please-config.json", + "manifest-file": ".release-please-manifest.json", + }, + "Release Please action", + ) + if _secret_references(workflow): raise CheckError("Release Please must not depend on repository credentials") def check_publish_workflow(text: str, live_smoke_text: str) -> None: """Validate fail-closed publication, live, permission, and evidence ordering.""" - prefix, separator, _jobs = text.partition("\njobs:\n") - if not separator: - raise CheckError("publish workflow has no jobs mapping") - _require( - prefix, - "on:\n release:\n types:\n - published", - "publication must be triggered only by a published GitHub release", - ) - if "workflow_dispatch:" in prefix or "push:" in prefix: - raise CheckError("production publication must not accept manual, push, or arbitrary refs") - _require_pattern( - prefix, - r"(?m)^permissions:\n contents: read$", - "workflow permissions must default to contents: read", - ) - _require_pattern( - prefix, - r"(?m)^concurrency:\n group: pypi-publish\n cancel-in-progress: false$", - "publication must serialize all releases without cancellation", - ) - if re.search(r"(?m)^\s+[\"']?(?:if|continue-on-error)[\"']?\s*:", text): - raise CheckError("release gates must not be conditional or allowed to continue on error") - shell_text = re.sub(r"\$\{\{[^}]*\}\}", "", text) - if "||" in shell_text or re.search(r"(?m)^\s*set\s+\+e(?:\s|$)", shell_text): - raise CheckError("release gate commands must not swallow shell failures") - if re.search(r"(?m)^\s*[\"']?permissions[\"']?\s*:\s*(?:write-all|read-all)\s*$", text): - raise CheckError("release workflow permissions must use explicit read-only job maps") - write_permissions = re.findall(r"(?m)^\s+[\"']?([a-z-]+)[\"']?\s*:\s*write\s*$", text) - if write_permissions != ["id-token"]: - raise CheckError("id-token: write on the publish job must be the only write permission") - - monitoring_live = _job(live_smoke_text, "smoke", source="live-smoke workflow") - _require_pattern( - live_smoke_text, - r"(?m)^concurrency:\n group: trusted-live-smoke\n cancel-in-progress: false$", - "release and monitoring live smokes must share one non-cancelling concurrency group", - ) - _require_pattern( - monitoring_live, - r"(?m)^ if: >-\n" - r" github\.ref == format\('refs/heads/\{0\}', " - r"github\.event\.repository\.default_branch\) &&\n" - r" vars\.LIVE_SMOKE_ENABLED == 'true'\n" - r" runs-on:", - "monitoring live smoke must run only against the canonical default branch and " - "require LIVE_SMOKE_ENABLED=true for every trigger", - ) - - build = _job(text, "build") - _require_pattern( + workflow = _load_workflow(text, "publish workflow") + live_workflow = _load_workflow(live_smoke_text, "live-smoke workflow") + + publish_triggers = _mapping(workflow.get("on"), "publish workflow triggers") + if set(publish_triggers) != {"release"}: + raise CheckError("publication must be triggered only by a published GitHub release") + release_trigger = _mapping(publish_triggers["release"], "publish release trigger") + if set(release_trigger) != {"types"}: + raise CheckError("publication release trigger must not use additional filters") + release_types = [ + _scalar(item, "publish release type") + for item in _sequence(release_trigger.get("types"), "publish release types") + ] + if release_types != ["published"]: + raise CheckError("publication must be triggered only by a published GitHub release") + _require_permissions(workflow, {"contents": "read"}, "publish workflow") + concurrency = _mapping(workflow.get("concurrency"), "publish workflow concurrency") + if concurrency != {"group": "pypi-publish", "cancel-in-progress": "false"}: + raise CheckError("publication must serialize all releases without cancellation") + if _mapping(workflow.get("env"), "publish workflow environment") != {"UV_VERSION": "0.11.8"}: + raise CheckError("publish workflow must retain its pinned uv frontend version") + + live_triggers = _mapping(live_workflow.get("on"), "live-smoke triggers") + if set(live_triggers) != {"schedule", "workflow_dispatch"}: + raise CheckError("monitoring live smoke must run only on schedule or manual dispatch") + if live_triggers["workflow_dispatch"] != "": + raise CheckError("monitoring live smoke manual dispatch must not accept inputs") + if _sequence(live_triggers["schedule"], "live-smoke schedule") != [{"cron": "17 3 * * *"}]: + raise CheckError("monitoring live smoke must retain its reviewed daily schedule") + _require_permissions(live_workflow, {"contents": "read"}, "live-smoke workflow") + if "defaults" in live_workflow: + raise CheckError("monitoring live smoke must not override command defaults") + live_concurrency = _mapping(live_workflow.get("concurrency"), "live-smoke workflow concurrency") + if live_concurrency != {"group": "trusted-live-smoke", "cancel-in-progress": "false"}: + raise CheckError( + "release and monitoring live smokes must share one non-cancelling concurrency group" + ) + live_environment = _mapping(live_workflow.get("env"), "live-smoke workflow environment") + if live_environment != { + "UV_VERSION": "0.11.8", + "COMETAPI_LIVE_MAX_REQUESTS": "4", + "COMETAPI_LIVE_MAX_OUTPUT_TOKENS": "16", + "COMETAPI_LIVE_MODEL": "gpt-5.4", + "COMETAPI_LIVE_REQUEST_TIMEOUT_SECONDS": "30", + "COMETAPI_LIVE_CONCURRENCY": "1", + "COMETAPI_LIVE_RUN": "1", + "COMETAPI_LIVE_STOP_ON_FAILURE": "1", + }: + raise CheckError("monitoring live smoke must retain its bounded execution budget") + + for mapping in _walk_mappings(workflow, "publish workflow"): + if "if" in mapping: + raise CheckError("release gates must not be conditional") + if "continue-on-error" in mapping: + raise CheckError("release gates must not be allowed to continue on error") + if "defaults" in mapping or "shell" in mapping: + raise CheckError("release gates must not override command execution") + + monitoring_job = _workflow_job(live_workflow, "smoke", "live-smoke workflow") + monitoring_condition = " ".join( + _scalar(monitoring_job.get("if"), "monitoring live-smoke condition").split() + ) + expected_monitoring_condition = ( + "github.ref == format('refs/heads/{0}', " + "github.event.repository.default_branch) && " + "vars.LIVE_SMOKE_ENABLED == 'true'" + ) + if monitoring_condition != expected_monitoring_condition: + raise CheckError( + "monitoring live smoke must run only against the canonical default branch and " + "require LIVE_SMOKE_ENABLED=true for every trigger" + ) + live_jobs = _mapping(live_workflow.get("jobs"), "live-smoke workflow jobs") + if set(live_jobs) != {"smoke"}: + raise CheckError("monitoring live smoke must contain only its bounded smoke job") + if "continue-on-error" in monitoring_job: + raise CheckError("monitoring live smoke must not allow its job to fail") + if any(key in monitoring_job for key in ("permissions", "defaults", "env", "shell")): + raise CheckError("monitoring live smoke must retain workflow-level execution controls") + for index, step in enumerate(_workflow_steps(monitoring_job, "monitoring live-smoke job")): + _require_unconditional(step, f"monitoring live-smoke step {index}") + if monitoring_job.get("timeout-minutes") != "10": + raise CheckError("monitoring live smoke must have a ten-minute timeout") + if monitoring_job.get("runs-on") != "ubuntu-latest": + raise CheckError("monitoring live smoke must use the reviewed GitHub-hosted runner") + if monitoring_job.get("environment") != "live-smoke": + raise CheckError("monitoring live smoke must use its protected environment") + _require_step_names( + monitoring_job, + [ + "Check out the trusted default branch", + "Set up Python", + "Install the pinned uv frontend", + "Reproduce the locked environment", + "Run the separately marked, bounded live suite", + ], + "monitoring live-smoke job", + ) + _require_step_environments( + monitoring_job, + { + "Run the separately marked, bounded live suite": { + "COMETAPI_KEY": "${{ secrets.COMETAPI_KEY }}" + } + }, + "monitoring live-smoke job", + ) + _, monitoring_checkout = _named_action_step( + monitoring_job, + "Check out the trusted default branch", + "actions/checkout", + "monitoring live-smoke job", + ) + _require_options( + monitoring_checkout, + { + "ref": "${{ github.event.repository.default_branch }}", + "persist-credentials": "false", + }, + "monitoring live-smoke checkout", + ) + _, monitoring_setup = _named_action_step( + monitoring_job, "Set up Python", "actions/setup-python", "monitoring live-smoke job" + ) + _require_options( + monitoring_setup, {"python-version": "3.14"}, "monitoring live-smoke Python setup" + ) + _named_run_step( + monitoring_job, + "Install the pinned uv frontend", + 'python -m pip install --disable-pip-version-check "uv==$UV_VERSION"', + "monitoring live-smoke job", + ) + _named_run_step( + monitoring_job, + "Reproduce the locked environment", + "uv sync --locked", + "monitoring live-smoke job", + ) + _, monitoring_test = _named_run_step( + monitoring_job, + "Run the separately marked, bounded live suite", + "uv run pytest -m live --maxfail=1 -q", + "monitoring live-smoke job", + ) + monitoring_test_environment = _mapping( + monitoring_test.get("env"), "monitoring live-smoke test environment" + ) + if monitoring_test_environment != {"COMETAPI_KEY": "${{ secrets.COMETAPI_KEY }}"}: + raise CheckError("monitoring live credentials must be scoped only to the bounded test step") + if _secret_references(live_workflow) != ["${{ secrets.COMETAPI_KEY }}"]: + raise CheckError("monitoring live smoke must use only its scoped COMETAPI_KEY") + + jobs = _mapping(workflow.get("jobs"), "publish workflow jobs") + if set(jobs) != {"build", "release-live-smoke", "publish", "verify-registry"}: + raise CheckError("publish workflow jobs must match the reviewed release chain") + build = _workflow_job(workflow, "build", "publish workflow") + release_live = _workflow_job(workflow, "release-live-smoke", "publish workflow") + publish = _workflow_job(workflow, "publish", "publish workflow") + registry = _workflow_job(workflow, "verify-registry", "publish workflow") + for name, job, timeout in ( + ("build", build, "25"), + ("release-live-smoke", release_live, "10"), + ("publish", publish, "10"), + ("verify-registry", registry, "10"), + ): + if job.get("runs-on") != "ubuntu-latest": + raise CheckError(f"release {name} job must use the reviewed GitHub-hosted runner") + if job.get("timeout-minutes") != timeout: + raise CheckError(f"release {name} job must retain its reviewed timeout") + for name, job in (("build", build), ("publish", publish), ("verify-registry", registry)): + if "env" in job: + raise CheckError(f"release {name} job must not override the workflow environment") + + _require_permissions(build, {"contents": "read"}, "release build job") + outputs = _mapping(build.get("outputs"), "release build outputs") + if outputs != { + "release-commit": "${{ steps.trust.outputs.release-commit }}", + "version": "${{ steps.version.outputs.version }}", + }: + raise CheckError("release build must expose only its verified commit and version") + _require_step_names( build, - r"(?m)^ permissions:\n contents: read$", - "verified build permissions must be explicitly read-only", + [ + "Check out the published release tag", + "Reject an untrusted release target", + "Set up Python", + "Install the pinned uv frontend", + "Reproduce the locked environment", + "Verify project, manifest, changelog, release docs, and tag agreement", + "Scan the immutable source for credentials and scope mistakes", + "Verify release-workflow trust semantics", + "Build wheel and source distribution from the tag", + "Verify artifact versions against the tag", + "Check package metadata rendering", + "Inspect artifact identity and shape", + "Install and smoke-test each exact artifact", + "Record immutable artifact digests", + "Retain only the verified release bundle", + ], + "release build job", ) - for needle, message in ( - ( - "release-commit: ${{ steps.trust.outputs.release-commit }}", - "build must expose the verified release commit", - ), - ( - "ref: refs/tags/${{ github.event.release.tag_name }}", - "build must check out the published tag ref", - ), - ("fetch-depth: 0", "build must fetch history for ancestry validation"), - ( - "DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", - "trust validation must use the protected default branch", - ), - ( - "RELEASE_IMMUTABLE: ${{ github.event.release.immutable }}", - "trust validation must receive release.immutable", + _require_step_environments( + build, + { + "Reject an untrusted release target": { + "DEFAULT_BRANCH": "${{ github.event.repository.default_branch }}", + "RELEASE_IMMUTABLE": "${{ github.event.release.immutable }}", + "RELEASE_TAG": "${{ github.event.release.tag_name }}", + }, + "Verify project, manifest, changelog, release docs, and tag agreement": { + "RELEASE_TAG": "${{ github.event.release.tag_name }}" + }, + "Verify artifact versions against the tag": { + "RELEASE_TAG": "${{ github.event.release.tag_name }}" + }, + }, + "release build job", + ) + _, build_checkout = _named_action_step( + build, + "Check out the published release tag", + "actions/checkout", + "release build job", + ) + _require_options( + build_checkout, + { + "ref": "refs/tags/${{ github.event.release.tag_name }}", + "fetch-depth": "0", + "persist-credentials": "false", + }, + "release build checkout", + ) + _, trust_step = _named_run_step( + build, + "Reject an untrusted release target", + "bash scripts/verify_release_trust.sh", + "release build job", + ) + if trust_step.get("id") != "trust": + raise CheckError("release trust step must expose the trust output id") + trust_environment = _mapping(trust_step.get("env"), "release trust environment") + if trust_environment != { + "DEFAULT_BRANCH": "${{ github.event.repository.default_branch }}", + "RELEASE_IMMUTABLE": "${{ github.event.release.immutable }}", + "RELEASE_TAG": "${{ github.event.release.tag_name }}", + }: + raise CheckError("release trust step must receive only the immutable release identity") + _, build_setup = _named_action_step( + build, "Set up Python", "actions/setup-python", "release build job" + ) + _require_options(build_setup, {"python-version": "3.14"}, "release build Python setup") + build_commands = { + "Install the pinned uv frontend": ( + 'python -m pip install --disable-pip-version-check "uv==$UV_VERSION"' ), - ( - "RELEASE_TAG: ${{ github.event.release.tag_name }}", - "trust validation must receive the release tag", + "Reproduce the locked environment": "uv sync --locked", + "Scan the immutable source for credentials and scope mistakes": ( + "uv run python scripts/check_secrets.py" ), - ( - "run: bash scripts/verify_release_trust.sh", - "build must execute the tested release-trust verifier", + "Verify release-workflow trust semantics": "uv run python scripts/check_workflows.py", + "Build wheel and source distribution from the tag": "uv build", + "Verify artifact versions against the tag": ( + 'uv run python scripts/check_version.py --tag "$RELEASE_TAG" ' + "--require-changelog --require-releasable-docs dist/*" ), - ( - "run: uv run python scripts/check_workflows.py", - "release build must run the local workflow semantic check", + "Check package metadata rendering": "uv run twine check dist/*", + "Inspect artifact identity and shape": "uv run python scripts/check_artifacts.py dist/*", + "Install and smoke-test each exact artifact": ( + "uv run python scripts/check_clean_install.py dist/*" ), - ): - _require(build, needle, message) - if "id-token: write" in build or "secrets.COMETAPI_KEY" in build: + "Record immutable artifact digests": "sha256sum dist/* > artifact-sha256.txt", + } + for name, command in build_commands.items(): + _named_run_step(build, name, command, "release build job") + _, version_step = _named_run_step( + build, + "Verify project, manifest, changelog, release docs, and tag agreement", + 'version=$(uv run python scripts/check_version.py --tag "$RELEASE_TAG" ' + "--require-changelog --require-releasable-docs --print-version)\n" + 'echo "version=$version" >> "$GITHUB_OUTPUT"\n', + "release build job", + ) + if version_step.get("id") != "version" or _mapping( + version_step.get("env"), "release version environment" + ) != {"RELEASE_TAG": "${{ github.event.release.tag_name }}"}: + raise CheckError("release version step must expose the verified tag-derived version") + _, artifact_upload = _named_action_step( + build, + "Retain only the verified release bundle", + "actions/upload-artifact", + "release build job", + ) + _require_options( + artifact_upload, + { + "name": "release-${{ steps.version.outputs.version }}", + "path": "dist/*\nartifact-sha256.txt\n", + "if-no-files-found": "error", + "retention-days": "30", + }, + "release bundle upload", + ) + + _require_permissions(release_live, {"contents": "read"}, "exact-release live-smoke job") + _require_needs(release_live, ["build"], "exact-release live-smoke job") + if release_live.get("timeout-minutes") != "10": + raise CheckError("exact-release live smoke must have a ten-minute timeout") + if release_live.get("environment") != "live-smoke": + raise CheckError("exact-release live smoke must use its protected environment") + release_live_concurrency = _mapping( + release_live.get("concurrency"), "exact-release live-smoke concurrency" + ) + if release_live_concurrency != { + "group": "trusted-live-smoke", + "cancel-in-progress": "false", + }: raise CheckError( - "build must receive neither OIDC publication permission nor live credentials" + "release and monitoring live smokes must share one non-cancelling concurrency group" ) + release_live_environment = _mapping( + release_live.get("env"), "exact-release live-smoke environment" + ) + if release_live_environment != { + "COMETAPI_LIVE_CONCURRENCY": "1", + "COMETAPI_LIVE_MAX_OUTPUT_TOKENS": "16", + "COMETAPI_LIVE_MAX_REQUESTS": "4", + "COMETAPI_LIVE_MODEL": "${{ vars.COMETAPI_LIVE_MODEL || 'gpt-5.4' }}", + "COMETAPI_LIVE_REQUEST_TIMEOUT_SECONDS": "30", + "COMETAPI_LIVE_RUN": "1", + "COMETAPI_LIVE_STOP_ON_FAILURE": "1", + }: + raise CheckError("exact-release live smoke must retain its bounded execution budget") + _require_step_names( + release_live, + [ + "Check out the verified release commit", + "Require the exact verified release commit", + "Set up Python", + "Install the pinned uv frontend", + "Reproduce the locked release environment", + "Run the bounded exact-release live suite", + ], + "exact-release live-smoke job", + ) + _require_step_environments( + release_live, + { + "Require the exact verified release commit": { + "RELEASE_COMMIT": "${{ needs.build.outputs.release-commit }}" + }, + "Run the bounded exact-release live suite": { + "COMETAPI_KEY": "${{ secrets.COMETAPI_KEY }}" + }, + }, + "exact-release live-smoke job", + ) + _, release_checkout = _named_action_step( + release_live, + "Check out the verified release commit", + "actions/checkout", + "exact-release live-smoke job", + ) + _require_options( + release_checkout, + { + "ref": "${{ needs.build.outputs.release-commit }}", + "persist-credentials": "false", + }, + "exact-release live-smoke checkout", + ) + _, commit_check = _named_run_step( + release_live, + "Require the exact verified release commit", + 'test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT"', + "exact-release live-smoke job", + ) + if _mapping(commit_check.get("env"), "exact-release commit-check environment") != { + "RELEASE_COMMIT": "${{ needs.build.outputs.release-commit }}" + }: + raise CheckError("exact-release commit check must use only the verified build output") + _, release_setup = _named_action_step( + release_live, "Set up Python", "actions/setup-python", "exact-release live-smoke job" + ) + _require_options( + release_setup, {"python-version": "3.14"}, "exact-release live-smoke Python setup" + ) + _named_run_step( + release_live, + "Install the pinned uv frontend", + 'python -m pip install --disable-pip-version-check "uv==$UV_VERSION"', + "exact-release live-smoke job", + ) + _named_run_step( + release_live, + "Reproduce the locked release environment", + "uv sync --locked", + "exact-release live-smoke job", + ) + _, live_test = _named_run_step( + release_live, + "Run the bounded exact-release live suite", + "uv run pytest -m live --maxfail=1 -q", + "exact-release live-smoke job", + ) + if _mapping(live_test.get("env"), "exact-release live test environment") != { + "COMETAPI_KEY": "${{ secrets.COMETAPI_KEY }}" + }: + raise CheckError("exact-release live credential must be scoped only to its test step") - live = _job(text, "release-live-smoke") - _require_pattern( - live, - r"(?m)^ permissions:\n contents: read$", - "release live-smoke permissions must be explicitly read-only", - ) - for needle, message in ( - ("needs:\n - build", "exact-release live smoke must depend on verified build"), - ( - "concurrency:\n group: trusted-live-smoke\n cancel-in-progress: false", - "release and monitoring live smokes must share one non-cancelling concurrency group", - ), - ( - "ref: ${{ needs.build.outputs.release-commit }}", - "release live smoke must check out the verified release commit", - ), - ( - 'run: test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT"', - "release live smoke must recheck its exact commit", - ), - ('COMETAPI_LIVE_MAX_REQUESTS: "4"', "release live smoke must cap requests at four"), - ( - 'COMETAPI_LIVE_MAX_OUTPUT_TOKENS: "16"', - "release live smoke must cap output tokens at 16", - ), - ( - 'COMETAPI_LIVE_REQUEST_TIMEOUT_SECONDS: "30"', - "release live smoke must cap request timeout at 30 seconds", - ), - ('COMETAPI_LIVE_CONCURRENCY: "1"', "release live smoke must use concurrency one"), - ('COMETAPI_LIVE_STOP_ON_FAILURE: "1"', "release live smoke must stop on failure"), - ("timeout-minutes: 10", "release live smoke must have a ten-minute job timeout"), - ( - "COMETAPI_LIVE_MODEL: ${{ vars.COMETAPI_LIVE_MODEL || 'gpt-5.4' }}", - "release live smoke must default an unset or empty model to gpt-5.4", - ), - ( - "COMETAPI_KEY: ${{ secrets.COMETAPI_KEY }}", - "release live smoke must receive the protected credential only at its test step", - ), - ): - _require(live, needle, message) - _require_pattern( - live, - r"(?m)^ environment: live-smoke$", - "release live smoke must use its protected environment", - ) - if "id-token: write" in live: - raise CheckError("release live smoke must not receive OIDC publication permission") - live_test = _step(live, "Run the bounded exact-release live suite") - _require( - live_test, - "env:\n COMETAPI_KEY: ${{ secrets.COMETAPI_KEY }}\n" - " run: uv run pytest -m live --maxfail=1 -q", - "release live credentials must be scoped only to the bounded test step", - ) - - publish = _job(text, "publish") - for needle, message in ( - ( - "needs:\n - build\n - release-live-smoke", - "publish must require both verified artifacts and successful exact-release live smoke", - ), - ("id-token: write", "only the publish job must receive PyPI OIDC permission"), - ): - _require(publish, needle, message) - _require_pattern( + _require_permissions(publish, {"contents": "read", "id-token": "write"}, "PyPI publish job") + _require_needs( publish, - r"(?m)^ environment:\n name: pypi\n" - r" url: https://pypi\.org/project/cometapi/" - r"\$\{\{ needs\.build\.outputs\.version \}\}/$", - "publish must use the protected pypi environment and exact package URL", + ["build", "release-live-smoke"], + "PyPI publish job", ) - _require_pattern( + publish_environment = _mapping(publish.get("environment"), "PyPI publish environment") + if publish_environment != { + "name": "pypi", + "url": "https://pypi.org/project/cometapi/${{ needs.build.outputs.version }}/", + }: + raise CheckError("publish must use the protected pypi environment and exact package URL") + _require_step_names( publish, - r"(?m)^ permissions:\n contents: read\n id-token: write$", - "publish must receive only read access plus PyPI OIDC permission", + [ + "Download the verified release bundle", + "Recheck immutable artifact digests", + "Publish through the configured PyPI Trusted Publisher", + ], + "PyPI publish job", ) - if "secrets.COMETAPI_KEY" in publish: - raise CheckError("publish must not receive the live API credential") - if "actions/checkout@" in publish: - raise CheckError("publish must consume the verified bundle without a source checkout") - - registry = _job(text, "verify-registry") - _require_pattern( - registry, - r"(?m)^ permissions:\n contents: read$", - "registry verification permissions must be explicitly read-only", + _require_step_environments(publish, {}, "PyPI publish job") + _, publish_download = _named_action_step( + publish, + "Download the verified release bundle", + "actions/download-artifact", + "PyPI publish job", ) - _require( - registry, - "needs:\n - build\n - publish", - "registry verification must require the verified build and completed publication", - ) - checkout = registry.find("- name: Check out the registry verification source") - download = registry.find("- name: Download the verified release bundle after checkout") - digest_guard = registry.find("run: test -f release-bundle/artifact-sha256.txt") - verification = registry.find("python scripts/check_registry_release.py") - if min(checkout, download, digest_guard, verification) < 0 or not ( - checkout < download < digest_guard < verification + _require_options( + publish_download, + { + "name": "release-${{ needs.build.outputs.version }}", + "path": "release-bundle", + }, + "PyPI release-bundle download", + ) + _, publish_digest = _named_run_step( + publish, + "Recheck immutable artifact digests", + "sha256sum --check artifact-sha256.txt", + "PyPI publish job", + ) + if publish_digest.get("working-directory") != "release-bundle": + raise CheckError("PyPI publish job must recheck digests inside the retained bundle") + _, pypi_publish = _named_action_step( + publish, + "Publish through the configured PyPI Trusted Publisher", + "pypa/gh-action-pypi-publish", + "PyPI publish job", + ) + _require_options( + pypi_publish, + { + "packages-dir": "release-bundle/dist/", + "print-hash": "true", + "attestations": "true", + }, + "PyPI Trusted Publisher action", + ) + + _require_permissions(registry, {"contents": "read"}, "registry verification job") + _require_needs(registry, ["build", "publish"], "registry verification job") + registry_step_names = [ + step.get("name") for step in _workflow_steps(registry, "registry verification job") + ] + checkout_name = "Check out the registry verification source" + download_name = "Download the verified release bundle after checkout" + if ( + checkout_name in registry_step_names + and download_name in registry_step_names + and registry_step_names.index(checkout_name) > registry_step_names.index(download_name) ): raise CheckError( - "registry verification must check out source before downloading and retain " - "digest evidence" + "registry verification must check out source before downloading the release bundle" ) - for needle, message in ( - ( - "ref: ${{ needs.build.outputs.release-commit }}", - "registry verification must use the verified release commit", - ), - ( - "--digest-file release-bundle/artifact-sha256.txt", - "registry verification must compare against the pre-publication digest manifest", - ), - ): - _require(registry, needle, message) - verifier_install = _step(registry, "Install the pinned provenance verifier") - for needle in ( - 'PIP_BUILD_CONSTRAINT: ""', - "PIP_CONFIG_FILE: /dev/null", - 'PIP_CONSTRAINT: ""', - 'PIP_EXTRA_INDEX_URL: ""', - 'PIP_FIND_LINKS: ""', - 'PIP_REQUIREMENT: ""', - "python -m pip --isolated install", - "--index-url https://pypi.org/simple/", - "--no-cache-dir", + _require_step_names( + registry, + [ + checkout_name, + download_name, + "Require retained pre-publication digest evidence", + "Set up Python", + "Install the pinned provenance verifier", + "Verify public artifact identity, digests, and provenance", + "Install from public PyPI and run the isolated mocked-call smoke", + ], + "registry verification job", + ) + _require_step_environments( + registry, + { + "Install the pinned provenance verifier": { + "PIP_BUILD_CONSTRAINT": "", + "PIP_CONFIG_FILE": "/dev/null", + "PIP_CONSTRAINT": "", + "PIP_EXTRA_INDEX_URL": "", + "PIP_FIND_LINKS": "", + "PIP_REQUIREMENT": "", + }, + "Verify public artifact identity, digests, and provenance": { + "RELEASE_VERSION": "${{ needs.build.outputs.version }}" + }, + "Install from public PyPI and run the isolated mocked-call smoke": { + "RELEASE_VERSION": "${{ needs.build.outputs.version }}" + }, + }, + "registry verification job", + ) + _, registry_checkout = _named_action_step( + registry, + "Check out the registry verification source", + "actions/checkout", + "registry verification job", + ) + _require_options( + registry_checkout, + { + "ref": "${{ needs.build.outputs.release-commit }}", + "persist-credentials": "false", + }, + "registry verification checkout", + ) + _, registry_download = _named_action_step( + registry, + "Download the verified release bundle after checkout", + "actions/download-artifact", + "registry verification job", + ) + _require_options( + registry_download, + { + "name": "release-${{ needs.build.outputs.version }}", + "path": "release-bundle", + }, + "registry release-bundle download", + ) + _named_run_step( + registry, + "Require retained pre-publication digest evidence", + "test -f release-bundle/artifact-sha256.txt", + "registry verification job", + ) + _, registry_setup = _named_action_step( + registry, "Set up Python", "actions/setup-python", "registry verification job" + ) + _require_options( + registry_setup, {"python-version": "3.14"}, "registry verification Python setup" + ) + _, verifier_install = _named_run_step( + registry, + "Install the pinned provenance verifier", + "python -m pip --isolated install --disable-pip-version-check " + "--index-url https://pypi.org/simple/ --no-cache-dir " '"pypi-attestations==0.0.29"', - ): - _require( - verifier_install, - needle, - "provenance-verifier bootstrap must retain its pinned isolated public-index setup", - ) - public_install = _step( - registry, "Install from public PyPI and run the isolated mocked-call smoke" + "registry verification job", ) - for needle in ( - "python scripts/check_clean_install.py", - '--requirement "cometapi==$RELEASE_VERSION"', - "--index-url https://pypi.org/simple/", - ): - _require( - public_install, - needle, - "registry clean-install smoke must install the exact version from public PyPI", - ) - if "id-token: write" in registry or "secrets.COMETAPI_KEY" in registry: - raise CheckError("registry verification must receive neither OIDC nor live credentials") + if _mapping(verifier_install.get("env"), "provenance verifier environment") != { + "PIP_BUILD_CONSTRAINT": "", + "PIP_CONFIG_FILE": "/dev/null", + "PIP_CONSTRAINT": "", + "PIP_EXTRA_INDEX_URL": "", + "PIP_FIND_LINKS": "", + "PIP_REQUIREMENT": "", + }: + raise CheckError("provenance verifier must ignore ambient package configuration") + _, registry_verification = _named_run_step( + registry, + "Verify public artifact identity, digests, and provenance", + 'python scripts/check_registry_release.py --version "$RELEASE_VERSION" ' + '--repository "https://github.com/${{ github.repository }}" ' + "--digest-file release-bundle/artifact-sha256.txt " + "--download-directory registry-artifacts --attempts 12 --retry-delay 10", + "registry verification job", + ) + if _mapping(registry_verification.get("env"), "registry verification environment") != { + "RELEASE_VERSION": "${{ needs.build.outputs.version }}" + }: + raise CheckError("registry verification must use the verified build version") + _, public_install = _named_run_step( + registry, + "Install from public PyPI and run the isolated mocked-call smoke", + 'python scripts/check_clean_install.py --expected-version "$RELEASE_VERSION" ' + '--requirement "cometapi==$RELEASE_VERSION" ' + "--index-url https://pypi.org/simple/ --attempts 12 --retry-delay 10", + "registry verification job", + ) + if _mapping(public_install.get("env"), "registry clean-install environment") != { + "RELEASE_VERSION": "${{ needs.build.outputs.version }}" + }: + raise CheckError("registry clean install must use the verified build version") - if text.count("secrets.COMETAPI_KEY") != 1: + if _secret_references(workflow) != ["${{ secrets.COMETAPI_KEY }}"]: raise CheckError("COMETAPI_KEY must appear only in the exact-release live-smoke job") +def workflow_paths(directory: Path) -> list[Path]: + return sorted(path for path in directory.iterdir() if path.suffix in {".yaml", ".yml"}) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -378,7 +1162,7 @@ def main() -> int: ) check_release_please_workflow(args.release_please_workflow.read_text(encoding="utf-8")) check_ci_workflow(args.ci_workflow.read_text(encoding="utf-8")) - for path in sorted(args.ci_workflow.parent.glob("*.yml")): + for path in workflow_paths(args.ci_workflow.parent): check_action_pins(path.read_text(encoding="utf-8"), path.name) print("release workflow semantic checks passed") return 0 diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index c38639b..257bbd8 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -121,6 +121,14 @@ def test_public_preview_documents_accept_durable_public_content( require_public_preview_docs() +def test_public_preview_documents_reject_codeowners(releasable_documents: Path) -> None: + codeowners = releasable_documents / ".github/CODEOWNERS" + codeowners.parent.mkdir(parents=True, exist_ok=True) + codeowners.write_text("* @placeholder\n", encoding="utf-8") + with pytest.raises(CheckError, match="CODEOWNERS: must remain absent"): + require_public_preview_docs() + + def test_public_preview_documents_report_all_violations_together( releasable_documents: Path, ) -> None: diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index e28823a..1973e07 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import subprocess from collections.abc import Callable from pathlib import Path @@ -12,6 +13,7 @@ check_ci_workflow, check_publish_workflow, check_release_please_workflow, + workflow_paths, ) from tests.live.test_live_smoke import resolve_live_model @@ -320,15 +322,71 @@ def test_live_model_defaults_when_unset_or_empty(configured: str | None) -> None def test_workflow_contract_rejects_mutable_action_reference() -> None: - text = CI_WORKFLOW.read_text(encoding="utf-8").replace( - "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683", - "actions/checkout@v4", - 1, + text, replacements = re.subn( + r"(actions/checkout@)[0-9a-f]{40}", + r"\g<1>v4", + CI_WORKFLOW.read_text(encoding="utf-8"), + count=1, ) + assert replacements == 1 with pytest.raises(RuntimeError, match="full commit SHA"): check_action_pins(text, CI_WORKFLOW.name) +@pytest.mark.parametrize( + "text", + [ + "jobs:\n check:\n runs-on: ubuntu-latest\n steps:\n" + ' - "uses": actions/checkout@v4\n', + "jobs:\n check:\n runs-on: ubuntu-latest\n steps: [{ uses: actions/checkout@v4 }]\n", + "jobs:\n check:\n runs-on: ubuntu-latest\n steps:\n" + " - uses: >-\n actions/checkout@v4\n", + "checkout: &checkout actions/checkout@v4\njobs:\n check:\n" + " runs-on: ubuntu-latest\n steps:\n - uses: *checkout\n", + "jobs:\n check:\n runs-on: ubuntu-latest\n steps:\n" + " - ? uses\n : actions/checkout@v4\n", + "jobs:\n check:\n runs-on: ubuntu-latest\n steps:\n" + " - uses: !!str actions/checkout@v4\n", + ], + ids=[ + "quoted-key", + "inline-mapping", + "folded-scalar", + "alias", + "explicit-mapping", + "tagged-scalar", + ], +) +def test_workflow_contract_rejects_disguised_mutable_action_reference(text: str) -> None: + with pytest.raises(RuntimeError, match="full commit SHA"): + check_action_pins(text, "adversarial workflow") + + +def test_workflow_contract_ignores_nonsemantic_uses_key() -> None: + text = """\ +env: + uses: actions/checkout@v4 +jobs: + check: + runs-on: ubuntu-latest + steps: + - run: echo checked +""" + check_action_pins(text, "nonsemantic uses workflow") + + +def test_workflow_contract_rejects_unpinned_docker_action() -> None: + text = """\ +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: docker://alpine:latest +""" + with pytest.raises(RuntimeError, match="Docker action references are not permitted"): + check_action_pins(text, "Docker workflow") + + @pytest.mark.parametrize( "mutation", PUBLICATION_BYPASSES, @@ -369,6 +427,117 @@ def test_semantic_contract_rejects_publication_bypasses( ) +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + "run: bash scripts/verify_release_trust.sh", + 'run: echo "bash scripts/verify_release_trust.sh"', + ), + ( + "run: uv run python scripts/check_workflows.py", + "run: uv run python scripts/check_workflows.py | true", + ), + ( + 'run: test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT"', + "run: true", + ), + ( + "run: uv run pytest -m live --maxfail=1 -q", + "run: uv run pytest -m live --maxfail=1 -q --collect-only", + ), + ( + "run: sha256sum --check artifact-sha256.txt", + 'run: echo "sha256sum --check artifact-sha256.txt"', + ), + ( + "uses: pypa/gh-action-pypi-publish@", + "uses: attacker/example-action@", + ), + ( + "python -m pip --isolated install", + "echo python -m pip --isolated install", + ), + ( + "python scripts/check_registry_release.py", + "echo python scripts/check_registry_release.py", + ), + ( + "python scripts/check_clean_install.py\n" + ' --expected-version "$RELEASE_VERSION"', + "echo python scripts/check_clean_install.py\n" + ' --expected-version "$RELEASE_VERSION"', + ), + ], + ids=[ + "trust-echo-decoy", + "workflow-check-pipe-bypass", + "commit-check-noop", + "live-collect-only", + "publish-digest-echo-decoy", + "arbitrary-oidc-action", + "provenance-install-echo-decoy", + "registry-verification-echo-decoy", + "registry-install-echo-decoy", + ], +) +def test_semantic_contract_rejects_exact_step_decoys(needle: str, replacement: str) -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError): + check_publish_workflow( + text.replace(needle, replacement, 1), + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + +def test_semantic_contract_rejects_registry_ref_decoy() -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + registry_start = text.index(" verify-registry:") + registry = text[registry_start:].replace( + "ref: ${{ needs.build.outputs.release-commit }}", + "ref: main\n" + " env:\n" + " EXPECTED_REF: ${{ needs.build.outputs.release-commit }}", + 1, + ) + with pytest.raises(RuntimeError): + check_publish_workflow( + text[:registry_start] + registry, + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + +def test_semantic_contract_rejects_publish_environment_test_bypass() -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8").replace( + " UV_VERSION: 0.11.8", + " UV_VERSION: 0.11.8\n PYTEST_ADDOPTS: --collect-only", + 1, + ) + with pytest.raises(RuntimeError, match="pinned uv"): + check_publish_workflow( + text, + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + +def test_semantic_contract_rejects_extra_oidc_step() -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8").replace( + " steps:\n - name: Download the verified release bundle", + " steps:\n" + " - name: Unreviewed OIDC consumer\n" + " uses: attacker/example-action@" + "0123456789abcdef0123456789abcdef01234567\n" + " - name: Download the verified release bundle", + 1, + ) + with pytest.raises(RuntimeError, match="reviewed sequence"): + check_publish_workflow( + text, + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + def test_semantic_contract_rejects_split_monitoring_live_concurrency() -> None: live_smoke = LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8").replace( "group: trusted-live-smoke", "group: monitoring-live-smoke", 1 @@ -423,6 +592,39 @@ def test_semantic_contract_checks_live_smoke_gate_on_smoke_job() -> None: ) +@pytest.mark.parametrize( + ("needle", "replacement", "message"), + [ + ( + "permissions:\n contents: read", + "permissions:\n contents: write", + "permissions", + ), + ( + "run: uv run pytest -m live --maxfail=1 -q", + "continue-on-error: true\n run: uv run pytest -m live --maxfail=1 -q", + "must not allow failure", + ), + ( + "on:\n schedule:", + "on:\n push:\n branches: [main]\n schedule:", + "only on schedule or manual dispatch", + ), + ("runs-on: ubuntu-latest", "runs-on: self-hosted", "GitHub-hosted runner"), + ], + ids=["write-token", "continued-test", "extra-trigger", "self-hosted-runner"], +) +def test_semantic_contract_rejects_monitoring_live_bypasses( + needle: str, replacement: str, message: str +) -> None: + live_smoke = LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8").replace(needle, replacement, 1) + with pytest.raises(RuntimeError, match=message): + check_publish_workflow( + PUBLISH_WORKFLOW.read_text(encoding="utf-8"), + live_smoke, + ) + + def test_current_release_please_workflow_is_disabled_by_default() -> None: check_release_please_workflow(RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8")) @@ -444,6 +646,94 @@ def test_release_please_requires_exact_enable_opt_in(replacement: str) -> None: check_release_please_workflow(text) +def test_release_please_checks_opt_in_on_real_job() -> None: + text = RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8").replace( + "if: vars.RELEASE_PLEASE_ENABLED == 'true'", + "if: github.ref == 'refs/heads/main'", + 1, + ) + text = text.replace( + " steps:\n", + " steps:\n" + " - name: Decoy condition text\n" + " run: >-\n" + " echo \"if: vars.RELEASE_PLEASE_ENABLED == 'true'\"\n", + 1, + ) + with pytest.raises(RuntimeError, match="RELEASE_PLEASE_ENABLED=true"): + check_release_please_workflow(text) + + +def test_release_please_rejects_an_additional_ungated_job() -> None: + text = ( + RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8") + + """ + decoy: + runs-on: ubuntu-latest + steps: + - run: echo bypass +""" + ) + with pytest.raises(RuntimeError, match="only its gated"): + check_release_please_workflow(text) + + +def test_release_please_rejects_trigger_text_hidden_in_name() -> None: + text = RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8").replace( + "name: Release Please", + "name: |\n push:\n branches:\n - main", + 1, + ) + text = text.replace( + "on:\n push:\n branches:\n - main", + 'on:\n schedule:\n - cron: "0 0 * * *"', + 1, + ) + with pytest.raises(RuntimeError, match="default-branch pushes"): + check_release_please_workflow(text) + + +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + " branches:\n - main", + " branches:\n - main\n paths:\n - src/**", + ), + ( + "uses: googleapis/release-please-action@", + "uses: attacker/example-action@", + ), + ( + "config-file: release-please-config.json", + "config-file: attacker-config.json", + ), + ( + "manifest-file: .release-please-manifest.json", + "manifest-file: attacker-manifest.json", + ), + ], + ids=["path-filter", "arbitrary-action", "config-decoy", "manifest-decoy"], +) +def test_release_please_rejects_structural_bypasses(needle: str, replacement: str) -> None: + text = RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError): + check_release_please_workflow(text.replace(needle, replacement, 1)) + + +def test_release_please_rejects_extra_privileged_step() -> None: + text = RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8").replace( + " steps:\n", + " steps:\n" + " - name: Unreviewed token consumer\n" + ' run: echo "${{ github.token }}"\n', + 1, + ) + with pytest.raises(RuntimeError, match="reviewed sequence"): + check_release_please_workflow(text) + + def test_current_ci_workflow_covers_private_remote_validation() -> None: check_ci_workflow(CI_WORKFLOW.read_text(encoding="utf-8")) @@ -456,8 +746,11 @@ def test_current_ci_workflow_covers_private_remote_validation() -> None: " run: uv run python scripts/check_version.py --require-public-preview-docs\n", " - name: Verify from a copied standalone repository\n" " run: python scripts/check_repository_independence.py\n", + " - name: Recheck retained artifact digests\n" + " working-directory: verified-artifacts\n" + " run: sha256sum --check artifact-sha256.txt\n", ], - ids=["lock", "public-content", "standalone"], + ids=["lock", "public-content", "standalone", "artifact-round-trip"], ) def test_ci_contract_rejects_missing_private_validation_gate(needle: str) -> None: text = CI_WORKFLOW.read_text(encoding="utf-8").replace(needle, "", 1) @@ -465,6 +758,186 @@ def test_ci_contract_rejects_missing_private_validation_gate(needle: str) -> Non check_ci_workflow(text) +def test_ci_contract_rejects_commented_public_content_decoy() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8").replace( + " - name: Check canonical public content and identity\n" + " run: uv run python scripts/check_version.py --require-public-preview-docs\n", + " # run: uv run python scripts/check_version.py --require-public-preview-docs\n", + 1, + ) + with pytest.raises(RuntimeError, match="public-preview-docs"): + check_ci_workflow(text) + + +@pytest.mark.parametrize( + ("needle", "replacement", "message"), + [ + ( + "permissions:\n contents: read", + "permissions:\n contents: write", + "permissions", + ), + (" package:\n", " package:\n if: github.event_name == 'never'\n", "conditional"), + (" quality:\n", " quality:\n continue-on-error: true\n", "allow failure"), + ("run: uv lock --check", "run: uv lock --check || true", "active run step"), + ( + "run: uv lock --check", + "run: uv lock --check\n shell: bash -c '{0} || true'", + "command shell", + ), + ], + ids=["write-token", "conditional-package", "continued-quality", "shell-or", "step-shell"], +) +def test_ci_contract_rejects_blocking_bypasses(needle: str, replacement: str, message: str) -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8").replace(needle, replacement, 1) + with pytest.raises(RuntimeError, match=message): + check_ci_workflow(text) + + +def test_ci_contract_rejects_workflow_shell_override() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8").replace( + "\njobs:\n", + "\ndefaults:\n run:\n shell: bash -c '{0} || true'\n\njobs:\n", + 1, + ) + with pytest.raises(RuntimeError, match="command defaults"): + check_ci_workflow(text) + + +def test_ci_contract_rejects_trigger_text_hidden_in_name() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8").replace( + "name: CI", + "name: |\n pull_request:\n push:\n branches:\n - main", + 1, + ) + text = text.replace( + "on:\n pull_request:\n push:\n branches:\n - main\n schedule:", + "on:\n schedule:", + 1, + ) + with pytest.raises(RuntimeError, match="pull requests"): + check_ci_workflow(text) + + +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + " pull_request:\n", + " pull_request:\n paths:\n - src/**\n", + ), + ( + " branches:\n - main\n schedule:", + " branches:\n - main\n paths:\n - src/**\n schedule:", + ), + (' - cron: "23 4 * * 1"', ' - cron: "23 4 * * 2"'), + ( + "if: github.event_name == 'schedule' || github.actor == 'dependabot[bot]'", + "if: github.event_name == 'never'", + ), + ( + 'uv pip install --python .venv/bin/python --upgrade "openai>=2.45.0,<3.0.0"', + 'uv pip install --python .venv/bin/python "openai==2.45.0"', + ), + ( + 'run: uv run --no-sync pytest -m "not live"', + 'run: uv run --no-sync pytest -m "not live" --collect-only', + ), + ( + "python-version: ${{ matrix.python-version }}", + 'python-version: "3.14"', + ), + ("runs-on: ubuntu-latest", "runs-on: self-hosted"), + ( + " UV_VERSION: 0.11.8", + " UV_VERSION: 0.11.8\n PYTEST_ADDOPTS: --collect-only", + ), + ( + "run: uv run python scripts/check_clean_install.py dist/*", + "run: uv run python scripts/check_clean_install.py dist/* | true", + ), + ], + ids=[ + "pull-request-path-filter", + "push-path-filter", + "weekly-schedule-change", + "latest-canary-condition", + "latest-openai-no-upgrade", + "canary-collect-only", + "runtime-matrix-decoy", + "self-hosted-runner", + "workflow-pytest-addopts", + "package-clean-install-pipe", + ], +) +def test_ci_contract_rejects_structural_bypasses(needle: str, replacement: str) -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError): + check_ci_workflow(text.replace(needle, replacement, 1)) + + +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + " quality:\n name:", + " quality:\n env:\n PYTEST_ADDOPTS: --collect-only\n name:", + ), + ( + ' run: uv run pytest -m "not live"', + " env:\n" + " PYTEST_ADDOPTS: --collect-only\n" + ' run: uv run pytest -m "not live"', + ), + ], + ids=["job-environment", "step-environment"], +) +def test_ci_contract_rejects_test_environment_bypasses(needle: str, replacement: str) -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError, match="environment"): + check_ci_workflow(text.replace(needle, replacement, 1)) + + +def test_ci_contract_rejects_bare_secrets_context() -> None: + text = ( + CI_WORKFLOW.read_text(encoding="utf-8") + + """ + secret-context-decoy: + runs-on: ubuntu-latest + steps: + - run: echo "${{ toJSON(secrets) }}" +""" + ) + with pytest.raises(RuntimeError, match="secrets"): + check_ci_workflow(text) + + +def test_ci_contract_keeps_downloaded_artifacts_out_of_copied_candidate() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8") + copied_start, copied_end = _step(text, "Verify from a copied standalone repository") + download_start, download_end = _step(text, "Download the verified package artifacts") + copied_block = text[copied_start:copied_end] + download_block = text[download_start:download_end] + text = ( + text[:copied_start] + + download_block + + text[copied_end:download_start] + + copied_block + + text[download_end:] + ) + with pytest.raises(RuntimeError, match="copied-checkout verification before downloading"): + check_ci_workflow(text) + + +def test_workflow_path_discovery_includes_yaml_and_yml(tmp_path: Path) -> None: + (tmp_path / "first.yml").write_text("name: first\n", encoding="utf-8") + (tmp_path / "second.yaml").write_text("name: second\n", encoding="utf-8") + (tmp_path / "ignored.txt").write_text("ignored\n", encoding="utf-8") + assert [path.name for path in workflow_paths(tmp_path)] == ["first.yml", "second.yaml"] + + def test_semantic_contract_rejects_download_before_checkout() -> None: text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") checkout_start, checkout_end = _step(text, "Check out the registry verification source") diff --git a/uv.lock b/uv.lock index 333fce2..4f6a4f9 100644 --- a/uv.lock +++ b/uv.lock @@ -235,9 +235,11 @@ dev = [ { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pyyaml" }, { name = "ruff" }, { name = "tomli" }, { name = "twine" }, + { name = "types-pyyaml" }, ] [package.metadata] @@ -251,9 +253,11 @@ dev = [ { name = "pyright", specifier = ">=1.1.408,<2.0.0" }, { name = "pytest", specifier = ">=8.3.5,<10.0.0" }, { name = "pytest-asyncio", specifier = ">=0.24.0,<2.0.0" }, + { name = "pyyaml", specifier = ">=6.0.2,<7.0.0" }, { name = "ruff", specifier = ">=0.12.0,<1.0.0" }, { name = "tomli", specifier = ">=2.2.1,<3.0.0" }, { name = "twine", specifier = ">=6.1.0,<7.0.0" }, + { name = "types-pyyaml", specifier = ">=6.0.12,<7.0.0" }, ] [[package]] @@ -892,6 +896,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "readme-renderer" version = "45.0" @@ -1088,6 +1156,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl", hash = "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", size = 42727, upload-time = "2025-09-04T15:43:15.994Z" }, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 45a4b9817c158a17a1bab5c52a76efc3af78323d Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 17:25:05 +0800 Subject: [PATCH 2/6] docs: record pre-visibility validation evidence --- ROADMAP.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 830828e..ba9b518 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -114,7 +114,47 @@ Pre-visibility dependency disposition: | Item | Disposition | Evidence and required action | | --- | --- | --- | -| Dependabot [PR #2](https://github.com/cometapi-dev/cometapi-python/pull/2): `actions/checkout` 4.2.2 to 7.0.1 | Deferred; must not merge as-is | Credential-free [CI run 29796719306](https://github.com/cometapi-dev/cometapi-python/actions/runs/29796719306) failed in every test lane because the mutable-action-reference regression test hard-codes the previous v4 checkout SHA and no longer exercises its replacement; dependent artifact and copied-checkout jobs were skipped. Revisit only with version-independent regression coverage and a completely successful replacement CI run. The failed run is not upgrade evidence. | +| Dependabot [PR #1](https://github.com/cometapi-dev/cometapi-python/pull/1): `actions/download-artifact` 4.3.0 to 8.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin throughout the release workflow and adds a credential-free CI artifact download plus SHA256 round trip. Its current-base [CI run 29907523251](https://github.com/cometapi-dev/cometapi-python/actions/runs/29907523251) passed. Close PR #1 after PR #9 merges; do not merge both. | +| Dependabot [PR #2](https://github.com/cometapi-dev/cometapi-python/pull/2): `actions/checkout` 4.2.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9); must not merge as-is | PR #2's [CI run 29796719306](https://github.com/cometapi-dev/cometapi-python/actions/runs/29796719306) failed because its regression test hard-coded the previous checkout SHA. PR #9 instead validates parsed action references independently of version and passed current-base CI run 29907523251. Close PR #2 after PR #9 merges; the failed PR #2 run remains negative evidence only. | +| Dependabot [PR #3](https://github.com/cometapi-dev/cometapi-python/pull/3): `pypa/gh-action-pypi-publish` 1.14.0 to 1.14.1 | Deferred; keep out of `main` | Pull-request CI does not execute the release-triggered OIDC publish action or prove PyPI publication, provenance, or registry installation. Revisit with an authorized release-path review and the separately required protected release evidence; credential-free CI success alone is insufficient. | +| Dependabot [PR #4](https://github.com/cometapi-dev/cometapi-python/pull/4): `actions/upload-artifact` 4.6.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin in CI and release builds, requires missing artifacts to fail, retains digest evidence, and passed current-base CI run 29907523251. Close PR #4 after PR #9 merges; do not merge both. | +| Dependabot [PR #5](https://github.com/cometapi-dev/cometapi-python/pull/5): `googleapis/release-please-action` 4.4.1 to 5.0.0 | Deferred; keep out of `main` | `RELEASE_PLEASE_ENABLED` remains disabled, and pull-request CI does not execute the gated write-capable Release Please action. Revisit only after its real config, manifest, permissions, and release behavior can be reviewed without treating a skipped action as execution evidence. | +| Dependabot [PR #6](https://github.com/cometapi-dev/cometapi-python/pull/6): `actions/setup-python` 5.6.0 to 7.0.0 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin across CI, monitoring, and release workflows and passed current-base CI run 29907523251 on Python 3.10 through 3.14, the minimum OpenAI lane, package builds, and copied-checkout verification. Close PR #6 after PR #9 merges; do not merge both. | + +Recorded pre-visibility replacement evidence on 2026-07-22: + +- Local commit `97a14ac6087db3c9205e66bcfbcc890dc23a7ca7` passed + `git diff --check`, `uv lock --check`, `uv sync --locked`, + `uv run ruff check src tests scripts`, + `uv run ruff format --check src tests scripts`, `uv run pyright`, + `uv run pytest -m "not live"`, both documented version checks, + `uv run python scripts/check_secrets.py`, and + `uv run python scripts/check_workflows.py`. The offline suite reported 173 + passed and one separately marked live test deselected. +- A clean `uv build` produced the `0.1.0a1` wheel and source distribution. + `uv run twine check dist/*`, + `uv run python scripts/check_artifacts.py dist/*`, and + `uv run python scripts/check_clean_install.py dist/*` passed for both exact + artifacts, including SHA256 digest generation and comparison. +- `uv run python scripts/check_repository_independence.py` passed after copying + the candidate into an empty temporary parent and rerunning its complete + offline, workflow, build, artifact, and two-artifact clean-install gates. + `uv run python scripts/run_actionlint.py` and the same command with + `--offline` both passed with checksum-pinned actionlint 1.7.12. +- An independent 76-case adversarial workflow review found no remaining accepted + hostile case after checking trigger filters, secret-context access, runner and + environment overrides, arbitrary privileged actions, mutable refs, no-op and + failure-swallowing commands, artifact ordering, and release-ref decoys. +- Private PR #9's credential-free CI run 29907523251 passed quality, Python 3.10 + through 3.14, minimum OpenAI, package, exact-artifact clean install, retained + artifact digest, and copied-checkout jobs. The PR-only latest-within-major + canary skipped as designed; scheduled or Dependabot execution remains separate + evidence. +- The live smoke, Release Please, immutable-release publish, PyPI OIDC, + provenance, and public-registry verification paths were not executed. No live + API request, visibility change, secret or environment configuration, tag, + release, publication, or repository-protection change was made. The canonical + repository was confirmed private after the successful replacement run. Changing the repository to public begins a short configuration interval; it does not establish Public Preview readiness by itself. The preview is ready From 88560a889017e2bddc47c52bcaf51e97fa42bcd4 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 17:59:09 +0800 Subject: [PATCH 3/6] ci: close workflow validation bypasses --- scripts/check_secrets.py | 8 +- scripts/check_version.py | 3 +- scripts/check_workflows.py | 233 ++++++++++++++++++++++++++++- tests/test_release_documents.py | 10 ++ tests/test_release_workflow.py | 250 ++++++++++++++++++++++++++++++++ 5 files changed, 494 insertions(+), 10 deletions(-) diff --git a/scripts/check_secrets.py b/scripts/check_secrets.py index 8e694c4..d8f2539 100644 --- a/scripts/check_secrets.py +++ b/scripts/check_secrets.py @@ -94,6 +94,8 @@ def _scan_content(root: Path) -> list[str]: def _scan_workflow_scope(root: Path) -> list[str]: findings: list[str] = [] workflow_root = root / ".github" / "workflows" + if not workflow_root.is_dir(): + return findings ci = workflow_root / "ci.yml" if ci.is_file() and re.search( r"\$\{\{\s*secrets\.", ci.read_text(encoding="utf-8"), flags=re.IGNORECASE @@ -110,7 +112,11 @@ def _scan_workflow_scope(root: Path) -> list[str]: findings.append( ".github/workflows/publish.yml: exactly one job must receive id-token: write" ) - for path in workflow_root.glob("*.yml"): + for path in sorted( + candidate + for candidate in workflow_root.iterdir() + if candidate.is_file() and candidate.suffix in {".yaml", ".yml"} + ): text = path.read_text(encoding="utf-8") if path.name != "publish.yml" and "id-token: write" in text: findings.append(f"{path.relative_to(root)}: id-token: write is publish-job-only") diff --git a/scripts/check_version.py b/scripts/check_version.py index 239171d..b406376 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -160,7 +160,8 @@ def require_public_preview_docs() -> None: documents = _read_public_documents(violations) _check_project_identity(violations) - if Path(".github/CODEOWNERS").exists(): + codeowners = Path(".github/CODEOWNERS") + if codeowners.exists() or codeowners.is_symlink(): violations.append( ".github/CODEOWNERS: must remain absent until a real multi-maintainer model exists" ) diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index d607f3e..c7a1dc8 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -41,6 +41,17 @@ def _scalar(value: object, label: str) -> str: return value +def _require_exact_keys(mapping: dict[str, object], expected: set[str], label: str) -> None: + actual = set(mapping) + if actual != expected: + missing = ", ".join(sorted(expected - actual)) or "none" + unexpected = ", ".join(sorted(actual - expected)) or "none" + raise CheckError( + f"{label} keys do not match the reviewed contract " + f"(missing: {missing}; unexpected: {unexpected})" + ) + + def _load_workflow(text: str, source: str) -> dict[str, object]: try: loaded: object = yaml.load(text, Loader=yaml.BaseLoader) @@ -214,6 +225,22 @@ def _require_step_environments( raise CheckError(f"{label} {name!r} step must not override the environment") +def _require_step_working_directories( + job: dict[str, object], expected: dict[str, str], label: str +) -> None: + matched: set[str] = set() + for index, step in enumerate(_workflow_steps(job, label)): + name = _scalar(step.get("name"), f"{label} step {index} name") + if name in expected: + if step.get("working-directory") != expected[name]: + raise CheckError(f"{label} {name!r} step must use its reviewed working directory") + matched.add(name) + elif "working-directory" in step: + raise CheckError(f"{label} {name!r} step must run from the checked-out repository root") + if matched != set(expected): + raise CheckError(f"{label} reviewed working-directory steps are missing") + + def _action_references(workflow: dict[str, object], source: str) -> Iterator[tuple[str, str]]: jobs = _mapping(workflow.get("jobs"), f"{source} jobs") for job_name, value in jobs.items(): @@ -272,18 +299,45 @@ def check_ci_workflow(text: str) -> None: schedule = _sequence(triggers["schedule"], "CI schedule trigger") if schedule != [{"cron": "23 4 * * 1"}]: raise CheckError("CI latest-OpenAI canary must run on the reviewed weekly schedule") + concurrency = _mapping(workflow.get("concurrency"), "CI workflow concurrency") + if concurrency != { + "group": "ci-${{ github.workflow }}-${{ github.ref }}", + "cancel-in-progress": "true", + }: + raise CheckError("CI must retain reviewed per-ref cancellation") + _require_exact_keys( + workflow, + {"name", "on", "permissions", "concurrency", "env", "jobs"}, + "CI workflow", + ) + if _secret_references(workflow): + raise CheckError("credential-free CI must not reference repository secrets") + jobs = _mapping(workflow.get("jobs"), "CI workflow jobs") + expected_jobs = { + "quality", + "locked-runtime", + "minimum-openai", + "latest-openai", + "package", + "standalone", + } + if set(jobs) != expected_jobs: + raise CheckError("CI jobs must match the reviewed validation chain") quality = _workflow_job(workflow, "quality", "CI workflow") locked_runtime = _workflow_job(workflow, "locked-runtime", "CI workflow") minimum_openai = _workflow_job(workflow, "minimum-openai", "CI workflow") latest_openai = _workflow_job(workflow, "latest-openai", "CI workflow") package = _workflow_job(workflow, "package", "CI workflow") standalone = _workflow_job(workflow, "standalone", "CI workflow") - jobs = _mapping(workflow.get("jobs"), "CI workflow jobs") - for name, value in jobs.items(): - job = _mapping(value, f"CI {name!r} job") - if "permissions" in job: - raise CheckError("CI jobs must not override credential-free workflow permissions") + expected_job_keys = { + "quality": {"name", "runs-on", "timeout-minutes", "steps"}, + "locked-runtime": {"name", "runs-on", "timeout-minutes", "strategy", "steps"}, + "minimum-openai": {"name", "runs-on", "timeout-minutes", "steps"}, + "latest-openai": {"name", "if", "runs-on", "timeout-minutes", "steps"}, + "package": {"name", "needs", "runs-on", "timeout-minutes", "steps"}, + "standalone": {"name", "needs", "runs-on", "timeout-minutes", "steps"}, + } for name, job in ( ("quality", quality), ("locked-runtime", locked_runtime), @@ -304,6 +358,9 @@ def check_ci_workflow(text: str) -> None: _require_unconditional(step, f"CI latest-openai step {index}") if "env" in step: raise CheckError("CI latest-OpenAI steps must not override the CI environment") + for name, value in jobs.items(): + job = _mapping(value, f"CI {name!r} job") + _require_exact_keys(job, expected_job_keys[name], f"CI {name!r} job") _require_needs( package, @@ -358,6 +415,67 @@ def check_ci_workflow(text: str) -> None: "package": package, "standalone": standalone, } + expected_step_names = { + "quality": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Check lock consistency", + "Reproduce the locked environment", + "Lint", + "Check formatting", + "Type check", + "Run offline unit and contract tests", + "Check release version agreement", + "Check canonical public content and identity", + "Scan for credentials and scope mistakes", + "Validate workflow syntax with checksum-pinned actionlint", + "Verify release-workflow trust semantics", + ], + "locked-runtime": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Reproduce the locked environment", + "Run offline tests", + ], + "minimum-openai": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Create the development environment", + "Select the minimum supported OpenAI dependency", + "Run offline tests without resyncing the lock", + ], + "latest-openai": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Create the development environment", + "Select latest OpenAI within the supported major", + "Run canary tests without resyncing the lock", + ], + "package": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Reproduce the locked environment", + "Build wheel and source distribution", + "Check package metadata rendering", + "Inspect artifact identity and shape", + "Install and smoke-test each exact artifact", + "Record immutable artifact digests", + "Retain verified artifacts", + ], + "standalone": [ + "Check out the candidate", + "Set up Python", + "Install the pinned uv frontend", + "Verify from a copied standalone repository", + "Download the verified package artifacts", + "Recheck retained artifact digests", + ], + } expected_timeouts = { "quality": "20", "locked-runtime": "20", @@ -382,6 +500,11 @@ def check_ci_workflow(text: str) -> None: _, setup_step = _named_action_step( job, "Set up Python", "actions/setup-python", f"CI {name} job" ) + _, checkout_step = _named_action_step( + job, "Check out the candidate", "actions/checkout", f"CI {name} job" + ) + if "with" in checkout_step: + raise CheckError(f"CI {name} checkout must use the triggering candidate defaults") _require_options( setup_step, {"python-version": expected_python[name]}, @@ -446,13 +569,27 @@ def check_ci_workflow(text: str) -> None: "CI must finish copied-checkout verification before downloading and " "rechecking retained artifacts" ) - if _secret_references(workflow): - raise CheckError("credential-free CI must not reference repository secrets") + for name, job in required_jobs.items(): + _require_step_names(job, expected_step_names[name], f"CI {name} job") + _require_step_working_directories( + job, + ( + {"Recheck retained artifact digests": "verified-artifacts"} + if name == "standalone" + else {} + ), + f"CI {name} job", + ) def check_release_please_workflow(text: str) -> None: """Require Release Please to remain explicitly disabled by default.""" workflow = _load_workflow(text, "Release Please workflow") + _require_exact_keys( + workflow, + {"name", "on", "permissions", "concurrency", "jobs"}, + "Release Please workflow", + ) _require_permissions(workflow, {"contents": "read"}, "Release Please workflow") if "env" in workflow: raise CheckError("Release Please workflow must not override the action environment") @@ -478,6 +615,11 @@ def check_release_please_workflow(text: str) -> None: if set(jobs) != {"release-please"}: raise CheckError("Release Please must contain only its gated release-please job") release_job = _workflow_job(workflow, "release-please", "Release Please workflow") + _require_exact_keys( + release_job, + {"name", "if", "runs-on", "timeout-minutes", "permissions", "steps"}, + "Release Please job", + ) if release_job.get("if") != "vars.RELEASE_PLEASE_ENABLED == 'true'": raise CheckError("Release Please must require RELEASE_PLEASE_ENABLED=true") if release_job.get("runs-on") != "ubuntu-latest": @@ -503,6 +645,7 @@ def check_release_please_workflow(text: str) -> None: "Release Please job", ) _require_step_environments(release_job, {}, "Release Please job") + _require_step_working_directories(release_job, {}, "Release Please job") _, release_step = _named_action_step( release_job, "Open or update the release PR, or create its approved release", @@ -525,6 +668,16 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: """Validate fail-closed publication, live, permission, and evidence ordering.""" workflow = _load_workflow(text, "publish workflow") live_workflow = _load_workflow(live_smoke_text, "live-smoke workflow") + _require_exact_keys( + workflow, + {"name", "on", "permissions", "concurrency", "env", "jobs"}, + "publish workflow", + ) + _require_exact_keys( + live_workflow, + {"name", "on", "permissions", "concurrency", "env", "jobs"}, + "live-smoke workflow", + ) publish_triggers = _mapping(workflow.get("on"), "publish workflow triggers") if set(publish_triggers) != {"release"}: @@ -582,6 +735,11 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: raise CheckError("release gates must not override command execution") monitoring_job = _workflow_job(live_workflow, "smoke", "live-smoke workflow") + _require_exact_keys( + monitoring_job, + {"name", "if", "runs-on", "timeout-minutes", "environment", "steps"}, + "monitoring live-smoke job", + ) monitoring_condition = " ".join( _scalar(monitoring_job.get("if"), "monitoring live-smoke condition").split() ) @@ -630,6 +788,7 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: }, "monitoring live-smoke job", ) + _require_step_working_directories(monitoring_job, {}, "monitoring live-smoke job") _, monitoring_checkout = _named_action_step( monitoring_job, "Check out the trusted default branch", @@ -683,6 +842,49 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: release_live = _workflow_job(workflow, "release-live-smoke", "publish workflow") publish = _workflow_job(workflow, "publish", "publish workflow") registry = _workflow_job(workflow, "verify-registry", "publish workflow") + expected_release_job_keys = { + "build": {"name", "runs-on", "timeout-minutes", "outputs", "permissions", "steps"}, + "release-live-smoke": { + "name", + "needs", + "concurrency", + "runs-on", + "timeout-minutes", + "permissions", + "environment", + "env", + "steps", + }, + "publish": { + "name", + "needs", + "runs-on", + "timeout-minutes", + "environment", + "permissions", + "steps", + }, + "verify-registry": { + "name", + "needs", + "runs-on", + "timeout-minutes", + "permissions", + "steps", + }, + } + for name, job in ( + ("build", build), + ("release-live-smoke", release_live), + ("publish", publish), + ("verify-registry", registry), + ): + _require_exact_keys(job, expected_release_job_keys[name], f"release {name} job") + _require_step_working_directories( + job, + ({"Recheck immutable artifact digests": "release-bundle"} if name == "publish" else {}), + f"release {name} job", + ) for name, job, timeout in ( ("build", build, "25"), ("release-live-smoke", release_live, "10"), @@ -1133,6 +1335,20 @@ def workflow_paths(directory: Path) -> list[Path]: return sorted(path for path in directory.iterdir() if path.suffix in {".yaml", ".yml"}) +def check_workflow_inventory(directory: Path) -> list[Path]: + paths = workflow_paths(directory) + expected = {"ci.yml", "live-smoke.yml", "publish.yml", "release-please.yml"} + actual = {path.name for path in paths} + if actual != expected: + missing = ", ".join(sorted(expected - actual)) or "none" + unexpected = ", ".join(sorted(actual - expected)) or "none" + raise CheckError( + "workflow inventory does not match the reviewed contract " + f"(missing: {missing}; unexpected: {unexpected})" + ) + return paths + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -1156,13 +1372,14 @@ def main() -> int: default=PROJECT_ROOT / ".github" / "workflows" / "ci.yml", ) args = parser.parse_args() + paths = check_workflow_inventory(args.ci_workflow.parent) check_publish_workflow( args.publish_workflow.read_text(encoding="utf-8"), args.live_smoke_workflow.read_text(encoding="utf-8"), ) check_release_please_workflow(args.release_please_workflow.read_text(encoding="utf-8")) check_ci_workflow(args.ci_workflow.read_text(encoding="utf-8")) - for path in workflow_paths(args.ci_workflow.parent): + for path in paths: check_action_pins(path.read_text(encoding="utf-8"), path.name) print("release workflow semantic checks passed") return 0 diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index 257bbd8..08b7213 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -129,6 +129,16 @@ def test_public_preview_documents_reject_codeowners(releasable_documents: Path) require_public_preview_docs() +def test_public_preview_documents_reject_broken_codeowners_symlink( + releasable_documents: Path, +) -> None: + codeowners = releasable_documents / ".github/CODEOWNERS" + codeowners.parent.mkdir(parents=True, exist_ok=True) + codeowners.symlink_to("missing") + with pytest.raises(CheckError, match="CODEOWNERS: must remain absent"): + require_public_preview_docs() + + def test_public_preview_documents_report_all_violations_together( releasable_documents: Path, ) -> None: diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index 1973e07..dd25c5a 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -3,6 +3,7 @@ import os import re import subprocess +import sys from collections.abc import Callable from pathlib import Path @@ -13,6 +14,7 @@ check_ci_workflow, check_publish_workflow, check_release_please_workflow, + check_workflow_inventory, workflow_paths, ) from tests.live.test_live_smoke import resolve_live_model @@ -625,6 +627,115 @@ def test_semantic_contract_rejects_monitoring_live_bypasses( ) +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + " run: bash scripts/verify_release_trust.sh", + " working-directory: decoy\n run: bash scripts/verify_release_trust.sh", + ), + ( + " run: uv run pytest -m live --maxfail=1 -q", + " working-directory: decoy\n run: uv run pytest -m live --maxfail=1 -q", + ), + ( + " run: >-\n python scripts/check_registry_release.py", + " working-directory: decoy\n" + " run: >-\n" + " python scripts/check_registry_release.py", + ), + ], + ids=["release-trust", "exact-release-live", "registry-provenance"], +) +def test_semantic_contract_rejects_release_working_directory_redirects( + needle: str, replacement: str +) -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError, match="repository root"): + check_publish_workflow( + text.replace(needle, replacement, 1), + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + +def test_semantic_contract_rejects_monitoring_working_directory_redirect() -> None: + live_smoke = LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8").replace( + " run: uv run pytest -m live --maxfail=1 -q", + " working-directory: decoy\n run: uv run pytest -m live --maxfail=1 -q", + 1, + ) + with pytest.raises(RuntimeError, match="repository root"): + check_publish_workflow( + PUBLISH_WORKFLOW.read_text(encoding="utf-8"), + live_smoke, + ) + + +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + " build:\n name:", + " build:\n container: attacker/image:latest\n name:", + ), + ( + " release-live-smoke:\n name:", + " release-live-smoke:\n" + " strategy:\n" + " matrix:\n" + " copy: [one, two]\n" + " name:", + ), + ( + " publish:\n name:", + " publish:\n strategy:\n matrix:\n copy: [one, two]\n name:", + ), + ( + " verify-registry:\n name:", + " verify-registry:\n" + " services:\n" + " unreviewed:\n" + " image: attacker/image:latest\n" + " name:", + ), + ], + ids=["build-container", "live-matrix", "publish-matrix", "registry-service"], +) +def test_semantic_contract_rejects_unreviewed_release_job_controls( + needle: str, replacement: str +) -> None: + text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError, match="reviewed contract"): + check_publish_workflow( + text.replace(needle, replacement, 1), + LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8"), + ) + + +@pytest.mark.parametrize( + "control", + [ + " container: attacker/image:latest\n", + " strategy:\n matrix:\n copy: [one, two]\n", + " services:\n unreviewed:\n image: attacker/image:latest\n", + ], + ids=["container", "matrix", "service"], +) +def test_semantic_contract_rejects_unreviewed_monitoring_job_controls(control: str) -> None: + live_smoke = LIVE_SMOKE_WORKFLOW.read_text(encoding="utf-8").replace( + " smoke:\n name:", + f" smoke:\n{control} name:", + 1, + ) + with pytest.raises(RuntimeError, match="reviewed contract"): + check_publish_workflow( + PUBLISH_WORKFLOW.read_text(encoding="utf-8"), + live_smoke, + ) + + def test_current_release_please_workflow_is_disabled_by_default() -> None: check_release_please_workflow(RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8")) @@ -734,10 +845,109 @@ def test_release_please_rejects_extra_privileged_step() -> None: check_release_please_workflow(text) +@pytest.mark.parametrize( + "control", + [ + " container: attacker/image:latest\n", + " strategy:\n matrix:\n copy: [one, two]\n", + " services:\n unreviewed:\n image: attacker/image:latest\n", + ], + ids=["container", "matrix", "service"], +) +def test_release_please_rejects_unreviewed_job_controls(control: str) -> None: + text = RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8").replace( + " release-please:\n name:", + f" release-please:\n{control} name:", + 1, + ) + with pytest.raises(RuntimeError, match="reviewed contract"): + check_release_please_workflow(text) + + def test_current_ci_workflow_covers_private_remote_validation() -> None: check_ci_workflow(CI_WORKFLOW.read_text(encoding="utf-8")) +@pytest.mark.parametrize( + ("needle", "replacement", "message"), + [ + ( + " - name: Check out the candidate\n", + " - name: Check out the candidate\n with:\n ref: main\n", + "triggering candidate", + ), + ( + " - name: Run offline unit and contract tests\n" + ' run: uv run pytest -m "not live"', + " - name: Run offline unit and contract tests\n" + " working-directory: decoy\n" + ' run: uv run pytest -m "not live"', + "repository root", + ), + ( + " quality:\n name:", + " quality:\n container: attacker/image:latest\n name:", + "reviewed contract", + ), + ( + " quality:\n name:", + " quality:\n" + " services:\n" + " unreviewed:\n" + " image: attacker/image:latest\n" + " name:", + "reviewed contract", + ), + ( + " quality:\n name:", + " quality:\n strategy:\n matrix:\n copy: [one, two]\n name:", + "reviewed contract", + ), + ( + " steps:\n - name: Check out the candidate", + " steps:\n" + " - name: Unreviewed action\n" + " uses: attacker/example-action@" + "0123456789abcdef0123456789abcdef01234567\n" + " - name: Check out the candidate", + "reviewed sequence", + ), + ], + ids=[ + "checkout-ref", + "working-directory", + "container", + "service", + "matrix", + "extra-action", + ], +) +def test_ci_contract_rejects_candidate_execution_redirects( + needle: str, replacement: str, message: str +) -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError, match=message): + check_ci_workflow(text.replace(needle, replacement, 1)) + + +def test_ci_contract_rejects_extra_skip_dependency_job() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8").replace( + " quality:\n", + " skip-gate:\n" + " name: Skip required evidence\n" + " if: github.event_name == 'never'\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: true\n\n" + " quality:\n" + " needs: skip-gate\n", + 1, + ) + with pytest.raises(RuntimeError, match="reviewed validation chain"): + check_ci_workflow(text) + + @pytest.mark.parametrize( "needle", [ @@ -938,6 +1148,46 @@ def test_workflow_path_discovery_includes_yaml_and_yml(tmp_path: Path) -> None: assert [path.name for path in workflow_paths(tmp_path)] == ["first.yml", "second.yaml"] +def test_workflow_inventory_rejects_unreviewed_workflow(tmp_path: Path) -> None: + for name in ("ci.yml", "live-smoke.yml", "publish.yml", "release-please.yml"): + (tmp_path / name).write_text("name: reviewed\n", encoding="utf-8") + assert {path.name for path in check_workflow_inventory(tmp_path)} == { + "ci.yml", + "live-smoke.yml", + "publish.yml", + "release-please.yml", + } + + (tmp_path / "rogue.yaml").write_text( + "permissions:\n id-token: write\njobs:\n rogue:\n runs-on: ubuntu-latest\n", + encoding="utf-8", + ) + with pytest.raises(RuntimeError, match=r"unexpected: rogue\.yaml"): + check_workflow_inventory(tmp_path) + + +def test_secret_scope_scan_includes_yaml_workflows(tmp_path: Path) -> None: + workflow_root = tmp_path / ".github/workflows" + workflow_root.mkdir(parents=True) + (workflow_root / "rogue.yaml").write_text( + "permissions:\n id-token: write\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + str(PROJECT_ROOT / "scripts/check_secrets.py"), + "--root", + str(tmp_path), + ], + text=True, + check=False, + capture_output=True, + ) + assert result.returncode != 0 + assert ".github/workflows/rogue.yaml: id-token: write is publish-job-only" in result.stderr + + def test_semantic_contract_rejects_download_before_checkout() -> None: text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") checkout_start, checkout_end = _step(text, "Check out the registry verification source") From 357995eec35ef612f81e724c692dc56bf5b9d6cb Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 18:02:50 +0800 Subject: [PATCH 4/6] ci: reject linked workflow definitions --- scripts/check_workflows.py | 3 +++ tests/test_release_workflow.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index c7a1dc8..f0f1934 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -1346,6 +1346,9 @@ def check_workflow_inventory(directory: Path) -> list[Path]: "workflow inventory does not match the reviewed contract " f"(missing: {missing}; unexpected: {unexpected})" ) + for path in paths: + if path.is_symlink() or not path.is_file(): + raise CheckError(f"{path.name}: reviewed workflows must be regular, non-symlink files") return paths diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index dd25c5a..e14a94d 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -1166,6 +1166,19 @@ def test_workflow_inventory_rejects_unreviewed_workflow(tmp_path: Path) -> None: check_workflow_inventory(tmp_path) +def test_workflow_inventory_rejects_expected_name_symlink(tmp_path: Path) -> None: + workflow_root = tmp_path / "workflows" + workflow_root.mkdir() + for name in ("live-smoke.yml", "publish.yml", "release-please.yml"): + (workflow_root / name).write_text("name: reviewed\n", encoding="utf-8") + outside = tmp_path / "outside-ci.yml" + outside.write_text("name: outside\n", encoding="utf-8") + (workflow_root / "ci.yml").symlink_to(outside) + + with pytest.raises(RuntimeError, match="regular, non-symlink files"): + check_workflow_inventory(workflow_root) + + def test_secret_scope_scan_includes_yaml_workflows(tmp_path: Path) -> None: workflow_root = tmp_path / ".github/workflows" workflow_root.mkdir(parents=True) From 668b78f89e8962cc8ab1d1aca8fe3d24c38723ac Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 18:16:34 +0800 Subject: [PATCH 5/6] ci: reject linked workflow directories --- scripts/check_workflows.py | 4 ++++ tests/test_release_workflow.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index f0f1934..2c492d2 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -1336,6 +1336,10 @@ def workflow_paths(directory: Path) -> list[Path]: def check_workflow_inventory(directory: Path) -> list[Path]: + if directory.is_symlink() or directory.parent.is_symlink() or not directory.is_dir(): + raise CheckError( + "workflow directory and .github parent must be real repository directories" + ) paths = workflow_paths(directory) expected = {"ci.yml", "live-smoke.yml", "publish.yml", "release-please.yml"} actual = {path.name for path in paths} diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index e14a94d..ad9a7e1 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -1179,6 +1179,27 @@ def test_workflow_inventory_rejects_expected_name_symlink(tmp_path: Path) -> Non check_workflow_inventory(workflow_root) +@pytest.mark.parametrize("linked_component", ["github", "workflows"]) +def test_workflow_inventory_rejects_linked_directory(tmp_path: Path, linked_component: str) -> None: + outside = tmp_path / "outside" + outside_workflows = outside / "workflows" + outside_workflows.mkdir(parents=True) + for name in ("ci.yml", "live-smoke.yml", "publish.yml", "release-please.yml"): + (outside_workflows / name).write_text("name: outside\n", encoding="utf-8") + + repository = tmp_path / "repository" + repository.mkdir() + github = repository / ".github" + if linked_component == "github": + github.symlink_to(outside, target_is_directory=True) + else: + github.mkdir() + (github / "workflows").symlink_to(outside_workflows, target_is_directory=True) + + with pytest.raises(RuntimeError, match="real repository directories"): + check_workflow_inventory(github / "workflows") + + def test_secret_scope_scan_includes_yaml_workflows(tmp_path: Path) -> None: workflow_root = tmp_path / ".github/workflows" workflow_root.mkdir(parents=True) From 5db7f012a1470564f4f60fe343b9a0799b58987d Mon Sep 17 00:00:00 2001 From: CometAPI Date: Wed, 22 Jul 2026 18:18:32 +0800 Subject: [PATCH 6/6] docs: finalize pre-visibility evidence --- ROADMAP.md | 134 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 103 insertions(+), 31 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index ba9b518..0f1c7a1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -114,47 +114,119 @@ Pre-visibility dependency disposition: | Item | Disposition | Evidence and required action | | --- | --- | --- | -| Dependabot [PR #1](https://github.com/cometapi-dev/cometapi-python/pull/1): `actions/download-artifact` 4.3.0 to 8.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin throughout the release workflow and adds a credential-free CI artifact download plus SHA256 round trip. Its current-base [CI run 29907523251](https://github.com/cometapi-dev/cometapi-python/actions/runs/29907523251) passed. Close PR #1 after PR #9 merges; do not merge both. | -| Dependabot [PR #2](https://github.com/cometapi-dev/cometapi-python/pull/2): `actions/checkout` 4.2.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9); must not merge as-is | PR #2's [CI run 29796719306](https://github.com/cometapi-dev/cometapi-python/actions/runs/29796719306) failed because its regression test hard-coded the previous checkout SHA. PR #9 instead validates parsed action references independently of version and passed current-base CI run 29907523251. Close PR #2 after PR #9 merges; the failed PR #2 run remains negative evidence only. | +| Dependabot [PR #1](https://github.com/cometapi-dev/cometapi-python/pull/1): `actions/download-artifact` 4.3.0 to 8.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin throughout the release workflow and adds a credential-free CI artifact download plus SHA256 round trip. Its initial code-bearing [CI run 29907523251](https://github.com/cometapi-dev/cometapi-python/actions/runs/29907523251) passed. Close PR #1 after PR #9 merges; do not merge both. | +| Dependabot [PR #2](https://github.com/cometapi-dev/cometapi-python/pull/2): `actions/checkout` 4.2.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9); must not merge as-is | PR #2's [CI run 29796719306](https://github.com/cometapi-dev/cometapi-python/actions/runs/29796719306) failed because its regression test hard-coded the previous checkout SHA. PR #9 instead validates parsed action references independently of version and passed initial code-bearing CI run 29907523251. Close PR #2 after PR #9 merges; the failed PR #2 run remains negative evidence only. | | Dependabot [PR #3](https://github.com/cometapi-dev/cometapi-python/pull/3): `pypa/gh-action-pypi-publish` 1.14.0 to 1.14.1 | Deferred; keep out of `main` | Pull-request CI does not execute the release-triggered OIDC publish action or prove PyPI publication, provenance, or registry installation. Revisit with an authorized release-path review and the separately required protected release evidence; credential-free CI success alone is insufficient. | -| Dependabot [PR #4](https://github.com/cometapi-dev/cometapi-python/pull/4): `actions/upload-artifact` 4.6.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin in CI and release builds, requires missing artifacts to fail, retains digest evidence, and passed current-base CI run 29907523251. Close PR #4 after PR #9 merges; do not merge both. | +| Dependabot [PR #4](https://github.com/cometapi-dev/cometapi-python/pull/4): `actions/upload-artifact` 4.6.2 to 7.0.1 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin in CI and release builds, requires missing artifacts to fail, retains digest evidence, and passed initial code-bearing CI run 29907523251. Close PR #4 after PR #9 merges; do not merge both. | | Dependabot [PR #5](https://github.com/cometapi-dev/cometapi-python/pull/5): `googleapis/release-please-action` 4.4.1 to 5.0.0 | Deferred; keep out of `main` | `RELEASE_PLEASE_ENABLED` remains disabled, and pull-request CI does not execute the gated write-capable Release Please action. Revisit only after its real config, manifest, permissions, and release behavior can be reviewed without treating a skipped action as execution evidence. | -| Dependabot [PR #6](https://github.com/cometapi-dev/cometapi-python/pull/6): `actions/setup-python` 5.6.0 to 7.0.0 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin across CI, monitoring, and release workflows and passed current-base CI run 29907523251 on Python 3.10 through 3.14, the minimum OpenAI lane, package builds, and copied-checkout verification. Close PR #6 after PR #9 merges; do not merge both. | +| Dependabot [PR #6](https://github.com/cometapi-dev/cometapi-python/pull/6): `actions/setup-python` 5.6.0 to 7.0.0 | Superseded by private [PR #9](https://github.com/cometapi-dev/cometapi-python/pull/9) | PR #9 applies the reviewed SHA pin across CI, monitoring, and release workflows and passed initial code-bearing CI run 29907523251 on Python 3.10 through 3.14, the minimum OpenAI lane, package builds, and copied-checkout verification. Close PR #6 after PR #9 merges; do not merge both. | Recorded pre-visibility replacement evidence on 2026-07-22: -- Local commit `97a14ac6087db3c9205e66bcfbcc890dc23a7ca7` passed - `git diff --check`, `uv lock --check`, `uv sync --locked`, - `uv run ruff check src tests scripts`, - `uv run ruff format --check src tests scripts`, `uv run pyright`, - `uv run pytest -m "not live"`, both documented version checks, - `uv run python scripts/check_secrets.py`, and - `uv run python scripts/check_workflows.py`. The offline suite reported 173 - passed and one separately marked live test deselected. -- A clean `uv build` produced the `0.1.0a1` wheel and source distribution. - `uv run twine check dist/*`, +Local and package evidence at commit +`97a14ac6087db3c9205e66bcfbcc890dc23a7ca7`: + +- `git diff --check`, `uv lock --check`, and `uv sync --locked` passed. +- `uv run ruff check src tests scripts`, + `uv run ruff format --check src tests scripts`, and `uv run pyright` passed. +- `uv run pytest -m "not live"` passed with 173 tests passed and one separately + marked live test deselected. +- `uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog` + and + `uv run python scripts/check_version.py --require-public-preview-docs` passed. +- `uv run python scripts/check_secrets.py` and + `uv run python scripts/check_workflows.py` passed. +- `rm -rf dist` completed, and `uv build` produced the `0.1.0a1` wheel and source + distribution in the clean output directory. +- `uv run twine check dist/*`, `uv run python scripts/check_artifacts.py dist/*`, and `uv run python scripts/check_clean_install.py dist/*` passed for both exact - artifacts, including SHA256 digest generation and comparison. + artifacts, including SHA256 digest generation. - `uv run python scripts/check_repository_independence.py` passed after copying the candidate into an empty temporary parent and rerunning its complete offline, workflow, build, artifact, and two-artifact clean-install gates. - `uv run python scripts/run_actionlint.py` and the same command with - `--offline` both passed with checksum-pinned actionlint 1.7.12. -- An independent 76-case adversarial workflow review found no remaining accepted - hostile case after checking trigger filters, secret-context access, runner and - environment overrides, arbitrary privileged actions, mutable refs, no-op and - failure-swallowing commands, artifact ordering, and release-ref decoys. -- Private PR #9's credential-free CI run 29907523251 passed quality, Python 3.10 - through 3.14, minimum OpenAI, package, exact-artifact clean install, retained - artifact digest, and copied-checkout jobs. The PR-only latest-within-major - canary skipped as designed; scheduled or Dependabot execution remains separate - evidence. -- The live smoke, Release Please, immutable-release publish, PyPI OIDC, - provenance, and public-registry verification paths were not executed. No live - API request, visibility change, secret or environment configuration, tag, - release, publication, or repository-protection change was made. The canonical - repository was confirmed private after the successful replacement run. +- `uv run python scripts/run_actionlint.py` and + `uv run python scripts/run_actionlint.py --offline` passed with + checksum-pinned actionlint 1.7.12. + +Follow-up verifier-hardening evidence at commit +`88560a889017e2bddc47c52bcaf51e97fa42bcd4`: + +- `git diff --check`, `uv lock --check`, and `uv sync --locked` passed. +- `uv run ruff check src tests scripts`, + `uv run ruff format --check src tests scripts`, and `uv run pyright` passed. +- `uv run pytest -m "not live"` passed with 197 tests passed and one separately + marked live test deselected. +- `uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog`, + `uv run python scripts/check_version.py --require-public-preview-docs`, + `uv run python scripts/check_secrets.py`, and + `uv run python scripts/check_workflows.py` passed. +- `uv run python scripts/run_actionlint.py` and + `uv run python scripts/run_actionlint.py --offline` passed with + checksum-pinned actionlint 1.7.12. +- `uv build`, `uv run twine check dist/*`, + `uv run python scripts/check_artifacts.py dist/*`, and + `uv run python scripts/check_clean_install.py dist/*` passed for the rebuilt + wheel and source distribution. +- `uv run python scripts/check_repository_independence.py` passed the complete + copied-checkout gate, including its offline suite, build, artifact checks, and + independent clean installs of both artifacts. +- Independent adversarial workflow review and targeted follow-up regression + coverage found no remaining accepted hostile case after checking trigger + filters, secret-context access, runner, container, matrix, working-directory, + checkout, job, step, and environment overrides, arbitrary privileged actions, + mutable refs, no-op and failure-swallowing commands, artifact ordering, and + release-ref decoys. + +Final workflow-inventory hardening evidence at commit +`668b78f89e8962cc8ab1d1aca8fe3d24c38723ac`: + +- `git diff --check`, `uv lock --check`, and `uv sync --locked` passed. +- `uv run ruff check src tests scripts`, + `uv run ruff format --check src tests scripts`, and `uv run pyright` passed. +- `uv run pytest -m "not live"` passed with 200 tests passed and one separately + marked live test deselected. +- `uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog`, + `uv run python scripts/check_version.py --require-public-preview-docs`, + `uv run python scripts/check_secrets.py`, and + `uv run python scripts/check_workflows.py` passed. +- `uv run python scripts/run_actionlint.py` and + `uv run python scripts/run_actionlint.py --offline` passed with + checksum-pinned actionlint 1.7.12. +- `uv run python scripts/check_repository_independence.py` passed the complete + copied-checkout gate, including 200 offline tests, the package build, artifact + inspection, and independent clean installs of the wheel and source + distribution. + +Failed or unavailable checks: + +- None of the recorded final-candidate checks failed or were unavailable. + Dependabot PR #2's failed run remains separate negative evidence for that PR, + not replacement evidence for PR #9. An earlier intentional offline actionlint + probe in a fresh detached worktree failed closed before the verified cache was + populated; it is not final-candidate validation evidence. + +Remote evidence: + +- Private PR #9's credential-free initial code-bearing CI run 29907523251 passed + quality, Python 3.10 through 3.14, minimum OpenAI, package, exact-artifact + clean install, retained artifact digest, and copied-checkout jobs. The PR-only + latest-within-major canary skipped as designed; scheduled or Dependabot + execution remains unverified. +- The canonical repository was confirmed private after the successful + replacement run. No visibility, secret, environment, or + repository-protection change was made. + +Live evidence: + +- The live-smoke path was not executed, and no live API request was made. + Transport success and provider behavior therefore remain unverified. + +Registry and release evidence: + +- Release Please, immutable-release publishing, PyPI OIDC, provenance, and + public-registry installation were not executed. No tag, release, or + publication was created. Changing the repository to public begins a short configuration interval; it does not establish Public Preview readiness by itself. The preview is ready