From 54ed21872bff4719886578fe4ad2ddfd86d12bc3 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Tue, 28 Jul 2026 16:42:06 +0800 Subject: [PATCH] feat: prepare the stable release --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 27 ++- .github/workflows/release-please.yml | 63 ++++- AGENTS.md | 32 ++- ARCHITECTURE.md | 9 +- CONTRIBUTING.md | 2 +- README.md | 6 +- RELEASING.md | 54 +++-- ROADMAP.md | 27 ++- release-please-config.json | 11 +- scripts/check_artifacts.py | 2 + scripts/check_clean_install.py | 342 ++++++++++++++++++++++++++- scripts/check_secrets.py | 21 +- scripts/check_workflows.py | 271 ++++++++++++++++++--- scripts/verify_release_trust.sh | 6 + tests/test_clean_install.py | 196 +++++++++++++++ tests/test_client.py | 3 +- tests/test_release_workflow.py | 113 ++++++++- tests/test_secrets.py | 45 ++++ 19 files changed, 1129 insertions(+), 103 deletions(-) create mode 100644 tests/test_clean_install.py create mode 100644 tests/test_secrets.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 704200c..bbd9285 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Run offline unit and contract tests run: uv run pytest -m "not live" - name: Check release version agreement - run: uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog + run: uv run python scripts/check_version.py --require-changelog - name: Check canonical public content and identity run: uv run python scripts/check_version.py --require-public-preview-docs - name: Scan for credentials and scope mistakes diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fa5bb86..f302504 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,9 +1,17 @@ name: Publish immutable release on: - release: - types: - - published + workflow_call: + inputs: + release-tag: + required: true + type: string + release-sha: + required: true + type: string + default-branch: + required: true + type: string permissions: contents: read @@ -29,15 +37,16 @@ jobs: - name: Check out the published release tag uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: refs/tags/${{ github.event.release.tag_name }} + ref: refs/tags/${{ inputs.release-tag }} fetch-depth: 0 persist-credentials: false - name: Reject an untrusted release target id: trust env: - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - RELEASE_IMMUTABLE: ${{ github.event.release.immutable }} - RELEASE_TAG: ${{ github.event.release.tag_name }} + DEFAULT_BRANCH: ${{ inputs.default-branch }} + EXPECTED_RELEASE_SHA: ${{ inputs.release-sha }} + RELEASE_IMMUTABLE: "true" + RELEASE_TAG: ${{ inputs.release-tag }} run: bash scripts/verify_release_trust.sh - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -50,7 +59,7 @@ jobs: - name: Verify project, manifest, changelog, release docs, and tag agreement id: version env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.release-tag }} run: | version=$(uv run python scripts/check_version.py --tag "$RELEASE_TAG" --require-changelog --require-releasable-docs --print-version) echo "version=$version" >> "$GITHUB_OUTPUT" @@ -62,7 +71,7 @@ jobs: run: uv build - name: Verify artifact versions against the tag env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TAG: ${{ inputs.release-tag }} run: uv run python scripts/check_version.py --tag "$RELEASE_TAG" --require-changelog --require-releasable-docs dist/* - name: Check package metadata rendering run: uv run twine check dist/* diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 54f8a88..ea8bb65 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -17,13 +17,74 @@ jobs: name: Maintain the reviewed release PR and release if: vars.RELEASE_PLEASE_ENABLED == 'true' runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 + outputs: + release-created: ${{ steps.release.outputs.release_created }} + release-sha: ${{ steps.verify-release.outputs.release-sha }} + release-tag: ${{ steps.verify-release.outputs.release-tag }} + release-verified: ${{ steps.verify-release.outputs.release-verified }} permissions: contents: write pull-requests: write steps: - name: Open or update the release PR, or create its approved release + id: release uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 with: config-file: release-please-config.json manifest-file: .release-please-manifest.json + - name: Verify the immutable release created by Release Please + id: verify-release + if: steps.release.outputs.release_created == 'true' + env: + EXPECTED_SHA: ${{ steps.release.outputs.sha }} + EXPECTED_TAG: ${{ steps.release.outputs.tag_name }} + GH_TOKEN: ${{ github.token }} + run: | + test -n "$EXPECTED_TAG" + test -n "$EXPECTED_SHA" + release="" + for attempt in $(seq 1 12); do + release=$(gh api "repos/${{ github.repository }}/releases/tags/$EXPECTED_TAG") || true + if test -n "$release" && test "$(jq -r .immutable <<<"$release")" = "true"; then + break + fi + if test "$attempt" -ge 12; then + echo "release did not become immutable" >&2 + exit 1 + fi + sleep 5 + done + test "$(jq -r .tag_name <<<"$release")" = "$EXPECTED_TAG" + test "$(jq -r .draft <<<"$release")" = "false" + test "$(jq -r .prerelease <<<"$release")" = "false" + test "$(jq -r .immutable <<<"$release")" = "true" + ref=$(gh api "repos/${{ github.repository }}/git/ref/tags/$EXPECTED_TAG") + tag_type=$(jq -r .object.type <<<"$ref") + tag_sha=$(jq -r .object.sha <<<"$ref") + if test "$tag_type" = "tag"; then + tag_sha=$(gh api "repos/${{ github.repository }}/git/tags/$tag_sha" --jq .object.sha) + else + test "$tag_type" = "commit" + fi + test "$tag_sha" = "$EXPECTED_SHA" + { + echo "release-tag=$EXPECTED_TAG" + echo "release-sha=$EXPECTED_SHA" + echo "release-verified=true" + } >> "$GITHUB_OUTPUT" + + publish-release: + name: Run the protected publication chain + needs: release-please + if: >- + needs.release-please.outputs.release-created == 'true' && + needs.release-please.outputs.release-verified == 'true' + permissions: + contents: read + id-token: write + uses: ./.github/workflows/publish.yml + with: + release-tag: ${{ needs.release-please.outputs.release-tag }} + release-sha: ${{ needs.release-please.outputs.release-sha }} + default-branch: ${{ github.event.repository.default_branch }} diff --git a/AGENTS.md b/AGENTS.md index 6363511..59fd9d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,9 +96,10 @@ Post-alpha invariants: authorizes that work, without reopening completed dispositions. 2. Keep `.github/CODEOWNERS` absent until a real multi-maintainer model exists. 3. Keep scheduled and manually dispatched live smoke fail-closed behind - `LIVE_SMOKE_ENABLED=true`, and keep `RELEASE_PLEASE_ENABLED` disabled until - a separate reviewed and tested `last-release-sha` bridge establishes the - recovery alpha as Release Please's previous-release boundary. + `LIVE_SMOKE_ENABLED=true`, and keep `RELEASE_PLEASE_ENABLED` disabled outside + an explicitly authorized release sequence. The stable-readiness + configuration establishes the recovery alpha boundary with a reviewed and + tested `last-release-sha` bridge. 4. Treat the recorded public rules, security reporting, immutable releases, and protected environments as readiness invariants. Any drift invalidates the readiness claim until it is explicitly authorized, restored, and verified. @@ -205,7 +206,7 @@ uv run ruff check src tests scripts uv run ruff format --check src tests scripts uv run pyright uv run pytest -m "not live" -uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog +uv run python scripts/check_version.py --require-changelog uv run python scripts/check_secrets.py uv run python scripts/check_workflows.py rm -rf dist @@ -242,8 +243,11 @@ committed. - Publication uses a reviewed immutable tag, a protected `pypi` environment, and PyPI OIDC Trusted Publishing. - The release commit must equal the tag target and belong to the protected - default branch. A protected live-smoke job must check out that exact commit - and succeed before the protected PyPI job can become eligible. + default branch. Release Please must independently confirm that the exact tag + and commit are immutable before directly calling the protected publication + workflow; do not rely on workflow-token release events to trigger it. A + protected live-smoke job must check out that exact commit and succeed before + the protected PyPI job can become eligible. - Scheduled/default-branch live smoke is monitoring evidence only and cannot satisfy the exact-release live gate. - Missing identity, credentials, environments, reviewers, protection, @@ -260,12 +264,16 @@ committed. accepted. The sole approved recovery tag is `v0.1.0-alpha.1+recovery.1`, which maps to package version `0.1.0a1`. Later releases must use their ordinary canonical tag spelling. -- Keep Release Please disabled after the recovery alpha. Its manifest cannot - infer the previous-release boundary from the recovery tag's build metadata; - enabling it requires a separate reviewed and tested `last-release-sha` - bridge. -- Keep third-party Actions pinned to full commit SHAs and grant - `id-token: write` only to the publishing job. +- Keep Release Please disabled outside an explicitly authorized release + sequence. Its stable-readiness configuration uses the reviewed and tested + `last-release-sha` bridge because the recovery tag's build metadata cannot be + inferred from the manifest. Remove the one-time bridge and prerelease + versioning controls in the human-finalized stable release PR before it is + merged. +- Keep third-party Actions pinned to full commit SHAs. Grant `id-token: write` + only to the reusable publication caller and the protected publishing job; + the caller passes this maximum permission and only the publishing job uses + the OIDC token. - Keep README, roadmap, compatibility matrix, examples, and changelog aligned with shipped behavior. Use currently supported model IDs. - All repository documentation is written in English. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 130b718..bf163b0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -114,6 +114,7 @@ Release evidence is intentionally ordered: ```text local mocked/package evidence -> immutable tag commit equals checkout and belongs to protected default branch + -> release API and tag ref confirm the exact immutable identity -> protected live-smoke job checks that exact commit -> protected PyPI OIDC job publishes the previously verified artifact -> public registry digest, provenance, install, import, and mocked smoke @@ -139,10 +140,10 @@ recovery release uses SemVer build metadata in `v0.1.0-alpha.1+recovery.1`. The build suffix does not change the package artifact identity: the PyPI version remains `0.1.0a1`. -Release Please remains disabled after this recovery release because its -manifest version does not include the recovery build metadata. A later, -separately reviewed change must establish the previous-release boundary with a -tested `last-release-sha` bridge before enabling automated release PRs. +Release Please remains disabled outside an explicitly authorized release +sequence. The stable-readiness configuration establishes the recovery commit +as the previous-release boundary with a tested `last-release-sha` bridge, so +the one-time build-metadata recovery tag cannot replay earlier history. ## Rejected 0.1 approaches diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98757d5..a8afc7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ uv run ruff check src tests scripts uv run ruff format --check src tests scripts uv run pyright uv run pytest -m "not live" -uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog +uv run python scripts/check_version.py --require-changelog uv run python scripts/check_secrets.py uv run python scripts/check_workflows.py rm -rf dist diff --git a/README.md b/README.md index 4ccb62c..c51f2bd 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ variable, then the default: ### Chat Completions + ```python from cometapi import CometAPI @@ -78,6 +79,7 @@ with CometAPI() as client: Streaming uses the official OpenAI stream type: + ```python from cometapi import CometAPI @@ -93,6 +95,7 @@ with CometAPI() as client: ### Responses and Models + ```python from cometapi import CometAPI @@ -109,6 +112,7 @@ print(models.data[0].id if models.data else "No models returned") ### Async client + ```python import asyncio @@ -171,7 +175,7 @@ uv run ruff check src tests scripts uv run ruff format --check src tests scripts uv run pyright uv run pytest -m "not live" -uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog +uv run python scripts/check_version.py --require-changelog uv run python scripts/check_secrets.py uv run python scripts/check_workflows.py rm -rf dist diff --git a/RELEASING.md b/RELEASING.md index 0df7928..74d7d42 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -45,10 +45,11 @@ one active maintainer. Before the historical first push, scheduled and manually dispatched live execution was required to fail closed unless `LIVE_SMOKE_ENABLED=true`. -`RELEASE_PLEASE_ENABLED` was kept disabled and remains disabled until a -separately reviewed and tested `last-release-sha` bridge establishes the -recovery alpha as the previous-release boundary. An unset or non-true value -prevents the corresponding gated job from executing. +`RELEASE_PLEASE_ENABLED` was kept disabled. The reviewed stable-readiness +configuration now uses an explicit `last-release-sha` bridge to establish the +recovery alpha as the previous-release boundary; the repository variable stays +disabled until maintainers intentionally start the stable release sequence. An +unset or non-true value prevents the corresponding gated job from executing. The release live-model configuration resolves an unset or empty `COMETAPI_LIVE_MODEL` to `gpt-5.4`. @@ -78,7 +79,7 @@ uv run ruff check src tests scripts uv run ruff format --check src tests scripts uv run pyright uv run pytest -m "not live" -uv run python scripts/check_version.py --expected 0.1.0a1 --require-changelog +uv run python scripts/check_version.py --require-changelog uv run python scripts/check_secrets.py uv run python scripts/check_workflows.py rm -rf dist @@ -181,14 +182,18 @@ violations in one run and still returns non-zero when any violation exists. `LIVE_SMOKE_ENABLED=true`. - `release-please.yml` maintains a human-reviewed version and changelog pull request from Conventional Commits after maintainers enable the - `RELEASE_PLEASE_ENABLED` repository variable. Keep it disabled after the - initial `v0.1.0-alpha.1+recovery.1` release: the checked-in manifest version - lacks the recovery tag's build metadata and cannot safely infer the previous - release boundary. Enable it only after a separate reviewed change configures - and tests an explicit `last-release-sha` bridge. -- `publish.yml` runs only for a published immutable GitHub release. It resolves - the tag to the checked-out commit, fetches the protected default branch, and - rejects a commit that is not reachable from that branch. A protected + `RELEASE_PLEASE_ENABLED` repository variable. The checked-in stable-readiness + configuration establishes the recovery release boundary with the reviewed + `last-release-sha` bridge. Keep the variable disabled except while executing + an explicitly authorized release sequence. When it creates an approved + release with the GitHub workflow token, it polls the GitHub API until that + exact tag and commit are independently reported as immutable, then invokes + the protected publication chain directly; workflow-token release events do + not trigger a second workflow run. +- `publish.yml` is called only with the independently verified immutable tag, + commit, and default branch. It resolves the tag to the checked-out commit, + fetches the protected default branch, and rejects a commit that is not + reachable from that branch. A protected `live-smoke` job then checks out that exact verified commit and must succeed before the protected `pypi` job becomes eligible. The workflow publishes the previously verified artifacts with OIDC, then checks the public package @@ -197,7 +202,9 @@ violations in one run and still returns non-zero when any violation exists. or empty live-model repository variable resolves to `gpt-5.4`. Third-party Actions are pinned to full commit SHAs. Workflow permissions are -read-only by default; only the publishing job receives `id-token: write`. +read-only by default. The reusable publication caller and protected publishing +job declare `id-token: write`; the caller passes the maximum permission and +only the publishing job requests the OIDC token. Publishing uses a protected `pypi` environment and concurrency control. Arbitrary-branch and manual publication are forbidden. @@ -272,8 +279,9 @@ changelog, GitHub release, wheel, and source distribution. SHA256 `a6820347317943ca22f7632acbe354dd992f31a122a6172dfe45b57960e3a093` and source-distribution SHA256 `98d86829ef14771e8b7ec180d452c6638289f49c14a39b7207be5c47cb64cde7`. -- `LIVE_SMOKE_ENABLED=false`. Release Please remains disabled until a separate - reviewed and tested `last-release-sha` bridge is merged. +- `LIVE_SMOKE_ENABLED=false`. Release Please remains disabled outside an + explicitly authorized release sequence; the reviewed `last-release-sha` + bridge is configured for the alpha-to-stable transition. ## Stable release sequence @@ -282,8 +290,11 @@ feature or fix pull request -> required offline CI -> merge to the default branch -> automated release pull request - -> human review and merge + -> human finalization of stable docs, metadata, and one-time bridge cleanup + -> required release-PR CI, review, and merge -> immutable tag and GitHub release + -> bounded API verification of immutable tag and commit identity + -> direct call to the protected publication workflow -> verify immutable tag commit and protected-default-branch ancestry -> rebuild and verify exact artifacts -> protected live smoke against that exact commit @@ -295,4 +306,11 @@ feature or fix pull request Stable `0.1.0` additionally requires the complete blocking Python matrix, executed README examples against the built package, trusted live evidence, and -reviewed release-PR and changelog agreement. +reviewed release-PR and changelog agreement. Before the stable release PR is +merged, its finalization commit must state that `0.1.0` is approved for PyPI +publication, use the stable installation command and classifier, and remove the +one-time `last-release-sha` plus prerelease-versioning controls. The manifest, +project metadata, lock file, and changelog must remain at the exact generated +`0.1.0` version. If GitHub requires approval before checks run on the automated +pull request, approve only that reviewed workflow execution and wait for every +blocking check. diff --git a/ROADMAP.md b/ROADMAP.md index 4d540bf..7a23a71 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -416,8 +416,10 @@ Accepted release evidence: `a6820347317943ca22f7632acbe354dd992f31a122a6172dfe45b57960e3a093` and its source-distribution SHA256 is `98d86829ef14771e8b7ec180d452c6638289f49c14a39b7207be5c47cb64cde7`. -- `LIVE_SMOKE_ENABLED=false`. Release Please remains disabled pending a - separately reviewed and tested `last-release-sha` bridge. +- `LIVE_SMOKE_ENABLED=false`. Release Please remains disabled outside an + explicitly authorized release sequence; the reviewed stable-readiness + configuration now establishes the recovery alpha boundary with + `last-release-sha`. ## `0.1.0`: OpenAI protocol foundation @@ -425,6 +427,8 @@ Stable 0.1 retains the alpha surface. Its additional exit criteria are: - Blocking Python runtime matrix for every supported runtime. - Human-reviewed release PR with exact version and changelog agreement. +- Stable documentation, classifier, and installation guidance finalized in the + release PR, with the one-time recovery bridge removed before merge. - Executed README examples against the built artifact. - Trusted live Chat Completions and Responses smoke evidence. - Immutable tag, GitHub release, wheel, source distribution, and changelog @@ -461,10 +465,13 @@ The repository maintains four independently auditable workflows: - `live-smoke.yml`: scheduled and manual default-branch monitoring capped at four requests, 16 output tokens per generation, a 30-second request timeout, concurrency one, a ten-minute workflow timeout, and stop on first failure. -- `release-please.yml`: a human-reviewed version and changelog pull request. -- `publish.yml`: immutable-release and default-branch ancestry verification, - exact-release protected live smoke, artifact rebuild and verification, - protected PyPI OIDC publication, provenance, and registry verification. +- `release-please.yml`: a human-reviewed version and changelog pull request, + followed by bounded API verification of the exact immutable release and a + direct call into the protected publication chain. +- `publish.yml`: reusable immutable-tag, commit, and default-branch ancestry + verification, exact-release protected live smoke, artifact rebuild and + verification, protected PyPI OIDC publication, provenance, and registry + verification. All workflow files must pass local `actionlint` 1.7.12. This is static validation only. Remote behavior remains unverified until each workflow runs @@ -472,10 +479,10 @@ successfully in the canonical GitHub repository. Scheduled and manually dispatched live smoke must require `LIVE_SMOKE_ENABLED=true`; an unset or other value prevents live execution. -Release Please requires -`RELEASE_PLEASE_ENABLED=true` and remains disabled after the recovery alpha -until a separate reviewed and tested `last-release-sha` bridge establishes its -previous-release boundary. Release jobs must resolve an unset or empty +Release Please requires `RELEASE_PLEASE_ENABLED=true` and remains disabled +outside an explicitly authorized release sequence. Its stable-readiness +configuration uses the reviewed `last-release-sha` bridge for the recovery +alpha boundary. Release jobs must resolve an unset or empty `COMETAPI_LIVE_MODEL` to `gpt-5.4` rather than attempt a request with an empty model. diff --git a/release-please-config.json b/release-please-config.json index ee65bbd..dbebe5d 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -3,12 +3,19 @@ "release-type": "python", "include-component-in-tag": false, "include-v-in-tag": true, - "prerelease": true, + "last-release-sha": "31b68904141489ca04932edbf305ccf88af09372", + "prerelease": false, "versioning": "prerelease", "packages": { ".": { "package-name": "cometapi", - "prerelease-type": "alpha" + "extra-files": [ + { + "type": "toml", + "path": "uv.lock", + "jsonpath": "$.package[?(@.name.value == 'cometapi')].version" + } + ] } } } diff --git a/scripts/check_artifacts.py b/scripts/check_artifacts.py index 03b60c6..23624b6 100644 --- a/scripts/check_artifacts.py +++ b/scripts/check_artifacts.py @@ -58,9 +58,11 @@ "tests/live/__init__.py", "tests/live/test_live_smoke.py", "tests/test_client.py", + "tests/test_clean_install.py", "tests/test_contract.py", "tests/test_release_documents.py", "tests/test_release_workflow.py", + "tests/test_secrets.py", "tests/typing/constructor_contract.py", } OPTIONAL_SDIST_FILES = {".gitignore"} diff --git a/scripts/check_clean_install.py b/scripts/check_clean_install.py index cedd190..b8b52c6 100644 --- a/scripts/check_clean_install.py +++ b/scripts/check_clean_install.py @@ -1,17 +1,60 @@ #!/usr/bin/env python3 -"""Install exact artifacts or a registry requirement and run mocked-call smoke tests.""" +"""Install exact artifacts and run package plus documented-example smoke tests.""" from __future__ import annotations import argparse +import json import os +import re import subprocess import sys import tempfile +import threading import time +from collections.abc import Generator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from typing import cast +from urllib.parse import urlsplit -from _checks import CheckError, normalize_version, read_project_version +try: + from scripts._checks import PROJECT_ROOT, CheckError, normalize_version, read_project_version +except ModuleNotFoundError: # Support direct execution from the repository root. + from _checks import PROJECT_ROOT, CheckError, normalize_version, read_project_version + +README_EXAMPLE_NAMES = ( + "sync-chat", + "sync-chat-stream", + "sync-responses-models", + "async-response", +) +README_EXAMPLE_MARKER = re.compile(r"") +README_EXAMPLE_MARKER_PREFIX = "", + "", + ) + + with pytest.raises(CheckError, match="unknown README example marker"): + read_readme_examples(readme) + + +def test_readme_examples_reject_duplicate_markers(tmp_path: Path) -> None: + readme = _mutated_readme( + tmp_path, + "", + "", + ) + + with pytest.raises(CheckError, match="duplicate README example marker"): + read_readme_examples(readme) + + +def test_readme_examples_reject_reordered_markers(tmp_path: Path) -> None: + readme = _mutated_readme( + tmp_path, + "", + "", + ) + + with pytest.raises(CheckError, match="out of order"): + read_readme_examples(readme) + + +def test_readme_examples_require_an_adjacent_python_block(tmp_path: Path) -> None: + readme = _mutated_readme( + tmp_path, + "\n```python", + "\n\n```python", + ) + + with pytest.raises(CheckError, match="followed immediately by a Python code block"): + read_readme_examples(readme) + + +def test_readme_examples_require_a_python_fence(tmp_path: Path) -> None: + readme = _mutated_readme( + tmp_path, + "\n```python", + "\n```py", + ) + + with pytest.raises(CheckError, match="followed immediately by a Python code block"): + read_readme_examples(readme) + + +def test_readme_examples_reject_unterminated_fence(tmp_path: Path) -> None: + examples = read_readme_examples() + source = "".join( + f"\n```python\n{code}" + + ("```\n" if name != README_EXAMPLE_NAMES[-1] else "") + for name, code in examples + ) + readme = tmp_path / "README.md" + readme.write_text(source, encoding="utf-8") + + with pytest.raises(CheckError, match="is unterminated"): + read_readme_examples(readme) + + +def test_readme_examples_reject_invalid_python(tmp_path: Path) -> None: + readme = _mutated_readme( + tmp_path, + "from cometapi import CometAPI\n\nwith CometAPI() as client:", + "from cometapi import CometAPI\n\nthis is not valid Python\nwith CometAPI() as client:", + ) + + with pytest.raises(CheckError, match="is not valid Python"): + read_readme_examples(readme) + + +def test_readme_examples_do_not_select_unmarked_direct_openai_block() -> None: + examples = read_readme_examples() + + assert all("from openai import OpenAI" not in code for _name, code in examples) + + +def test_readme_example_server_records_sse_and_rejects_extra_routes() -> None: + with readme_example_server() as (server, base_url): + environment = os.environ.copy() + environment.update( + { + "COMETAPI_BASE_URL": base_url, + "COMETAPI_KEY": "readme-example-key", + "NO_PROXY": "127.0.0.1,localhost", + } + ) + source = """ +import json +import os +import urllib.request + +request = urllib.request.Request( + os.environ["COMETAPI_BASE_URL"] + "/chat/completions", + data=json.dumps({"model": "gpt-5.4", "messages": [], "stream": True}).encode(), + headers={"Authorization": "Bearer readme-example-key", "Content-Type": "application/json"}, +) +with urllib.request.urlopen(request) as response: + payload = response.read().decode() +assert response.headers["Content-Type"] == "text/event-stream" +assert "data: [DONE]" in payload +""" + subprocess.run( + [sys.executable, "-I", "-c", README_EXAMPLE_BOOTSTRAP, "fixture-sse", source], + env=environment, + check=True, + timeout=10, + ) + import urllib.error + import urllib.request + + request = urllib.request.Request( + base_url + "/models?unexpected=1", + headers={"Authorization": "Bearer readme-example-key"}, + ) + with pytest.raises(urllib.error.HTTPError) as error: + urllib.request.urlopen(request, timeout=5) + assert error.value.code == 500 + + assert server.requests == [ + ( + "POST", + "/v1/chat/completions", + {"model": "gpt-5.4", "messages": [], "stream": True}, + ) + ] + assert server.errors == ["README example requests must not include a query string"] + + +def test_readme_example_bootstrap_rejects_non_loopback_network() -> None: + source = """ +import socket + +socket.create_connection(("192.0.2.1", 80), timeout=0.1) +""" + result = subprocess.run( + [sys.executable, "-I", "-c", README_EXAMPLE_BOOTSTRAP, "network", source], + capture_output=True, + text=True, + timeout=10, + ) + + assert result.returncode != 0 + assert "README examples may connect only to the loopback fixture" in result.stderr + + +def test_fixture_payloads_are_json_serializable() -> None: + from scripts.check_clean_install import CHAT_CHUNK, RESPONSE + + assert json.loads(json.dumps(CHAT_CHUNK))["object"] == "chat.completion.chunk" + assert json.loads(json.dumps(RESPONSE))["object"] == "response" diff --git a/tests/test_client.py b/tests/test_client.py index 71eb7cb..eb3cdf6 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -11,6 +11,7 @@ import cometapi from cometapi import AsyncCometAPI, CometAPI from cometapi._config import DEFAULT_BASE_URL +from scripts._checks import read_project_version from .conftest import ( API_KEY, @@ -22,7 +23,7 @@ def test_public_api_exports_only_accepted_client_names() -> None: - assert cometapi.__version__ == "0.1.0a1" + assert cometapi.__version__ == read_project_version() assert cometapi.__all__ == ["AsyncCometAPI", "CometAPI", "__version__"] assert cometapi.CometAPI is CometAPI assert cometapi.AsyncCometAPI is AsyncCometAPI diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index a1147c1..8776894 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -13,6 +13,7 @@ check_action_pins, check_ci_workflow, check_publish_workflow, + check_release_please_config, check_release_please_workflow, check_workflow_inventory, workflow_paths, @@ -24,6 +25,8 @@ PUBLISH_WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "publish.yml" LIVE_SMOKE_WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "live-smoke.yml" RELEASE_PLEASE_WORKFLOW = PROJECT_ROOT / ".github" / "workflows" / "release-please.yml" +RELEASE_PLEASE_CONFIG = PROJECT_ROOT / "release-please-config.json" +RELEASE_PLEASE_MANIFEST = PROJECT_ROOT / ".release-please-manifest.json" TRUST_SCRIPT = PROJECT_ROOT / "scripts" / "verify_release_trust.sh" @@ -66,12 +69,15 @@ def _verify_trust( *, immutable: str = "true", tag: str = "v0.1.0-alpha.1", + expected_release_sha: str | None = None, ) -> subprocess.CompletedProcess[str]: output = tmp_path / "github-output.txt" environment = os.environ.copy() environment.update( { "DEFAULT_BRANCH": "main", + "EXPECTED_RELEASE_SHA": expected_release_sha + or _git(repository, "rev-parse", f"{tag}^{{commit}}"), "GITHUB_OUTPUT": str(output), "RELEASE_IMMUTABLE": immutable, "RELEASE_TAG": tag, @@ -95,8 +101,8 @@ def _step(text: str, name: str) -> tuple[int, int]: def _bypass_immutable_event(text: str) -> str: return text.replace( - "RELEASE_IMMUTABLE: ${{ github.event.release.immutable }}", 'RELEASE_IMMUTABLE: "true"', + 'RELEASE_IMMUTABLE: "false"', 1, ) @@ -740,6 +746,96 @@ def test_current_release_please_workflow_is_disabled_by_default() -> None: check_release_please_workflow(RELEASE_PLEASE_WORKFLOW.read_text(encoding="utf-8")) +def test_current_release_please_config_has_reviewed_stable_bridge() -> None: + check_release_please_config( + RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8"), + RELEASE_PLEASE_MANIFEST.read_text(encoding="utf-8"), + ) + + +@pytest.mark.parametrize( + ("needle", "replacement", "message"), + [ + ( + '"31b68904141489ca04932edbf305ccf88af09372"', + '"f39b4dc9f2e18e91ab3cbac202246f85658f71fd"', + "recovery alpha commit", + ), + ('"prerelease": false', '"prerelease": true', "prerelease-to-stable"), + ('"versioning": "prerelease"', '"versioning": "default"', "prerelease-to-stable"), + ('"path": "uv.lock"', '"path": "pyproject.toml"', "uv.lock"), + ( + "$.package[?(@.name.value == 'cometapi')].version", + "$.package[0].version", + "uv.lock", + ), + ], + ids=["wrong-boundary", "prerelease", "versioning", "wrong-path", "wrong-jsonpath"], +) +def test_release_please_config_rejects_bridge_drift( + needle: str, replacement: str, message: str +) -> None: + text = RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8") + assert needle in text + with pytest.raises(RuntimeError, match=message): + check_release_please_config( + text.replace(needle, replacement, 1), + RELEASE_PLEASE_MANIFEST.read_text(encoding="utf-8"), + ) + + +def test_release_please_config_rejects_alpha_type_and_extra_updaters() -> None: + text = RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8").replace( + '"package-name": "cometapi",', + '"package-name": "cometapi",\n "prerelease-type": "alpha",', + 1, + ) + with pytest.raises(RuntimeError, match="reviewed contract"): + check_release_please_config( + text, + RELEASE_PLEASE_MANIFEST.read_text(encoding="utf-8"), + ) + + +def test_release_please_config_rejects_manifest_drift() -> None: + with pytest.raises(RuntimeError, match="reviewed bridge or stable version"): + check_release_please_config( + RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8"), + '{".": "0.1.0-alpha.2"}\n', + ) + + +def test_release_please_config_accepts_exact_stable_cleanup() -> None: + config = RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8") + for line in ( + ' "last-release-sha": "31b68904141489ca04932edbf305ccf88af09372",\n', + ' "prerelease": false,\n', + ' "versioning": "prerelease",\n', + ): + config = config.replace(line, "", 1) + check_release_please_config(config, '{".": "0.1.0"}\n') + + +def test_release_please_config_rejects_stable_manifest_with_bridge() -> None: + with pytest.raises(RuntimeError, match="remove the one-time bridge"): + check_release_please_config( + RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8"), + '{".": "0.1.0"}\n', + ) + + +def test_release_please_config_rejects_bridge_cleanup_before_stable() -> None: + config = RELEASE_PLEASE_CONFIG.read_text(encoding="utf-8") + for line in ( + ' "last-release-sha": "31b68904141489ca04932edbf305ccf88af09372",\n', + ' "prerelease": false,\n', + ' "versioning": "prerelease",\n', + ): + config = config.replace(line, "", 1) + with pytest.raises(RuntimeError, match="only after stable"): + check_release_please_config(config, '{".": "0.1.0-alpha.1"}\n') + + @pytest.mark.parametrize( "replacement", [ @@ -1219,7 +1315,10 @@ def test_secret_scope_scan_includes_yaml_workflows(tmp_path: Path) -> None: capture_output=True, ) assert result.returncode != 0 - assert ".github/workflows/rogue.yaml: id-token: write is publish-job-only" in result.stderr + assert ( + ".github/workflows/rogue.yaml: id-token: write must match the reviewed " + "publication chain count (0)" + ) in result.stderr def test_semantic_contract_rejects_download_before_checkout() -> None: @@ -1275,6 +1374,16 @@ def test_release_trust_rejects_non_immutable_release( assert "immutable=true" in result.stderr +def test_release_trust_rejects_release_sha_that_differs_from_tag( + release_repository: tuple[Path, str], tmp_path: Path +) -> None: + repository, _release_commit = release_repository + result = _verify_trust(repository, tmp_path, expected_release_sha="0" * 40) + + assert result.returncode != 0 + assert "expected 0000000000000000000000000000000000000000" in result.stderr + + def test_release_trust_rejects_checkout_that_differs_from_tag( release_repository: tuple[Path, str], tmp_path: Path ) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..516964b --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.check_secrets import scan_workflow_scope + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def test_current_workflows_retain_reviewed_oidc_scope() -> None: + assert scan_workflow_scope(PROJECT_ROOT) == [] + + +def test_scope_scan_rejects_oidc_on_an_unreviewed_workflow(tmp_path: Path) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + for name in ("ci.yml", "publish.yml", "release-please.yml"): + (workflows / name).write_text( + (PROJECT_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8"), + encoding="utf-8", + ) + (workflows / "live-smoke.yml").write_text( + "permissions:\n id-token: write\n", + encoding="utf-8", + ) + + assert scan_workflow_scope(tmp_path) == [ + ".github/workflows/live-smoke.yml: id-token: write must match the reviewed " + "publication chain count (0)" + ] + + +def test_scope_scan_rejects_missing_reusable_caller_oidc(tmp_path: Path) -> None: + workflows = tmp_path / ".github" / "workflows" + workflows.mkdir(parents=True) + for name in ("ci.yml", "publish.yml", "release-please.yml"): + text = (PROJECT_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") + if name == "release-please.yml": + text = text.replace(" id-token: write\n", "", 1) + (workflows / name).write_text(text, encoding="utf-8") + + assert scan_workflow_scope(tmp_path) == [ + ".github/workflows/release-please.yml: id-token: write must match the reviewed " + "publication chain count (1)" + ]