diff --git a/.agents/skills/frontal-sdk b/.agents/skills/frontal-sdk new file mode 120000 index 0000000..7d40552 --- /dev/null +++ b/.agents/skills/frontal-sdk @@ -0,0 +1 @@ +../../skills/frontal-sdk \ No newline at end of file diff --git a/.agents/skills/python-lint b/.agents/skills/python-lint new file mode 120000 index 0000000..260f8af --- /dev/null +++ b/.agents/skills/python-lint @@ -0,0 +1 @@ +../../skills/python-lint \ No newline at end of file diff --git a/.agents/skills/python-packaging b/.agents/skills/python-packaging new file mode 120000 index 0000000..d01377b --- /dev/null +++ b/.agents/skills/python-packaging @@ -0,0 +1 @@ +../../skills/python-packaging \ No newline at end of file diff --git a/.agents/skills/python-pydantic-httpx-contracts b/.agents/skills/python-pydantic-httpx-contracts new file mode 120000 index 0000000..422f761 --- /dev/null +++ b/.agents/skills/python-pydantic-httpx-contracts @@ -0,0 +1 @@ +../../skills/python-pydantic-httpx-contracts \ No newline at end of file diff --git a/.agents/skills/python-supply-chain b/.agents/skills/python-supply-chain new file mode 120000 index 0000000..f949d83 --- /dev/null +++ b/.agents/skills/python-supply-chain @@ -0,0 +1 @@ +../../skills/python-supply-chain \ No newline at end of file diff --git a/.agents/skills/python-sync-async-clients b/.agents/skills/python-sync-async-clients new file mode 120000 index 0000000..1242222 --- /dev/null +++ b/.agents/skills/python-sync-async-clients @@ -0,0 +1 @@ +../../skills/python-sync-async-clients \ No newline at end of file diff --git a/.agents/skills/python-testing b/.agents/skills/python-testing new file mode 120000 index 0000000..dbdf983 --- /dev/null +++ b/.agents/skills/python-testing @@ -0,0 +1 @@ +../../skills/python-testing \ No newline at end of file diff --git a/.agents/skills/python-typing b/.agents/skills/python-typing new file mode 120000 index 0000000..9d7924b --- /dev/null +++ b/.agents/skills/python-typing @@ -0,0 +1 @@ +../../skills/python-typing \ No newline at end of file diff --git a/.claude/rules/sdk-language.md b/.claude/rules/sdk-language.md new file mode 100644 index 0000000..a73bc07 --- /dev/null +++ b/.claude/rules/sdk-language.md @@ -0,0 +1,10 @@ +--- +paths: + - "**/*.py" +--- + +# Python changes + +Support Python 3.9–3.12. Keep the import package frontal_sdk separate from the PyPI distribution name frontal. Use HTTPX, Pydantic v2, uv/Hatchling, Ruff, mypy, pytest, anyio, and offline RESPX tests. Treat contracts as endpoint authority. + +Follow the repository's `AGENTS.md` and `CLAUDE.md`. Use the relevant installed language skill under `.claude/skills/`. Add deterministic, offline tests for behavior changes, and run the repository's relevant format, lint, type/build, and contract checks. diff --git a/.claude/settings.json b/.claude/settings.json index 8b311a3..63b5817 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,5 +1,12 @@ { + "$schema": "https://json.schemastore.org/claude-code-settings.json", "permissions": { - "allow": [] + "allow": [], + "deny": [ + "Read(./.env)", + "Read(./.env.*)", + "Read(**/.env)", + "Read(**/.env.*)" + ] } } diff --git a/.claude/skills/frontal-sdk b/.claude/skills/frontal-sdk new file mode 120000 index 0000000..7d40552 --- /dev/null +++ b/.claude/skills/frontal-sdk @@ -0,0 +1 @@ +../../skills/frontal-sdk \ No newline at end of file diff --git a/.claude/skills/python-lint b/.claude/skills/python-lint new file mode 120000 index 0000000..260f8af --- /dev/null +++ b/.claude/skills/python-lint @@ -0,0 +1 @@ +../../skills/python-lint \ No newline at end of file diff --git a/.claude/skills/python-packaging b/.claude/skills/python-packaging new file mode 120000 index 0000000..d01377b --- /dev/null +++ b/.claude/skills/python-packaging @@ -0,0 +1 @@ +../../skills/python-packaging \ No newline at end of file diff --git a/.claude/skills/python-pydantic-httpx-contracts b/.claude/skills/python-pydantic-httpx-contracts new file mode 120000 index 0000000..422f761 --- /dev/null +++ b/.claude/skills/python-pydantic-httpx-contracts @@ -0,0 +1 @@ +../../skills/python-pydantic-httpx-contracts \ No newline at end of file diff --git a/.claude/skills/python-supply-chain b/.claude/skills/python-supply-chain new file mode 120000 index 0000000..f949d83 --- /dev/null +++ b/.claude/skills/python-supply-chain @@ -0,0 +1 @@ +../../skills/python-supply-chain \ No newline at end of file diff --git a/.claude/skills/python-sync-async-clients b/.claude/skills/python-sync-async-clients new file mode 120000 index 0000000..1242222 --- /dev/null +++ b/.claude/skills/python-sync-async-clients @@ -0,0 +1 @@ +../../skills/python-sync-async-clients \ No newline at end of file diff --git a/.claude/skills/python-testing b/.claude/skills/python-testing new file mode 120000 index 0000000..dbdf983 --- /dev/null +++ b/.claude/skills/python-testing @@ -0,0 +1 @@ +../../skills/python-testing \ No newline at end of file diff --git a/.claude/skills/python-typing b/.claude/skills/python-typing new file mode 120000 index 0000000..9d7924b --- /dev/null +++ b/.claude/skills/python-typing @@ -0,0 +1 @@ +../../skills/python-typing \ No newline at end of file diff --git a/.cursor.json b/.cursor.json deleted file mode 100644 index 666a82e..0000000 --- a/.cursor.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "Frontal Python SDK", - "description": "Python SDK workspace. Follow AGENTS.md, use the native toolchain, and check contracts/ before adding service operations." -} diff --git a/.cursor/README.md b/.cursor/README.md new file mode 100644 index 0000000..179c3c3 --- /dev/null +++ b/.cursor/README.md @@ -0,0 +1,5 @@ +# Cursor project configuration + +This directory uses Cursor Project Rules in `.cursor/rules/*.mdc` and project commands in `.cursor/commands/*.md`. The shared rule applies to every task; the language rule is auto-attached to `**/*.py` files. The commands provide reusable review and verification workflows. root `.cursorignore` excludes generated output, dependency caches, and local secrets from Cursor context. + +Cursor Agent also reads root `AGENTS.md` and `CLAUDE.md`; Claude Code reads `CLAUDE.md`, which imports `AGENTS.md`. Project instruction sources are therefore shared without relying on the obsolete `.cursor.json` format. diff --git a/.cursor/cli.json b/.cursor/cli.json new file mode 100644 index 0000000..733f5ac --- /dev/null +++ b/.cursor/cli.json @@ -0,0 +1,10 @@ +{ + "permissions": { + "deny": [ + "Read(.env*)", + "Read(**/.env*)", + "Write(.env*)", + "Write(**/.env*)" + ] + } +} diff --git a/.cursor/commands/review-sdk-contract.md b/.cursor/commands/review-sdk-contract.md new file mode 100644 index 0000000..4d3a5a5 --- /dev/null +++ b/.cursor/commands/review-sdk-contract.md @@ -0,0 +1 @@ +Review the current diff for SDK contract correctness. Compare any changed endpoint paths, methods, request/response fields, and error behavior with `contracts/sdk-endpoints.json` and the corresponding committed OpenAPI snapshot. Flag unsupported API surface, accidental breaking changes, credential exposure, and missing offline coverage. Do not modify files. Report findings with file and line references; if none, say so. diff --git a/.cursor/commands/verify-sdk-change.md b/.cursor/commands/verify-sdk-change.md new file mode 100644 index 0000000..f5b11df --- /dev/null +++ b/.cursor/commands/verify-sdk-change.md @@ -0,0 +1,10 @@ +Review the current diff and identify affected modules, public behavior, and relevant contract entries. Run only checks relevant to the changes, choosing from the repository commands below. Do not edit code unless I explicitly ask you to fix a failure. Report checks run and their results. + +```sh +uv run ruff format --check . +uv build +uv run ruff check . +uv run mypy --strict +uv run python -m pytest +uv run python scripts/check_contracts.py +``` diff --git a/.cursor/rules/00-project.mdc b/.cursor/rules/00-project.mdc new file mode 100644 index 0000000..b23716c --- /dev/null +++ b/.cursor/rules/00-project.mdc @@ -0,0 +1,13 @@ +--- +description: Shared Frontal SDK repository constraints +alwaysApply: true +--- + +# Project guidance + +- Read `AGENTS.md` for repository layout, conventions, and canonical commands. `CLAUDE.md` imports it for Claude Code compatibility. +- Treat `contracts/sdk-endpoints.json` and committed OpenAPI snapshots as authoritative for endpoint paths and wire shapes. Do not invent public operations or fields. +- Preserve established public API and compatibility unless the requested change explicitly requires otherwise. +- Keep credentials, tokens, and customer data out of code, logs, fixtures, and examples. +- Follow relevant repository skills in `skills/` when changing SDK behavior. +- Inspect the existing diff before editing. Run only the checks relevant to the change and report exactly what was run. diff --git a/.cursor/rules/10-sdk-language.mdc b/.cursor/rules/10-sdk-language.mdc new file mode 100644 index 0000000..f744f46 --- /dev/null +++ b/.cursor/rules/10-sdk-language.mdc @@ -0,0 +1,11 @@ +--- +description: Python SDK code conventions and checks +globs: **/*.py +alwaysApply: false +--- + +# Python development rules + +Support Python 3.9–3.12. Keep the frontal_sdk import package distinct from the frontal distribution. Follow existing uv/Hatchling, HTTPX, Pydantic v2, Ruff, mypy, pytest, anyio, and RESPX patterns. Keep service behavior grounded in the committed contracts. + +When implementing behavior changes, add deterministic offline coverage using the repository's existing test tools. Update docs and runnable examples for public API changes. See `AGENTS.md` for project-specific rules and use the applicable skill under `skills/`. diff --git a/.cursorignore b/.cursorignore index e57f796..e4fd7c2 100644 --- a/.cursorignore +++ b/.cursorignore @@ -1,3 +1,4 @@ +# Cursor context exclusions .git/ .venv/ __pycache__/ @@ -7,3 +8,14 @@ __pycache__/ .pyright/ /build/ /dist/ + +# Local secrets and generated caches +.env +.env.* +!.env.example +!.env.*.example +.gradle/ +.venv/ +venv/ +__pycache__/ +*.egg-info/ diff --git a/.gitignore b/.gitignore index f43f343..8726692 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ # Operating systems and editors .DS_Store -.vscode/ *.swp *.swo diff --git a/.vscode/README.md b/.vscode/README.md new file mode 100644 index 0000000..b06a924 --- /dev/null +++ b/.vscode/README.md @@ -0,0 +1,7 @@ +# VS Code workspace setup + +This folder contains portable workspace settings, extension recommendations, and manually invoked tasks. Open the repository root as a folder in VS Code; no machine-specific paths are configured. Tasks do not run automatically. + +Python testing is configured for pytest in tests/. The formatter and import actions use Ruff. Select the project interpreter with “Python: Select Interpreter”; no local interpreter path is hard-coded. + +The repository's `.editorconfig` remains authoritative for whitespace and line endings. See `AGENTS.md` for the full development workflow. diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..6605ada --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.vscode-pylance", + "charliermarsh.ruff", + "editorconfig.editorconfig" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..c6d5eff --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,34 @@ +{ + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "files.encoding": "utf8", + "editor.renderWhitespace": "boundary", + "editor.minimap.enabled": false, + "editor.formatOnPaste": false, + "files.exclude": { + "**/.venv": true, + "**/__pycache__": true, + "**/*.egg-info": true, + "**/dist": true, + "**/build": true + }, + "search.exclude": { + "**/.git": true, + "**/coverage": true, + "**/test-results": true + }, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.ruff": "explicit", + "source.organizeImports.ruff": "explicit" + } + }, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": [ + "tests" + ], + "python.terminal.activateEnvironment": true +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..8f62026 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,119 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Python: Sync development environment", + "type": "process", + "command": "uv", + "args": [ + "sync", + "--extra", + "dev" + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Ruff format check", + "type": "process", + "command": "uv", + "args": [ + "run", + "ruff", + "format", + "--check", + "." + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Ruff lint", + "type": "process", + "command": "uv", + "args": [ + "run", + "ruff", + "check", + "." + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Strict mypy", + "type": "process", + "command": "uv", + "args": [ + "run", + "mypy", + "--strict" + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Tests", + "type": "process", + "command": "uv", + "args": [ + "run", + "python", + "-m", + "pytest" + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Build distributions", + "type": "process", + "command": "uv", + "args": [ + "build" + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + }, + { + "label": "Python: Check endpoint contracts", + "type": "process", + "command": "uv", + "args": [ + "run", + "python", + "scripts/check_contracts.py" + ], + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "shared", + "clear": false + } + } + ] +} diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 0000000..42de9e7 --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,12 @@ +{ + "code_actions_on_format": { + "source.fixAll.biome": true, + "source.organizeImports.biome": true + }, + "format_on_save": "on", + "formatter": { + "language_server": { + "name": "biome" + } + } +} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..be7fa04 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,28 @@ +@AGENTS.md + +# Python SDK repository instructions + +Read `AGENTS.md` first for repository layout, project conventions, and contract rules. Read `README.md` and the relevant design docs before changing public SDK behavior. + +## Contract and compatibility + +Support Python 3.9–3.12. Keep the import package frontal_sdk separate from the PyPI distribution name frontal. Use HTTPX, Pydantic v2, uv/Hatchling, Ruff, mypy, pytest, anyio, and offline RESPX tests. Treat contracts as endpoint authority. + +Do not add public endpoints, wire fields, or behavior unsupported by the committed contracts. Keep credentials, tokens, and customer data out of source, logs, fixtures, and examples. Make the smallest compatible change and update documentation/examples when public behavior changes. + +## Skills + +Use the matching skill in `.claude/skills/` when its topic applies. Skills are also linked from `.agents/skills/`; source files live in `skills/` and selected upstream sources are recorded in `skills/SOURCES.md`. + +## Verification commands + +Run only the commands relevant to the change; do not claim verification unless it was run. + +```sh +uv run ruff format --check . +uv build +uv run ruff check . +uv run mypy --strict +uv run python -m pytest +uv run python scripts/check_contracts.py +``` diff --git a/README.md b/README.md index 2c4f6fc..f572850 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ![Frontal Banner](./banner.png) +[![skills.sh](https://skills.sh/b/frontal-labs/sdk-python)](https://skills.sh/frontal-labs/sdk-python) + The `frontal` package provides synchronous and asynchronous Python clients for the Frontal API. Services are available on the client as typed attributes such as `client.ai`, `client.agents`, and `client.workflows`. @@ -87,6 +89,14 @@ The tests use RESPX at the HTTPX layer and do not need a running API or live credentials. See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md), [`docs/TESTING.md`](docs/TESTING.md), and [`docs/PUBLISHING.md`](docs/PUBLISHING.md). +## Agent skills + +Install this repository's Python-specific agent skills with the [skills CLI](https://skills.sh/docs/cli): + +```bash +npx skills add frontal-labs/sdk-python +``` + ## License Apache-2.0. See [`LICENSE.md`](LICENSE.md). diff --git a/skills.sh.json b/skills.sh.json new file mode 100644 index 0000000..20cf8ce --- /dev/null +++ b/skills.sh.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://skills.sh/schemas/skills.sh.schema.json", + "notGrouped": "bottom", + "groupings": [ + { + "title": "Python SDK Engineering", + "description": "Python client lifecycle, Pydantic models, and contract testing for Frontal.", + "skills": [ + "frontal-sdk", + "python-sync-async-clients", + "python-pydantic-httpx-contracts", + "python-testing", + "python-typing", + "python-packaging", + "python-lint", + "python-supply-chain" + ] + } + ] +} diff --git a/skills/SOURCES.md b/skills/SOURCES.md new file mode 100644 index 0000000..c7ccf70 --- /dev/null +++ b/skills/SOURCES.md @@ -0,0 +1,5 @@ +# Upstream skill sources + +paldom/python-skills (MIT), skills/python-testing, python-typing, python-packaging, python-lint, and python-supply-chain; imported from skills.sh. + +Local repository-specific skills in this directory were authored for the Frontal SDK and remain the source of truth for Frontal-specific architecture and contract workflow. diff --git a/skills/frontal-sdk/SKILL.md b/skills/frontal-sdk/SKILL.md index c7aedfe..fc03df9 100644 --- a/skills/frontal-sdk/SKILL.md +++ b/skills/frontal-sdk/SKILL.md @@ -1,16 +1,55 @@ --- name: frontal-sdk-python -description: Guidance for adding or reviewing integrations using the Frontal Python SDK. +description: Build Python integrations with Frontal or change, test, and document the Frontal Python SDK. --- # Frontal Python SDK -The package provides both `Frontal` and `AsyncFrontal`, a shared HTTPX transport, Pydantic v2 boundary models, and service methods for every operation in `contracts/sdk-endpoints.json`. Keep client and resource methods hand-written. Use the contract checker to validate endpoint coverage and drift; do not generate public methods. +Use this skill both for application code using Frontal and for changes inside this SDK. Confirm current call signatures and supported fields in `frontal_sdk/` and the repository docs. `contracts/sdk-endpoints.json` plus `contracts/openapi/` are authoritative for routes and wire behavior. -## Configuration +## Choose a client -Use `Frontal()` or `AsyncFrontal()` to read `FRONTAL_API_KEY`, `FRONTAL_API_URL`, `FRONTAL_ENV`, and `FRONTAL_DEBUG` from the process environment, or pass configuration directly. The SDK does not load `.env` files. See the root `.env.example` for local development conventions. +The `frontal` distribution exposes the `frontal_sdk` import package. Use `Frontal` for synchronous programs and `AsyncFrontal` for async programs. Both expose service attributes such as `client.ai`, `client.agents`, and `client.workflows`. -## Modules +```python +from frontal_sdk import Frontal -See [the architecture guide](../../docs/ARCHITECTURE.md) and [resource guide](../../docs/RESOURCES.md). +with Frontal() as client: + result = client.ai.generate_text( + {"model": "frontal-ai-fast", "prompt": "Say hello."} + ) +``` + +For async work, use `async with AsyncFrontal() as client` and await operations. Avoid mixing sync calls into an async request path. Pass configuration directly when it is already available; environment defaults are `FRONTAL_API_KEY`, `FRONTAL_API_URL`, `FRONTAL_ENV`, and `FRONTAL_DEBUG`. The SDK does not load `.env` files itself. + +## Models, errors, and long-running work + +- Shared HTTPX behavior belongs in `frontal_sdk/core`; Pydantic v2 boundary models belong in `frontal_sdk/models`; domain methods belong in `frontal_sdk/resources`. +- Preserve typed return models and validation. Do not add speculative fields or route methods; use the contract inventory and contract checker. +- Handle the SDK's typed API/network errors at the appropriate boundary. Keep request IDs when logging diagnostics, and never log credentials or customer payloads by default. +- Use the provided pagination, polling, and SSE helpers when they match the endpoint. Close sync and async clients with their respective context managers. +- Do not retry writes without a contract-supported idempotency strategy. + +## Change the SDK + +Keep client and resource methods hand-written. Do not generate public methods from OpenAPI. Add shared transport behavior to `frontal_sdk/core`, typed boundary models to `frontal_sdk/models`, and service methods to the matching `frontal_sdk/resources` module. Preserve Python 3.9–3.12 compatibility, inline typing, and `frontal_sdk/py.typed`. Follow the configured HTTPX, Pydantic v2, Ruff, mypy, pytest, and anyio versions in `pyproject.toml` and `uv.lock`. + +For endpoint changes, consult contracts, implement the method, and use `scripts/check_contracts.py` to catch coverage or drift. Add offline tests with RESPX at the HTTPX layer, including sync/async behavior as appropriate. Add examples only for working, implemented behavior; README Python fences are executed in CI. + +## Tests and quality + +Keep tests independent of live Frontal services and secrets. Use pytest and RESPX; use anyio patterns for async coverage. Cover request serialization, response validation, errors, pagination, resource cleanup, and contract alignment for the behavior changed. + +Useful checks from the root: + +```bash +uv sync --extra dev +uv run ruff format --check . +uv build +uv run ruff check . +uv run mypy --strict +uv run python -m pytest +uv run python scripts/check_contracts.py +``` + +Also review `AGENTS.md`, `docs/ARCHITECTURE.md`, `docs/RESOURCES.md`, and the focused guide under `docs/` before making structural changes. diff --git a/skills/python-lint/SKILL.md b/skills/python-lint/SKILL.md new file mode 100644 index 0000000..aa2c7de --- /dev/null +++ b/skills/python-lint/SKILL.md @@ -0,0 +1,251 @@ +--- +name: python-lint +description: Sets up and tunes Ruff linting and formatting for a Python package — pyproject [tool.ruff] rule selection, Black/Flake8/isort migration — and fixes lint or format failures. Use when the user says 'set up linting', 'add ruff', 'fix lint errors', 'configure the formatter'. Not for type errors, non-Python file formatting, pre-commit wiring, or CI workflows. +license: MIT +--- + +# python-lint + +## Purpose + +Configure Ruff as the single linter and formatter for a Python package — replacing +Black, Flake8 (+ plugins), isort, and pyupgrade — with an explicit rule set in +`pyproject.toml`, then drive `ruff check` and `ruff format --check` to a clean exit. +Covers fresh setup, migration from the legacy stack, rule tuning, and fixing lint or +format failures. Every command below runs in the user's package repository. + +## When NOT to use + +- **Type errors / mypy / pyright / ty** — Ruff performs no type inference; `ANN` + rules only police annotation *style*. The python-typing skill, if installed, + owns this. +- **Pre-commit wiring** (`.pre-commit-config.yaml`, hook rev pinning, staged-file + runs) — the python-precommit skill. This skill only defines what Ruff should do. +- **CI workflow YAML** (GitHub Actions jobs, matrices, caching) — the python-ci + skill. This skill supplies the commands a CI job should run, not the workflow. +- **Formatting YAML / JSON / TOML / Markdown prose** — Ruff formats Python + (including, since 0.16, Python code fences inside Markdown) and nothing else, + by deliberate maintainer decision + ([astral-sh/ruff#10738](https://github.com/astral-sh/ruff/issues/10738)); + the python-precommit skill covers non-Python files. +- **Running Ruff automatically on agent edits** (Claude Code hooks) — the + agent-guardrails skill. + +## Workflow + +### 1. Inspect before touching anything + +```bash +ls .flake8 .isort.cfg setup.cfg tox.ini ruff.toml .ruff.toml 2>/dev/null +grep -nE '^\[tool\.(ruff|black|isort)|^\[flake8\]|^\[isort\]' pyproject.toml setup.cfg tox.ini 2>/dev/null +grep -n 'requires-python' pyproject.toml +``` + +Pick the path: fresh setup (steps 2–3), migration off Black/Flake8/isort (step 5), +tuning an existing config (step 4), or fixing a red lint gate (step 6). + +### 2. Install a pinned Ruff + +```bash +uv add --dev "ruff==0.16.6" # example pin — check the latest release and pin that +# one-off, no project change: +uvx --from 'ruff==0.16.6' ruff check . +``` + +No-uv fallback (once, applies to every command below): `python -m pip install +'ruff==0.16.6'` and drop the `uv run` prefix. + +Why pin — formatter output and the rule catalog change between minor releases: +0.15 shipped a "2026 style guide" that changed lambda and empty-line formatting +([astral.sh/blog/ruff-v0.15.0](https://astral.sh/blog/ruff-v0.15.0)), and 0.16 +(2026-07-23) grew the **default** rule set from 59 to 413 rules while dropping 18 +`E`/`F` codes from it, so an unconfigured repo's findings changed overnight +([astral.sh/blog/ruff-v0.16.0](https://astral.sh/blog/ruff-v0.16.0)). An unpinned +Ruff makes local, teammate, and CI runs disagree — the top reported lint-workflow +failure. Optionally enforce the pin in config with +`required-version = "0.16.6"` under `[tool.ruff]`. Note that `uv format` +(experimental since uv 0.8.13) runs a Ruff it bundles itself, so its version can +differ from the project's pin — keep `ruff` in dev dependencies as the source of +truth and treat `uv format` as a convenience wrapper. + +### 3. Write the config (fresh setup) + +All Ruff config lives in `pyproject.toml`. Keep exactly one config file — a +`ruff.toml` next to a `[tool.ruff]` section is a recipe for confusion. Starter: + +```toml +[tool.ruff] +target-version = "py311" # lowest Python you support; gates UP rewrites +line-length = 88 +extend-exclude = ["migrations"] # generated code, if any + +[tool.ruff.lint] +select = [ + "E", "W", # pycodestyle + "F", # pyflakes + "I", # isort (import sorting) + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "RUF", # Ruff-specific rules +] +ignore = ["E501"] # line length is the formatter's job + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["S101"] # assert is fine in tests (relevant once S is on) +"__init__.py" = ["F401"] # re-exports + +[tool.ruff.lint.isort] +known-first-party = ["yourpackage"] + +[tool.ruff.format] +quote-style = "double" +docstring-code-format = true # formats Python snippets inside docstrings +``` + +Non-obvious decisions baked into this block: + +- **Never rely on Ruff's default rule set.** Before 0.16 it was `E4`/`E7`/`E9` + + `F` (59 rules, less than a Flake8+plugins stack); since 0.16 it is 413 rules across + `B`, `UP`, `SIM`, `PL` and more, with 18 former `E`/`F` defaults removed — a repo + without `select` gets a different gate on every upgrade. `select` **replaces** + the default set, so an explicit list freezes it; `select = ["E4", "E7", "E9", + "F"]` reproduces the pre-0.16 baseline if a team wants exactly that. +- **`ignore = ["E501"]` whenever the formatter is enabled** — `ruff format` already + wraps at `line-length`; linting it too double-reports on lines the formatter + cannot break (URLs, long strings). +- **`target-version` gates `UP` (pyupgrade) rewrites.** If unset, Ruff falls back + to `project.requires-python`; if that is also absent it assumes py310, and + autofixes can emit syntax that crashes older supported runtimes. +- Avoid `select = ["ALL"]` — it silently enables every new rule on every Ruff + upgrade, breaking checks unpredictably. Official guidance is to start narrow and + grow one category at a time + ([docs.astral.sh/ruff/linter/](https://docs.astral.sh/ruff/linter/)). + +Deeper rule catalog, tier strategy, and sub-table options (isort, pydocstyle, +bugbear): `references/rule-selection.md`. + +### 4. Tune an existing config + +- Noise in one area (tests, scripts, notebooks) → scope it with + `[tool.ruff.lint.per-file-ignores]`; never delete the rule from `select` globally + because tests complain. Common: `"tests/**/*.py" = ["S101", "PLR2004"]`, + `"scripts/*" = ["T201"]`. +- A single intentional violation → `# noqa: ` with the specific code and a + short reason. Bare `# noqa` hides every rule on the line. +- Codes from non-Ruff tools in `# noqa` comments (custom Flake8 plugin) → + declare `external = ["XX"]` in `[tool.ruff.lint]`, or `RUF100` strips those + comments on the next `--fix`. +- Widen coverage one prefix at a time (`"S"`, `"PT"`, `"PTH"`, …), run + `uv run ruff check .` after each addition, and fix or scope before adding more. +- Sanity-check the result: `python3 "${CLAUDE_SKILL_DIR}/scripts/check_ruff_config.py" --root .` + (read-only; flags E501 double-reporting, `ALL`, formatter-conflicting rules, + leftover legacy configs, missing target-version). + +### 5. Migrate from Black / Flake8 / isort + +Full mapping tables (commands, config keys, Flake8-plugin→prefix) and the +large-legacy-codebase strategy: `references/migration.md`. The sequence: + +1. **Translate config first.** `[tool.black]` → `[tool.ruff]` + `[tool.ruff.format]`, + `[tool.isort]`/`.isort.cfg` → `[tool.ruff.lint.isort]` (kebab-case keys), + `.flake8` → `select`/`ignore`/`per-file-ignores`. Ruff never reads + `[tool.black]` or `.flake8` — anything not translated is silently lost. +2. **Check plugin parity.** For each Flake8 plugin in use, confirm a Ruff prefix + exists (table in the reference). A business-critical plugin with no equivalent + → hybrid: keep Flake8 scoped to that plugin, add its codes to + `external = [...]`. +3. **One-time cleanup as its own reviewed commit**, before any gate is switched: + ```bash + uv run ruff check . --fix && uv run ruff format . + ``` + Review the diff — Ruff's formatter is close to Black on large codebases but not + byte-identical; deviations are documented at + [docs.astral.sh/ruff/formatter/black/](https://docs.astral.sh/ruff/formatter/black/). + For a large reformat, record the commit in `.git-blame-ignore-revs`. +4. **Retire the old stack in the same PR**: remove black/flake8/isort (and their + plugins) from dev dependencies; delete `.flake8`, `.isort.cfg`, `[tool.black]`, + `[tool.isort]`. Two active formatters will fight over the same lines + ([astral-sh/ruff#1307](https://github.com/astral-sh/ruff/issues/1307)). +5. **Tell the user to repoint whatever invoked the old tools** — pre-commit hooks + (python-precommit skill) and CI jobs (python-ci skill). Provide only the + commands: `ruff check .` and `ruff format --check .`. + +An existing Black+Flake8 setup that works and hurts nobody is a valid reason to +not migrate — churn has cost. Migrate when there is friction (speed, config +sprawl, plugin abandonment), not for fashion. + +### 6. Fix lint / format failures + +```bash +uv run ruff check . --fix # 1. autofix what's safe +uv run ruff format . # 2. THEN format — fixes can leave code needing reformat +uv run ruff check . # 3. see what's left; --fix exiting clean != zero violations +``` + +Order matters: `--fix` rewrites imports/comprehensions that then need reformatting. +For what remains: + +- Fix the code. Reach for suppressions only when the violation is intentional, and + then the narrowest one: specific `# noqa: ` > per-file-ignores > global + `ignore`. Never make a red gate green by deleting rules from `select`. +- Some fixes are deliberately held back as unsafe (could change behavior). Review + them explicitly with `uv run ruff check . --fix --unsafe-fixes --diff` before + applying. +- In GitHub Actions logs, `uv run ruff check . --output-format=github` emits + file/line annotations on the PR (the workflow YAML itself is python-ci + territory). + +### 7. Verify done + +```bash +uv run ruff check . && uv run ruff format --check . && echo LINT-OK +python3 "${CLAUDE_SKILL_DIR}/scripts/check_ruff_config.py" --root . +``` + +## Output spec + +Done means all of: + +- `pyproject.toml` has `[tool.ruff]` with an explicit `select`, `E501` ignored + while the formatter is in use, and a resolved `target-version` (or + `requires-python`). +- Ruff pinned in dev dependencies (and optionally `required-version` in config). +- `uv run ruff check .` and `uv run ruff format --check .` both exit 0. +- No leftover `.flake8` / `.isort.cfg` / `[tool.black]` / `[tool.isort]`, and no + black/flake8/isort in dev dependencies (unless the documented hybrid pattern). +- Migrations: the mechanical reformat isolated in its own commit; suppressions + carry rule codes and reasons. +- `scripts/check_ruff_config.py` reports 0 errors. + +## Failure modes & gotchas + +| Symptom | Cause | Fix | +| --- | --- | --- | +| Ruff installed but finds too little (pre-0.16) or floods CI after an upgrade (0.16+) | Running on the default rule set, which changed from `E`+`F` to 413 rules in 0.16 | Write an explicit `select` (step 3) — it replaces the defaults, so upgrades stop moving the gate | +| E501 violations the formatter refuses to fix | Linter and formatter both own line length | `ignore = ["E501"]`; the formatter wraps what it can | +| Routine Ruff upgrade suddenly fails checks | `select = ["ALL"]` auto-enables new rules | Explicit `select`; pin the Ruff version | +| Formatter and linter fight (commas, quotes, indentation) | `COM812`/`COM819`, `Q000`–`Q004`, `D203`/`D206`/`D300`, `W191`, `E111`/`E114`/`E117` (and `ISC002` without `ISC001` + `allow-multiline = false`) conflict with `ruff format` | Drop them from `select`/add to `ignore` — see [conflicting rules](https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules) | +| `--fix` deletes `# noqa: XY123` comments | `RUF100` treats unknown codes as unused suppressions | `external = ["XY"]` in `[tool.ruff.lint]` | +| Autofix emits syntax that breaks the oldest supported Python | No `target-version` / `requires-python` → py310 assumed | Set `target-version` to the real minimum | +| Excluded files still linted when paths are passed one-by-one | Explicit file args bypass `exclude` ([ruff#9585](https://github.com/astral-sh/ruff/issues/9585)) | Use per-file-ignores for must-hold rules; hook-layer wiring belongs to python-precommit | +| `ruff check --fix` exits clean, CI still red | Only *safe* fixes auto-apply; the rest still report | Re-run plain `check`; fix manually or review `--unsafe-fixes --diff` | +| Files keep flip-flopping between formats | Black (or an IDE Black plugin) still active alongside Ruff | One formatter only — remove Black, update editor settings | +| Old `.flake8` / `[tool.black]` settings "stopped working" | Ruff never reads them | Translate into `[tool.ruff]`, then delete the originals | +| Passes locally, fails elsewhere | Version drift across dev/CI/hooks | Same pinned version everywhere; `required-version` makes mismatch a hard error | +| Expected Ruff to format YAML/Markdown/TOML | Python-only by design; since 0.16 Python *code fences* in Markdown are formatted by default (opt out with `fmt: off` comments or `extend-exclude`) | python-precommit skill covers non-Python formatting; scope Prettier/mdformat so they do not fight Ruff on fenced Python | +| First CI run after adding Ruff fails hard | No local cleanup pass before wiring the gate | Run step 6 locally, commit, then gate | +| `ruff check` green but diffs look unformatted | Lint and format are separate concerns — `check` doesn't verify formatting | Always run both gates (step 7) | + +## Bundled resources + +- `references/rule-selection.md` — rule-prefix→origin map, tiered adoption + strategy, `select` vs `extend-select` vs `ALL` trade-offs, per-file-ignore + conventions, isort/pydocstyle/bugbear sub-tables, fix-safety knobs. +- `references/migration.md` — old-command→Ruff mapping, Black/isort/Flake8 config + key translation, Flake8-plugin→prefix table, custom-plugin hybrid, incremental + adoption for large legacy codebases. +- `scripts/check_ruff_config.py` — read-only config sanity checker; exits non-zero + on findings. Run `python3 "${CLAUDE_SKILL_DIR}/scripts/check_ruff_config.py" --root ` + (`--strict` to fail on warnings too). diff --git a/skills/python-lint/evals/evals.json b/skills/python-lint/evals/evals.json new file mode 100644 index 0000000..ae1c639 --- /dev/null +++ b/skills/python-lint/evals/evals.json @@ -0,0 +1,80 @@ +{ + "skill": "python-lint", + "cases": [ + { "type": "should_trigger", "prompt": "set up ruff for this package" }, + { "type": "should_trigger", "prompt": "Please replace our Black, Flake8, and isort toolchain with Ruff, preserving current behavior as closely as possible." }, + { "type": "should_trigger", "prompt": "which ruff rules should we enable in pyproject.toml?" }, + { "type": "should_trigger", "prompt": "set up linting for this python project" }, + { "type": "should_trigger", "prompt": "fix the lint errors in src/ so the checks go green" }, + { "type": "should_trigger", "prompt": "configure the formatter to use 100-character lines" }, + { "type": "should_trigger", "prompt": "can u add a [tool.ruff] section to pyproject pls" }, + { "type": "should_trigger", "prompt": "ruf check failing everywhere after the upgrade, fix teh lint errors" }, + { "type": "should_trigger", "prompt": "ruff keeps flagging E501 but the formatter already handles line length, sort this out" }, + { "type": "should_trigger", "prompt": "allow assert in tests but keep the bandit security rules everywhere else" }, + { "type": "should_trigger", "prompt": "we have a custom flake8 plugin, can we still move to ruff?" }, + { "type": "should_trigger", "prompt": "huge legacy codebase — how do we adopt ruff without one giant reformat commit?" }, + { "type": "should_not_trigger", "prompt": "add ruff and ruff-format hooks to our pre-commit config" }, + { "type": "should_not_trigger", "prompt": "set up a GitHub Actions workflow that runs lint, type-check, and test jobs on every PR" }, + { "type": "should_not_trigger", "prompt": "make ruff format run automatically whenever claude edits a python file" }, + { "type": "should_not_trigger", "prompt": "mypy is reporting 40 errors after the upgrade, fix them" }, + { "type": "should_not_trigger", "prompt": "format the YAML and Markdown files in this repo" }, + { "type": "should_not_trigger", "prompt": "set up strict type checking for this package" }, + { "type": "should_not_trigger", "prompt": "ruff passes locally but fails in pre-commit — is the hook rev outdated?" }, + { "type": "should_not_trigger", "prompt": "the lint job in CI needs uv caching and an all-checks-passed aggregator" }, + { "type": "should_not_trigger", "prompt": "set up pytest with coverage and a fail_under gate" }, + { "type": "should_not_trigger", "prompt": "bump the ruff-pre-commit rev so it matches the version CI uses" }, + { + "type": "quality", + "prompt": "Set up ruff linting and formatting for my Python package — src layout, requires-python >=3.11, no linter today.", + "expected_behavior": [ + "adds [tool.ruff] config to pyproject.toml with an explicit [tool.ruff.lint] select list beyond the E/F defaults (a curated set including at least E, F, I, UP, B or similar)", + "sets target-version or confirms project.requires-python covers it, noting it gates UP (pyupgrade) rewrites", + "puts E501 in lint ignore (or explains its absence) because line length is the formatter's job", + "pins ruff to a specific version as a dev dependency instead of installing it unpinned", + "runs ruff check --fix before ruff format, then reports any remaining violations instead of claiming success" + ] + }, + { + "type": "quality", + "prompt": "Migrate this repo from black + flake8 + isort to ruff.", + "expected_behavior": [ + "translates existing [tool.black], [tool.isort], and .flake8 settings (line length, known-first-party, ignore lists) into [tool.ruff] equivalents before deleting them", + "runs the one-time cleanup (ruff check --fix, then ruff format) as a separate reviewed commit and warns output is close to Black but not byte-identical", + "checks for Flake8 plugins with no Ruff equivalent and proposes the hybrid pattern with external codes if one exists", + "removes black, flake8, and isort from dev dependencies and deletes their config files/sections so two formatters cannot fight", + "tells the user to update pre-commit and CI invocations without authoring those files itself" + ] + }, + { + "type": "quality", + "prompt": "The lint gate is red — fix all the ruff errors in src/.", + "expected_behavior": [ + "runs ruff check --fix before ruff format and respects that order", + "fixes remaining violations by editing code rather than adding blanket suppressions or deleting rules from select", + "any suppression added is a targeted noqa with the specific rule code or a scoped per-file-ignores entry, with a stated reason", + "re-runs ruff check and ruff format --check and confirms both exit clean before declaring done", + "does not modify CI workflow files or pre-commit config to make the gate pass" + ] + }, + { + "type": "quality", + "prompt": "Ruff is way too noisy in our test files — asserts and magic numbers get flagged constantly. Tune the config.", + "expected_behavior": [ + "adds per-file-ignores entries scoped to the tests directory (e.g. S101 for assert, PLR2004 for magic values) instead of disabling those rules globally", + "keeps the same rules active for production code", + "briefly justifies why each ignored rule is acceptable in tests", + "re-runs ruff check to confirm the test noise is gone and nothing new broke" + ] + }, + { + "type": "quality", + "prompt": "We have a 300k-line legacy codebase. Adopt ruff without a giant reformat commit breaking git blame.", + "expected_behavior": [ + "offers incremental adoption options — a minimal starting select expanded category by category, and/or ruff check --add-noqa bulk suppression tightened module by module", + "mentions protecting git blame via an isolated reformat commit recorded in .git-blame-ignore-revs, or changed-lines-only tools such as Darker, with trade-offs", + "gets the local baseline clean before recommending any blocking gate", + "sets target-version to the oldest supported Python so UP autofixes cannot break it" + ] + } + ] +} diff --git a/skills/python-lint/references/migration.md b/skills/python-lint/references/migration.md new file mode 100644 index 0000000..9ffb4e6 --- /dev/null +++ b/skills/python-lint/references/migration.md @@ -0,0 +1,187 @@ +# Migrating to Ruff from Black / Flake8 / isort / pyupgrade + +**Contents:** [Decision guide](#decision-guide) · [Migration sequence](#migration-sequence) · +[Command mapping](#command-mapping) · [Config key translation](#config-key-translation) · +[Flake8 plugin → Ruff prefix](#flake8-plugin--ruff-prefix) · +[Custom Flake8 plugins (hybrid pattern)](#custom-flake8-plugins-hybrid-pattern) · +[Large legacy codebases](#large-legacy-codebases) · [Parity caveats](#parity-caveats) · +[Cleanup checklist](#cleanup-checklist) · [Sources](#sources) + +## Decision guide + +| Scenario | Recommendation | +| --- | --- | +| New project, no legacy tooling | Ruff only (`ruff check` + `ruff format`); never install Black/Flake8/isort | +| Existing Black+Flake8 works, no pain | No urgent need — migration churn has cost; revisit when friction appears | +| Lint speed or config sprawl is real friction | Migrate; the mapping below is mechanical | +| Business-critical custom Flake8 plugin, no Ruff equivalent | Hybrid: Ruff for everything else, Flake8 scoped to that plugin | +| Huge legacy codebase, blame matters | Incremental: `--add-noqa` or changed-lines-only formatting (below) | +| Team standardizes on uv | `uv format` (experimental since uv 0.8.13) runs a bundled Ruff — keep the project's own `ruff` pin as the source of truth | + +## Migration sequence + +1. Install pinned Ruff: `uv add --dev "ruff==0.15.6"` (check latest, pin it). +2. Translate configs using the tables below into `[tool.ruff*]` sections. +3. Verify plugin parity (table below) for every Flake8 plugin in + requirements/pre-commit. +4. One-time mechanical cleanup, its own commit, reviewed: + `uv run ruff check . --fix && uv run ruff format .` + Add that commit's SHA to `.git-blame-ignore-revs` so `git blame` skips it + (GitHub honors this file automatically). +5. Remove old dev dependencies (black, flake8 + plugins, isort, pyupgrade, + autoflake) and delete their config files/sections. +6. Repoint invocations: local docs/Makefile to `ruff check .` + + `ruff format --check .`; pre-commit and CI rewiring belong to the + python-precommit and python-ci skills respectively. + +Do NOT wire any blocking gate before step 4 is committed — the first gated run +on an uncleaned tree fails hard. + +## Command mapping + +| Old command | Ruff equivalent | +| --- | --- | +| `flake8 .` | `ruff check .` | +| `black .` | `ruff format .` | +| `black --check --diff .` | `ruff format --check --diff .` | +| `isort .` | `ruff check --select I --fix .` | +| `isort --check .` | `ruff check --select I .` | +| `pyupgrade --py311-plus f.py` | `ruff check --select UP --fix f.py` (with `target-version = "py311"`) | +| `autoflake --remove-all-unused-imports` | `ruff check --select F401 --fix .` | + +## Config key translation + +Ruff reads **only** `[tool.ruff*]` (or `ruff.toml`). It silently ignores +`[tool.black]`, `[tool.isort]`, `.flake8`, and `setup.cfg` — any setting not +translated is simply lost, with no warning. + +### From `[tool.black]` + +| Black | Ruff | +| --- | --- | +| `line-length = 88` | `[tool.ruff] line-length = 88` | +| `target-version = ["py311"]` | `[tool.ruff] target-version = "py311"` (single string) | +| `skip-string-normalization = true` | `[tool.ruff.format] quote-style = "preserve"` | +| `skip-magic-trailing-comma = true` | `[tool.ruff.format] skip-magic-trailing-comma = true` | +| `preview = true` | no direct equivalent; Ruff has its own `preview` flag with different contents | + +### From `[tool.isort]` / `.isort.cfg` + +Keys move to `[tool.ruff.lint.isort]` and become kebab-case: + +| isort | Ruff | +| --- | --- | +| `profile = "black"` | drop it — Ruff's `I` rules are formatter-consistent already | +| `known_first_party = ["pkg"]` | `known-first-party = ["pkg"]` | +| `combine_as_imports = true` | `combine-as-imports = true` | +| `force_sort_within_sections = true` | `force-sort-within-sections = true` | +| `add_imports = ["from __future__ import annotations"]` | `required-imports = [...]` | +| `skip` / `# isort: skip` comments | `# isort: skip` etc. respected as-is | + +### From `.flake8` / `setup.cfg [flake8]` + +| Flake8 | Ruff | +| --- | --- | +| `max-line-length = 100` | `[tool.ruff] line-length = 100` (then ignore `E501` if formatting) | +| `extend-ignore = E203, W503` | usually deletable — Ruff's defaults don't fight the formatter the way pycodestyle fought Black | +| `ignore = ...` | `[tool.ruff.lint] ignore = [...]` | +| `per-file-ignores = tests/*:S101` | `[tool.ruff.lint.per-file-ignores] "tests/*" = ["S101"]` | +| `exclude = ...` | `[tool.ruff] extend-exclude = [...]` | +| `max-complexity = 10` | `[tool.ruff.lint.mccabe] max-complexity = 10` + select `C90` | +| installed plugins | `select` prefixes — table below | + +## Flake8 plugin → Ruff prefix + +| Flake8 plugin | Ruff prefix | +| --- | --- | +| pycodestyle | `E` / `W` | +| pyflakes | `F` | +| pep8-naming | `N` | +| flake8-bugbear | `B` | +| flake8-comprehensions | `C4` | +| flake8-simplify | `SIM` | +| flake8-bandit | `S` | +| flake8-pytest-style | `PT` | +| flake8-quotes | `Q` (skip when using `ruff format`) | +| flake8-tidy-imports | `TID` | +| flake8-type-checking | `TC` | +| flake8-print | `T20` | +| flake8-docstrings / pydocstyle | `D` — **partial parity**, verify the specific checks you rely on | +| mccabe | `C90` | + +Anything not in this table: search +[docs.astral.sh/ruff/rules/](https://docs.astral.sh/ruff/rules/) — Ruff +reimplements most popular plugins but not all; verify before deleting a plugin +a team relies on. + +## Custom Flake8 plugins (hybrid pattern) + +Ruff has no plugin API. For an in-house plugin with no Ruff equivalent: + +1. Keep Flake8 installed, scoped to only that plugin's codes: + `flake8 --select=WH .` (whatever the plugin's prefix is). +2. Declare the foreign codes so Ruff leaves their suppressions alone: + + ```toml + [tool.ruff.lint] + external = ["WH"] + ``` + + Without this, `RUF100` treats `# noqa: WH001` as an unused suppression and + **deletes it on the next `--fix`**. +3. Plan an exit: file a rule request upstream or port the check to a standalone + script; a two-linter setup is a liability, not a destination. + +## Large legacy codebases + +Options, in increasing order of disruption: + +1. **Minimal select, grow later** — start `select = ["E", "F"]`, expand one + prefix at a time as modules get cleaned. +2. **Bulk-suppress, then tighten** — `uv run ruff check --add-noqa .` writes a + targeted `# noqa` on every existing violation; the gate goes green + immediately, and the debt is visible and grep-able. Remove noqas module by + module. (This mutates the tree — run it on a branch and review.) +3. **Changed-lines-only formatting** — [Darker](https://github.com/akaihola/darker) + applies formatting only to lines touched by a commit, avoiding the big-bang + reformat entirely, at the cost of a long transition with mixed styles. +4. **Big-bang reformat** — one mechanical commit, reviewed, recorded in + `.git-blame-ignore-revs`. Simplest end state; noisiest single diff. + +Whichever option: set `target-version` before any `--fix`, or `UP` rewrites can +emit syntax newer than the oldest Python the project still supports. + +## Parity caveats + +- **Formatter vs Black** — near-identical on large codebases per Astral's own + benchmarks (not independently verified); known deviations around complex + f-strings, block-open newlines, and preview styles are documented at + [docs.astral.sh/ruff/formatter/black/](https://docs.astral.sh/ruff/formatter/black/). + Diff a representative file before promising byte-identical output. +- **isort** — edge cases around aliased imports, inline comments, exotic + multi-line grouping; a few projects keep isort for those files only. +- **pydocstyle (`D`)** — partial rule coverage; check the rules page for the + specific checks in use. +- **Exact-parity expectations are the top migration friction** reported by + users (e.g. [r/Python migration thread](https://www.reddit.com/r/Python/comments/1gvnfvi/migrating_from_black_and_flake8_to_ruff/)) — + set the expectation of a small, reviewable style delta, not zero delta. + +## Cleanup checklist + +After migrating, all of these should be gone (leftovers cause silent conflicts): + +- [ ] `.flake8`, `.isort.cfg`, `tox.ini [flake8]`, `setup.cfg [flake8]`/`[isort]` +- [ ] `[tool.black]`, `[tool.isort]` in `pyproject.toml` +- [ ] black / flake8 / flake8-* / isort / pyupgrade / autoflake in dev deps +- [ ] Editor settings invoking Black (VS Code `python.formatting.provider`, etc.) +- [ ] Makefile / docs / CONTRIBUTING referencing the old commands +- [ ] Old pre-commit hooks and CI steps (owned by python-precommit / python-ci) + +The bundled `scripts/check_ruff_config.py` detects most of these automatically. + +## Sources + +- Ruff FAQ (tool replacement matrix) — +- pydevtools migration how-to — +- Real-world OSS migration discussions — , +- Black/Flake8/Ruff comparison — diff --git a/skills/python-lint/references/rule-selection.md b/skills/python-lint/references/rule-selection.md new file mode 100644 index 0000000..0a2c589 --- /dev/null +++ b/skills/python-lint/references/rule-selection.md @@ -0,0 +1,177 @@ +# Ruff rule selection and tuning + +**Contents:** [How selection works](#how-selection-works) · +[Rule prefix → origin map](#rule-prefix--origin-map) · +[Tiered adoption strategy](#tiered-adoption-strategy) · +[Per-file-ignores conventions](#per-file-ignores-conventions) · +[Sub-table options](#sub-table-options) · [Fix safety](#fix-safety) · +[Formatter-conflicting rules](#formatter-conflicting-rules) · [Sources](#sources) + +## How selection works + +- Ruff's default set changed in 0.16 (2026-07-23): from **only `E` and `F`** (59 + rules) to **413 rules** spanning `B`, `UP`, `SIM`, `PL` and more, with 18 former + `E`/`F` defaults (E401, E402, E501-adjacent E7xx style codes, F403, F405, …) + removed — see https://docs.astral.sh/ruff/default-rules/. (Even before 0.16 + the default was `E4`/`E7`/`E9` + `F`, never all of `E`.) The lesson is + unchanged: write `[tool.ruff.lint] select` explicitly so the enabled set is + yours, not the release's. +- `select` **replaces** the default set; `extend-select` **adds to** whatever the + current basis is. Trade-off, both positions defensible: + - `select` (Astral's documented lean, [docs.astral.sh/ruff/linter/](https://docs.astral.sh/ruff/linter/)) — + the enabled set is fully explicit in one place; upgrades cannot silently + change it. + - `extend-select` (recommended by some guides, e.g. pydevtools) — builds on + defaults, shorter config; but the effective set now depends on what the + installed Ruff version considers default. + Prefer `select` for packages with multiple contributors; the explicitness is + worth three extra lines. +- `select = ["ALL"]` enables every rule **including ones added in future Ruff + releases** — checks break on routine upgrades with rules nobody chose. Ruff + auto-disables a few mutually-conflicting rules under `ALL` (e.g. D203/D211), + but that does not make it stable. If a team insists on `ALL`, pair it with a + hard-pinned Ruff version and an explicit `ignore` list, and treat every + version bump as a config-review event. +- Precedence: CLI flags (`--select`) override `pyproject.toml`; the nearest + `pyproject.toml`/`ruff.toml` up the directory tree from each file wins. One + root config covers a whole monorepo — per-package differences go in + `per-file-ignores` keyed by path, not per-package config files. + +## Rule prefix → origin map + +| Prefix | Origin tool / plugin | What it catches | +| --- | --- | --- | +| `E`, `W` | pycodestyle | PEP 8 style errors / warnings | +| `F` | Pyflakes | unused imports/variables, undefined names | +| `I` | isort | import order and grouping | +| `N` | pep8-naming | naming conventions | +| `UP` | pyupgrade | outdated syntax for your target-version | +| `B` | flake8-bugbear | likely bugs and design problems | +| `C4` | flake8-comprehensions | inefficient comprehensions | +| `SIM` | flake8-simplify | needlessly complex constructs | +| `S` | flake8-bandit | security issues (asserts, subprocess, hashes) | +| `T20` | flake8-print | stray `print` / `pprint` | +| `PT` | flake8-pytest-style | pytest idioms | +| `Q` | flake8-quotes | quote style (conflicts with the formatter — skip) | +| `TID` | flake8-tidy-imports | banned/relative imports | +| `TC` (formerly `TCH`) | flake8-type-checking | imports movable into `TYPE_CHECKING` | +| `D` | pydocstyle | docstring conventions (**partial parity** — some checks missing) | +| `PL` (`PLC/PLE/PLR/PLW`) | Pylint (subset) | refactoring, magic values (`PLR2004`) | +| `C90` | mccabe | cyclomatic complexity | +| `ISC` | flake8-implicit-str-concat | implicit concatenation (formatter-safe; `ISC002` conflicts only without `ISC001`) | +| `ICN` | flake8-import-conventions | `import numpy as np`-style aliases | +| `ANN` | flake8-annotations | annotation *presence/style* — NOT type checking | +| `LOG`, `G` | flake8-logging(-format) | logging misuse | +| `FLY` | flynt | `%`/`.format` → f-string | +| `PTH` | flake8-use-pathlib | `os.path` → `pathlib` | +| `PERF` | Perflint | performance anti-patterns | +| `ARG` | flake8-unused-arguments | unused function arguments | +| `RUF` | Ruff-specific | rules with no upstream equivalent, incl. `RUF100` (unused noqa) | + +Full catalog (900+ rules): [docs.astral.sh/ruff/rules/](https://docs.astral.sh/ruff/rules/). + +## Tiered adoption strategy + +Grow the set one tier at a time; run `ruff check .`, fix or scope, then advance. + +| Tier | Add | Rationale | +| --- | --- | --- | +| 1 — baseline | `E`, `W`, `F`, `I` | old Flake8+isort parity; near-zero false positives | +| 2 — correctness | `B`, `C4`, `UP`, `SIM`, `RUF` | real-bug detectors and modernization; autofix-heavy | +| 3 — opinionated | `N`, `S`, `T20`, `PT`, `PTH`, `PERF`, `TC` | pick per project; each adds review burden | +| 4 — docs | `D` with `convention = "google"` (or numpy/pep257) | only when docstrings are a maintained artifact | + +Rules of thumb: + +- Enable `S` (security) only with the `tests/` per-file-ignore in place, or the + suite drowns in `S101` (assert). +- Skip `Q`, `COM` (and `D203`/`D206`/`D300`, `W191`/`E11x`) when using `ruff format` (see + [formatter-conflicting rules](#formatter-conflicting-rules)). +- `ANN` is high-noise; if the package uses a real type checker, that layer + already enforces annotations where they matter. + +## Per-file-ignores conventions + +```toml +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert is the point of tests + "PLR2004", # magic values are fine in assertions + "ARG001", # fixtures produce "unused" arguments +] +"__init__.py" = ["F401"] # re-export pattern +"scripts/*" = ["T201"] # CLI scripts may print +"docs/conf.py" = ["A001"] # sphinx shadows builtins +``` + +- Generated code (Django `migrations/`, protobuf output) goes in + `extend-exclude`, not per-file-ignores — don't lint it at all. +- Keep per-file-ignores keyed by **pattern relative to the config file**; verify + patterns actually match files (a typo silently ignores nothing). The bundled + `check_ruff_config.py` flags non-matching patterns. +- Caution: when a runner passes file paths explicitly (pre-commit does), + `exclude` can be bypassed ([ruff#9585](https://github.com/astral-sh/ruff/issues/9585)); + per-file-ignores are more robust than exclude for rules that must never fire. + +## Sub-table options + +```toml +[tool.ruff.lint.isort] +known-first-party = ["mypackage"] # fixes first/third-party misgrouping +combine-as-imports = true +required-imports = ["from __future__ import annotations"] + +[tool.ruff.lint.pydocstyle] +convention = "google" # or "numpy", "pep257" + +[tool.ruff.lint.flake8-bugbear] +# stop B008 false positives on FastAPI-style call-in-default patterns +extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] + +[tool.ruff.lint.mccabe] +max-complexity = 12 +``` + +isort options translate from snake_case to kebab-case +(`known_first_party` → `known-first-party`); Ruff also respects +`# isort: skip` / `# isort: on/off` action comments. Parity has edge cases +around aliased imports and exotic multi-line grouping — diff one real commit +before trusting drop-in behavior. + +## Fix safety + +- `ruff check --fix` applies **safe fixes only** by default. A clean `--fix` run + does not mean zero violations — unsafe-fixable and unfixable diagnostics + still report. +- `--unsafe-fixes` (or `unsafe-fixes = true` in config) opts into + behavior-affecting rewrites. Review with `--diff` first; unsafe fixes have + occasionally introduced real bugs. +- Scope what autofix may touch with `fixable` / `unfixable` / + `extend-fixable` in `[tool.ruff.lint]` — e.g. `unfixable = ["F401"]` stops + `--fix` from deleting an import you were about to use, while still reporting it. +- Roughly speaking `--fix` clears the large majority of style debt automatically; + the remainder needs human edits. + +## Formatter-conflicting rules + +When `ruff format` is in use, these lint rules fight it — leave them out of +`select` (Ruff warns about some of them at runtime): + +- `E501` (line length — the formatter owns it; `ignore = ["E501"]`) +- `COM812` / `COM819` (trailing commas) +- `ISC002` — only when `ISC001` is *not* selected and `allow-multiline = false` + (`ISC001` itself is fine with the formatter) +- `Q000`–`Q004` (quote style — set `[tool.ruff.format] quote-style` instead) +- `W191`, `E111`, `E114`, `E117`, `D203`, `D206`, `D300` (indentation/docstring + mechanics the formatter normalizes) + +Current list: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules. + +Authoritative list: [docs.astral.sh/ruff/formatter/#conflicting-lint-rules](https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules). + +## Sources + +- Ruff linter docs & rule catalog — , +- Ruff settings reference — +- pydevtools Ruff guide — +- Community config threads — diff --git a/skills/python-lint/scripts/check_ruff_config.py b/skills/python-lint/scripts/check_ruff_config.py new file mode 100644 index 0000000..53cff0a --- /dev/null +++ b/skills/python-lint/scripts/check_ruff_config.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Read-only sanity checker for a repo's Ruff lint/format configuration. + +Detects the footguns that most often bite Ruff setups and migrations: +leftover Black/Flake8/isort configs, reliance on Ruff's (version-dependent) default rule set, +E501 double-reporting, formatter-conflicting rules, select = ["ALL"], +missing target-version, deprecated top-level lint keys, and +per-file-ignores patterns that match no files. + +Usage: + python3 check_ruff_config.py [--root PATH] [--strict] + +Never modifies any file. Output is machine-readable, one finding per line: + ERROR : + WARN : + +Exit codes: 0 = clean (warnings allowed unless --strict), 1 = findings, +2 = environment/usage problem (e.g. Python < 3.11, missing pyproject.toml). +""" + +# Keep full, actionable lint guidance in this standalone checker. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # Python < 3.11 + print("ERROR environment: Python 3.11+ required (tomllib)", file=sys.stderr) + sys.exit(2) + +# Lint rules that fight `ruff format` (docs.astral.sh/ruff/formatter/#conflicting-lint-rules). +# W191/E111/E114/E117 also conflict but are preview-only rules — a plain +# select = ["E", "W"] does not enable them, so warning on them would be noise. +FORMATTER_CONFLICTS = ( # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules (2026-09) + "COM812", + "COM819", + "Q000", + "Q001", + "Q002", + "Q003", + "Q004", + "W191", + "E111", + "E114", + "E117", + "D203", + "D206", + "D300", +) # ISC001 is fine with the formatter; ISC002 conflicts only without ISC001 + allow-multiline=false +LEGACY_TOOLS = ("black", "flake8", "isort", "pyupgrade", "autoflake") +PYPROJECT = "pyproject.toml" +# Lint settings that belong under [tool.ruff.lint], not top-level [tool.ruff] +DEPRECATED_TOP_LEVEL = ( + "select", + "extend-select", + "ignore", + "extend-ignore", + "per-file-ignores", + "fixable", + "unfixable", + "external", +) + +findings: list[tuple[str, str, str]] = [] # (level, file, message) + + +def err(where: str, msg: str) -> None: + findings.append(("ERROR", where, msg)) + + +def warn(where: str, msg: str) -> None: + findings.append(("WARN ", where, msg)) + + +def _split_code(code: str) -> tuple[str, str]: + """Split 'ISC001' -> ('ISC', '001'); 'E5' -> ('E', '5').""" + m = re.match(r"^([A-Z]+)(\d*)$", code) + return (m.group(1), m.group(2)) if m else (code, "") + + +def selected(code: str, select: list[str]) -> bool: + """True if `select` turns on rule `code`, using Ruff's selector semantics. + + A selector matches when its alphabetic linter prefix equals the code's + (so "I" matches I001 but NOT ISC001, "C" matches C4xx/C9xx but NOT COM812) + and its digit part is a prefix of the code's digits. + """ + if "ALL" in select: + return True + code_alpha, code_digits = _split_code(code) + for sel in select: + sel_alpha, sel_digits = _split_code(sel.strip()) + if sel_alpha == code_alpha and code_digits.startswith(sel_digits): + return True + return False + + +def check_legacy_files(root: Path) -> None: + for name in (".flake8", ".isort.cfg"): + if (root / name).is_file(): + err( + name, + "legacy config file present — Ruff never reads it; translate to [tool.ruff*] and delete", + ) + + +def check_legacy_ini_sections(root: Path) -> None: + for name in ("setup.cfg", "tox.ini"): + p = root / name + if p.is_file(): + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for section in ("[flake8]", "[isort]"): + if re.search(rf"^\s*{re.escape(section)}\s*$", text, re.MULTILINE): + err( + name, + f"legacy {section} section present — Ruff never reads it; translate and delete", + ) + + +def check_legacy_tables(pyproject: dict) -> None: + tool = pyproject.get("tool", {}) + for legacy in ("black", "isort"): + if legacy in tool: + err( + PYPROJECT, + f"[tool.{legacy}] present alongside Ruff — silently ignored by Ruff and a conflict source; translate and delete", + ) + + +def check_legacy_dependencies(pyproject: dict) -> None: + # legacy tools still declared as dependencies + dep_lists: list[tuple[str, list]] = [] + project = pyproject.get("project", {}) + dep_lists.append(("project.dependencies", project.get("dependencies", []) or [])) + for group, deps in (project.get("optional-dependencies", {}) or {}).items(): + dep_lists.append((f"project.optional-dependencies.{group}", deps or [])) + for group, deps in (pyproject.get("dependency-groups", {}) or {}).items(): + deps = [ + d for d in (deps or []) if isinstance(d, str) + ] # skip include-group tables + dep_lists.append((f"dependency-groups.{group}", deps)) + for where, deps in dep_lists: + for dep in deps: + base = re.split(r"[\s\[<>=!~;]", dep.strip(), maxsplit=1)[0].lower() + if base in LEGACY_TOOLS or base.startswith("flake8-"): + warn( + PYPROJECT, + f"{where} still lists {base!r} — Ruff replaces it; remove unless using the documented hybrid pattern", + ) + + +def check_legacy_remnants(root: Path, pyproject: dict) -> None: + check_legacy_files(root) + check_legacy_ini_sections(root) + check_legacy_tables(pyproject) + check_legacy_dependencies(pyproject) + + +def check_select_rules(effective_select: list[str], ignore: set[str]) -> None: + if not effective_select: + warn( + PYPROJECT, + "no [tool.ruff.lint] select — you are on Ruff's default rule set, which changed from E+F to 413 rules in 0.16 and moves on upgrades; write an explicit select", + ) + if "ALL" in effective_select: + warn( + PYPROJECT, + 'select includes "ALL" — every Ruff upgrade silently enables new rules; prefer an explicit list (or hard-pin Ruff and review each bump)', + ) + if selected("E501", effective_select) and "E501" not in ignore: + warn( + PYPROJECT, + 'E501 enabled while using ruff format — the formatter owns line length; add "E501" to [tool.ruff.lint] ignore', + ) + for code in FORMATTER_CONFLICTS: + if code not in ignore and selected(code, effective_select): + warn( + PYPROJECT, + f"{code} conflicts with ruff format — remove its prefix from select or add it to ignore (docs.astral.sh/ruff/formatter/#conflicting-lint-rules)", + ) + + +def check_per_file_ignores(root: Path, pfi: dict) -> None: + for pattern in pfi: + try: + matched = any(root.glob(pattern)) or any(root.glob(f"**/{pattern}")) + except (ValueError, NotImplementedError): + continue + if not matched: + warn( + PYPROJECT, + f"per-file-ignores pattern {pattern!r} matches no files — typo? the ignore silently never applies", + ) + + +def check_ruff_tables(root: Path, ruff: dict) -> None: + where = PYPROJECT + + for key in DEPRECATED_TOP_LEVEL: + if key in ruff: + warn( + where, + f"[tool.ruff] {key!r} at top level is deprecated — move it under [tool.ruff.lint]", + ) + + lint = ruff.get("lint", {}) + select = [ + s + for s in (lint.get("select") or ruff.get("select") or []) + if isinstance(s, str) + ] + extend_select = [s for s in (lint.get("extend-select") or []) if isinstance(s, str)] + ignore = { + s + for s in (lint.get("ignore") or ruff.get("ignore") or []) + if isinstance(s, str) + } + effective_select = select + extend_select + + check_select_rules(effective_select, ignore) + + pfi = lint.get("per-file-ignores") or ruff.get("per-file-ignores") or {} + check_per_file_ignores(root, pfi) + + +def check_target_version(pyproject: dict) -> None: + ruff = pyproject.get("tool", {}).get("ruff", {}) + requires = pyproject.get("project", {}).get("requires-python") + if not ruff.get("target-version") and not requires: + warn( + PYPROJECT, + "neither [tool.ruff] target-version nor project.requires-python is set — Ruff assumes py310 and UP autofixes may emit syntax your oldest runtime cannot parse", + ) + + +def load_pyproject(root: Path) -> dict | None: + pyproject_path = root / PYPROJECT + if not pyproject_path.is_file(): + print( + f"ERROR {pyproject_path}: not found — run from (or --root to) the package repo", + file=sys.stderr, + ) + return None + try: + return tomllib.loads(pyproject_path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + print(f"ERROR pyproject.toml: cannot parse: {exc}", file=sys.stderr) + return None + + +def check_configuration(root: Path, pyproject: dict) -> None: + ruff_cfg = pyproject.get("tool", {}).get("ruff") + ruff_toml = [n for n in ("ruff.toml", ".ruff.toml") if (root / n).is_file()] + + if ruff_cfg is None and not ruff_toml: + err( + PYPROJECT, + "no [tool.ruff] section (and no ruff.toml) — Ruff is not configured; see SKILL.md step 3", + ) + if ruff_cfg is not None and ruff_toml: + warn( + PYPROJECT, + f"both [tool.ruff] and {ruff_toml[0]} exist — keep exactly one config source to avoid drift", + ) + if ruff_toml and ruff_cfg is None: + warn( + ruff_toml[0], + "config lives in ruff.toml — fine, but this checker only inspects pyproject.toml tables; legacy-remnant checks still apply", + ) + + if ruff_cfg is not None: + check_ruff_tables(root, ruff_cfg) + check_target_version(pyproject) + if ruff_cfg is not None or ruff_toml: + check_legacy_remnants(root, pyproject) + + +def report_findings(strict: bool) -> int: + n_err = n_warn = 0 + for level, where, msg in findings: + print(f"{level} {where}: {msg}", file=sys.stderr) + if level == "ERROR": + n_err += 1 + else: + n_warn += 1 + failed = n_err > 0 or (strict and n_warn > 0) + print(f"{'FAIL' if failed else 'OK'}: {n_err} error(s), {n_warn} warning(s)") + return 1 if failed else 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--root", + type=Path, + default=Path.cwd(), + help="repo root to check (default: cwd)", + ) + ap.add_argument("--strict", action="store_true", help="exit 1 on warnings too") + args = ap.parse_args() + root = args.root.resolve() + if not root.is_dir(): + print(f"ERROR usage: --root {root} is not a directory", file=sys.stderr) + return 2 + pyproject = load_pyproject(root) + if pyproject is None: + return 2 + check_configuration(root, pyproject) + return report_findings(args.strict) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/python-packaging/SKILL.md b/skills/python-packaging/SKILL.md new file mode 100644 index 0000000..970b441 --- /dev/null +++ b/skills/python-packaging/SKILL.md @@ -0,0 +1,273 @@ +--- +name: python-packaging +description: Configures packaging for a Python library — pyproject.toml metadata, build backend, src layout, CLI entry points, uv lockfiles, where dev dependencies go (dependency-groups vs optional-dependencies), wheels and sdists. Use to make a project installable, set up pyproject.toml, build a wheel, move off requirements.txt, or switch to uv. Not for publishing to PyPI or release automation. +license: MIT +--- + +# python-packaging + +Make a Python library installable and buildable with verified artifacts: one +`pyproject.toml` as the single source of truth, a deliberately chosen PEP 517 +build backend, `src/` layout, uv-managed dependencies with a committed lockfile, +and an sdist + wheel whose contents are checked rather than assumed. The failures +this skill prevents: declared files (`py.typed`, data files) silently missing from +the wheel, backends that cannot handle the project's layout or extensions, Poetry +migrations that switch the backend before resolution is verified, and `pip` calls +that desync a uv-managed environment. + +## When NOT to use + +- Publishing to PyPI, version bumps, changelogs, git tags, trusted publishing — + the python-release skill, if installed. Stop after artifacts are verified. +- Type-checker choice or configuration, and whether/how to ship `py.typed` — the + python-typing skill. This skill only verifies that an existing `py.typed` + actually lands in the built wheel. +- `[tool.ruff]` config — python-lint. pytest/coverage config — python-testing. + GitHub Actions — python-ci. Dependency vulnerability audits — python-supply-chain. +- Single-file scripts that just need dependencies — PEP 723 inline script metadata + plus `uv run script.py` covers that with no packaging at all. + +## Workflow + +### 1. Assess what exists + +```bash +ls setup.py setup.cfg MANIFEST.in requirements*.txt poetry.lock Pipfile uv.lock 2>/dev/null +grep -n "build-backend\|tool.poetry\|dependency-groups\|optional-dependencies" pyproject.toml 2>/dev/null +find . -name "*.pyx" -o -name "*.c" -o -name "Cargo.toml" | grep -v .venv | head +``` + +Record three facts before changing anything: (a) native extensions or pure +Python — this decides the backend; (b) flat or `src/` layout; (c) which +dev-dependency convention the repo already uses (`[dependency-groups]`, +a `dev` extra under `[project.optional-dependencies]`, or legacy +`[tool.uv] dev-dependencies`) — match it or migrate it wholesale, never mix. + +### 2. Choose the build backend + +| Backend | Pick when | Avoid when | +| --- | --- | --- | +| `uv_build` | pure-Python library; zero-config `src/` discovery, strict metadata validation | any C/Cython/Rust extension modules (unsupported); VCS-tag dynamic versioning needed | +| `hatchling` | dynamic versioning from git tags via `hatch-vcs` matters; widely treated as the general-guidance default; plugin ecosystem | you want uv_build's stricter validation and zero config | +| `setuptools` | extension modules (`ext_modules`), deep legacy config | greenfield pure-Python — looser validation, auto-discovery surprises | + +Do not adopt `poetry-core` for new projects; the ecosystem consolidated on +PEP 621 `[project]` metadata. Note that `uv build` is a backend-agnostic PEP 517 +frontend — it drives hatchling or setuptools just as well, so adopting uv as the +project manager does not force a backend migration; treat that as a separate, +verified step. + +```toml +[build-system] +requires = ["uv_build>=0.12,<0.13"] +build-backend = "uv_build" +``` + +Keep the upper bound on `uv_build`: it follows uv's versioning policy, where +breaking changes may land in minor releases, so an unbounded requirement can +break your build the day a new version ships. Hatchling and setuptools have more +conservative compatibility norms; a floor (e.g. `hatchling>=1.27`) is customary. +Risk framing: uv as a whole is pre-1.0, but the `uv_build` backend specifically +is marked Production/Stable on PyPI and has been the `uv init` default since +mid-2025 — do not conflate the two in either direction. + +Full trade-offs, `[tool.uv.build-backend]` keys, and per-backend file-inclusion +config: [references/build-backends.md](references/build-backends.md). + +### 3. Lay out the package (src/) + +``` +my-package/ +├── pyproject.toml +├── README.md +├── src/ +│ └── my_package/ +│ ├── __init__.py +│ └── py.typed # only if the project already ships one +└── tests/ +``` + +Use `src/` for libraries: tests and tools then import the *installed* package. +With a flat layout, `import my_package` resolves to the working-directory copy +even when the built wheel is broken or missing files — packaging bugs stay +invisible until release. For a new project, `uv init --lib my-package` scaffolds +`src/`, `py.typed`, and the uv_build backend in one step. + +uv_build expects `src//__init__.py` with the module named after the +normalized project name. If the import name differs, declare it: + +```toml +[tool.uv.build-backend] +module-name = "actual_import_name" +``` + +### 4. Write [project] metadata + +Minimal publishable set — `validate-pyproject` (step 6) will hold you to it: + +```toml +[project] +name = "my-package" +version = "0.1.0" +description = "One line, benefit first." +readme = "README.md" +license = "MIT" # SPDX expression (PEP 639; needs a current backend) +requires-python = ">=3.10" +authors = [{ name = "Your Name", email = "you@example.com" }] +dependencies = ["httpx>=0.27"] +classifiers = ["Programming Language :: Python :: 3"] + +[project.scripts] +my-cli = "my_package.cli:main" # callable must take no required args; parse sys.argv inside + +[dependency-groups] +dev = ["pytest>=8"] +``` + +Dev tooling goes in PEP 735 `[dependency-groups]` (written by `uv add --dev`), +not in a published extra: extras under `[project.optional-dependencies]` are +installable by consumers from PyPI, dependency groups are not. A `dev` extra +leaks your toolchain into downstream resolvers. Optional *runtime* features +(e.g. `my-package[cli]`) are what extras are for. + +### 5. Manage the project with uv + +```bash +uv init --lib my-package # new projects only (uv >= 0.12 also declares a build system for plain `uv init`; `--no-package` opts out) +uv add httpx # runtime dep -> [project.dependencies] + uv.lock +uv add --dev pytest # dev dep -> [dependency-groups].dev +uv sync # build .venv from uv.lock (--no-dev in production) +uv lock --upgrade-package httpx # targeted upgrade; rest of the lockfile intact +uv run pytest # run tools in the env, no activation +``` + +Commit `uv.lock`; never hand-edit it. uv verifies lockfile freshness against +`pyproject.toml` and environment freshness against the lockfile before every +`uv run`, which is what makes results reproducible. Never call `pip` inside a +uv project — it mutates `.venv` behind the lockfile's back. + +Migrations: + +- **From setup.py/setup.cfg** — inventory every section first: `[options]` + metadata goes to `[project]`, `package_data`/`MANIFEST.in` directives become + the backend's file-inclusion config, and tool sections (`[tool:pytest]`, + `[flake8]`, `[mypy]`) move to `pyproject.toml` or their owning skill. Build + once with the old config, once with the new, diff the wheel/sdist listings, + and only then delete `setup.py`, `setup.cfg`, and `MANIFEST.in` (steps 6–8 + verify the result). Exception: setuptools with C extensions still needs a + `setup.py` for `ext_modules` — keep only that part. +- **From Poetry** — run `uvx 'migrate-to-uv==0.12.0'` (pin the version: unpinned + `uvx` executes whatever was published to PyPI most recently, which is both + irreproducible and a supply-chain hole; bump pins deliberately). It maps + `[tool.poetry.*]` to PEP 621 and `poetry.lock` to `uv.lock`. Switch + `[build-system]` away from `poetry-core` **last**, after `uv lock && uv sync` + resolve cleanly — flipping the backend first is the classic way to debug two + migrations at once. +- **From requirements.txt** — `uv add -r requirements.txt` (plus + `-c constraints.txt` if present), then delete the file. If downstream tooling + still needs one, generate it with `uv export`. + +### 6. Validate the metadata + +```bash +uvx --from 'validate-pyproject[all]==0.26' validate-pyproject pyproject.toml +``` + +A `pyproject.toml` that parses as valid TOML can still violate the packaging +PEP schemas (bad license shape, misplaced key, malformed entry point) — this +catches what formatters and TOML parsers do not. Run it after every metadata +edit; it is cheap. + +### 7. Build + +```bash +uv build # sdist (.tar.gz) + wheel (.whl) into dist/ +``` + +To debug `uv_build` under any frontend, set `RUST_LOG=uv=debug`. No-uv fallback +(once, for environments where uv is unavailable): `python -m pip install +build==1.6.0 && python -m build` produces the same artifacts, and +`pip install -e . --group dev` (pip >= 25.1; groups are not extras) replaces `uv sync` for dev installs — at the cost of +lockfile reproducibility. + +### 8. Verify the artifacts — never skip + +Run the bundled checker (path relative to this skill's folder): + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/check_wheel.py" --project-root /path/to/project /path/to/project/dist +``` + +It is read-only and exits non-zero with `ERROR` lines if: the wheel lacks +`METADATA`/`WHEEL`/`RECORD`; any `py.typed` present in the source tree is +missing from the wheel; `tests/`, `docs/`, or similar directories leaked into +the wheel; or the sdist lacks `pyproject.toml`. Manual equivalents: +`unzip -l dist/*.whl` and `tar -tzf dist/*.tar.gz`. + +Then smoke-test the wheel itself (not the source tree): + +```bash +uv run --isolated --no-project --with dist/my_package-0.1.0-py3-none-any.whl \ + python -c "import my_package; print(my_package.__name__)" +``` + +## Output spec — what done looks like + +- `pyproject.toml` is the only packaging config; `setup.py`/`setup.cfg`/ + `MANIFEST.in`/`requirements.txt` are gone (except `setup.py` kept solely for + `ext_modules`). +- `validate-pyproject` exits 0. +- `uv build` produces both `dist/*.tar.gz` and `dist/*.whl`. +- `scripts/check_wheel.py` exits 0 — expected files (including `py.typed` when + the source ships one) verified inside the wheel; no test/doc leakage. +- The built wheel imports in an isolated environment. +- `uv.lock` exists and is committed. + +## Failure modes & gotchas + +- **`py.typed` in the repo, missing from the wheel.** No backend can be assumed + to ship arbitrary non-`.py` files: setuptools needs + `[tool.setuptools.package-data]`, hatchling ships files inside the package dir + only if the wheel target points at it, uv_build includes files under the + module root. Consumers of a wheel without the marker silently get `Any` for + your whole API — verify the artifact, not the source tree. Per-backend + snippets in [references/build-backends.md](references/build-backends.md). +- **uv_build "module not found" / metadata validation failure.** By default it + expects one module named after the normalized project name at `src//` + or `.//`. Declare a different name, several modules, or a namespace + package with `[tool.uv.build-backend]` `module-name = [...]` / `module-root` / + `namespace` (an explicit list is recommended over `namespace = true`, which + disables the safety checks). +- **Build breaks after a uv release.** Unbounded `requires = ["uv_build"]` — + restore the `>=X,/__init__.py or flat /, module named after the normalized project name)", + "uses [tool.uv.build-backend] module-name or module-root when the import name differs from the project name", + "mentions that uv_build discovers one module by default and needs module-name (a list for several modules) declared in [tool.uv.build-backend] otherwise", + "re-runs uv build to confirm the fix" + ] + } + ] +} diff --git a/skills/python-packaging/references/build-backends.md b/skills/python-packaging/references/build-backends.md new file mode 100644 index 0000000..31a7771 --- /dev/null +++ b/skills/python-packaging/references/build-backends.md @@ -0,0 +1,227 @@ +# Build backends, file inclusion, and migration paths + +Long-form companion to the python-packaging skill. Everything here targets the +user's library repo; commands are copy-runnable there. + +**Contents:** +[Decision table](#decision-table) · +[uv_build](#uv_build) · +[Hatchling](#hatchling) · +[setuptools](#setuptools) · +[py.typed and data files per backend](#pytyped-and-data-files-per-backend) · +[Migration paths](#migration-paths) · +[Legacy tool → uv command map](#legacy-tool--uv-command-map) · +[Evidence notes](#evidence-notes) · +[Sources](#sources) + +## Decision table + +| Situation | Backend | +| --- | --- | +| Pure-Python library, greenfield or modernization | `uv_build` — zero-config `src/` and flat discovery, strict metadata validation | +| Version must derive from git tags at build time | `hatchling` + `hatch-vcs` (uv_build has no plugin API for this; `uv-dynamic-versioning` exists but is hatchling-based) | +| C / Cython / Rust extension modules | `setuptools` (or `maturin`/`scikit-build-core` for Rust/CMake) — uv_build does not build extensions | +| Deep legacy setuptools config already working | keep `setuptools`, migrate metadata to `[project]` only | +| New project considering Poetry | `poetry-core` reads PEP 621 `[project]` since Poetry 2.0, so metadata is no longer the objection — the reason to prefer uv_build/hatchling is one toolchain for env + lock + build, not a metadata format | + +`uv build` is a PEP 517 **frontend**: it invokes whatever `[build-system]` +declares (hatchling, setuptools, flit-core, …). Choosing uv as project manager +and choosing the backend are independent decisions. + +## uv_build + +```toml +[build-system] +requires = ["uv_build>=0.12,<0.13"] +build-backend = "uv_build" +``` + +- **Always keep the upper bound.** uv_build follows uv's versioning policy — + breaking changes may arrive in minor releases (not strict SemVer) — so + `requires = ["uv_build"]` unbounded can break builds when a new version lands. +- **Stability**: uv_build is marked Development Status 5 — Production/Stable on + PyPI and has been the `uv init` default since mid-2025. uv overall is still + pre-1.0. Assess risk for the backend and the tool separately. +- **Layout discovery**: expects `src//__init__.py` (preferred) or + `.//`, with `` = the normalized project name + (`my-package` → `my_package`). +- **Constraint**: modules must be declared — one by default (the normalized + project name), several via `module-name = ["a", "b"]`, or a namespace via + `namespace = true` (disables safety checks; prefer the explicit list). Data + files must live under the module root. Anything the discovery cannot express + is a setuptools/hatchling job. +- **No extension modules** (C/Cython/Rust). Silently choosing uv_build for a + project with extensions is a dead end. + +Customization lives under `[tool.uv.build-backend]`: + +```toml +[tool.uv.build-backend] +module-name = "my_actual_import_name" # dots allowed for namespace packages: "foo.bar" +module-root = "" # "" = flat layout; default is "src" +data = { scripts = "data/scripts" } # extra data directories +source-include = ["docs/*.md"] # extra files for the sdist +source-exclude = ["**/*_test.py"] +wheel-exclude = ["**/*.dev.json"] +``` + +Exclusions always take precedence over inclusions. To debug uv_build when it +is driven by another frontend (`pip`, `python -m build`), set +`RUST_LOG=uv=debug` (or `RUST_LOG=uv=verbose`). Known edge case: build +isolation ignores project constraints while resolving dynamic build-backend +requirements (astral-sh/uv issue #17950). + +If uv detects TOML 1.1-only syntax in `pyproject.toml` it may reformat the file +and keep the original as `pyproject.toml.orig` — do not be surprised by that +artifact appearing. + +## Hatchling + +```toml +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/my_package"] +``` + +- With `src/` layout, point the wheel target at the package as above; hatchling + then ships everything inside that directory (including non-`.py` files such + as `py.typed`) automatically. +- Files *outside* the package directory: `packages` is `only-include` under the + hood and **ignores `include` patterns**, so an `include = [...]` next to it + silently ships nothing. Either move the assets inside the package (then they + ride along automatically) or map them explicitly: + +```toml +[tool.hatch.build.targets.wheel] +packages = ["src/my_package"] + +[tool.hatch.build.targets.wheel.force-include] +"assets/templates" = "my_package/templates" # source path -> path inside the wheel +``` + +- Dynamic versioning from git tags is hatchling's headline feature via + `hatch-vcs` — configuring the *release* process around it belongs to the + python-release skill, if installed; here it is only a backend-selection input. + +## setuptools + +```toml +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +my_package = ["py.typed", "data/*.json"] +``` + +- Constrain discovery with `where = ["src"]` (or an equivalent `include` list). + Left unconstrained on a flat layout, auto-discovery happily packages `tests/` + or `docs/` if they contain `__init__.py`. +- Non-`.py` files require `[tool.setuptools.package-data]` (with + `include-package-data` defaulting appropriately when using modern setuptools); + `MANIFEST.in` controls the *sdist* and is unnecessary for most projects once + `package-data` is set. +- `setup.py` is still legitimately required for `ext_modules` (C extensions). + Keep it for that single purpose and move every other field into `[project]`. + +## py.typed and data files per backend + +`py.typed` is an empty marker file inside the package (`src/my_package/py.typed`). +Whether to ship one, and typing strategy generally, is the python-typing skill's +domain — this table is only about getting an existing file into the wheel. +No backend ships arbitrary non-`.py` files "for free" in every configuration; +the frequent silent failure is a wheel that builds fine with the marker missing, +giving every consumer `Any` for your whole API. + +| Backend | What makes `py.typed` land in the wheel | +| --- | --- | +| `uv_build` | keep it under the module root (`src/my_package/py.typed`); `uv init --lib` scaffolds it | +| `hatchling` | keep it inside the directory listed in `[tool.hatch.build.targets.wheel] packages` | +| `setuptools` | `[tool.setuptools.package-data] my_package = ["py.typed"]` | +| `poetry-core` | an `include` entry under `[tool.poetry]` | + +Always confirm in the artifact: `unzip -l dist/*.whl | grep py.typed` or run the +bundled `scripts/check_wheel.py`, which fails if a source-tree `py.typed` is +absent from the wheel. + +## Migration paths + +### From setup.py / setup.cfg + +1. Translate metadata to `[project]` (PEP 621): `name`, `version`, + `description`, `readme`, `requires-python`, `license`, `authors`, + `dependencies`, `classifiers`, entry points to `[project.scripts]`. +2. Set `[build-system]` per the decision table. +3. Delete `setup.py`, `setup.cfg`, `MANIFEST.in` (exception: `setup.py` kept + solely for `ext_modules`). +4. `uv build`, then verify artifacts (skill step 8). Compare the wheel's file + list against one built from the old configuration if in doubt. + +### From Poetry + +1. `uvx 'migrate-to-uv==0.12.0'` — converts `[tool.poetry.dependencies]` to + `[project.dependencies]`, dev groups to `[dependency-groups]`, and + `poetry.lock` to `uv.lock`. Pin the version: unpinned `uvx` runs whatever + was most recently published, which is irreproducible and a supply-chain + exposure; review and bump pins deliberately. +2. Verify resolution: `uv lock && uv sync`, run the test suite. +3. Switch `[build-system]` away from `poetry-core` **last**. Changing the + backend before dependencies resolve is the classic mistake — you end up + debugging metadata translation and backend behavior at the same time. +4. Do not leave a repo half-migrated: uv and Poetry in one repo is a known + footgun; finish or revert. + +### From requirements.txt / pip-compile + +1. `uv add -r requirements.txt` (add `-c constraints.txt` if used) — populates + `[project.dependencies]` and `uv.lock` together. +2. Move dev-only entries with `uv add --dev`. +3. Delete `requirements*.txt`. If downstream tooling still needs one, + `uv export` emits a compatible file — generate it, never hand-maintain it. + +## Legacy tool → uv command map + +| Task | Legacy tool(s) | uv equivalent | +| --- | --- | --- | +| Create venv | `venv`, `virtualenv` | `uv venv` (auto-created on first project command) | +| Resolve/lock | `pip-tools`, `poetry lock` | `uv lock` → `uv.lock` | +| Install deps | `pip install -r requirements.txt` | `uv sync` | +| Run in env | manual `source .venv/bin/activate` | `uv run ` | +| Build sdist+wheel | `python -m build` | `uv build` | +| Run a tool once | `pipx run` | `uvx` | +| Install a global tool | `pipx install` | `uv tool install` | +| Export pins | `pip freeze` | `uv export` | + +(Version bumping and publishing — `uv version --bump`, `uv publish` — belong to +the python-release skill, if installed.) + +## Evidence notes + +- Build-speed multipliers for uv_build (for example "10–35x faster than + hatchling/setuptools") come from practitioner blog posts, not independent + controlled benchmarks. Directionally credible; do not quote the numbers as + established fact. +- "Hatchling is the current recommended default" reflects general packaging + guidance predating uv_build's stabilization; inside the uv ecosystem, + uv_build is the default for new pure-Python projects. Both statements are + true in their contexts — present the trade-off, not a universal winner. + +## Sources + +- uv build backend concepts — https://docs.astral.sh/uv/concepts/build-backend/ +- Building and publishing a package (uv docs) — https://docs.astral.sh/uv/guides/package/ +- uv versioning policy — https://docs.astral.sh/uv/reference/policies/versioning/ +- uv projects guide — https://docs.astral.sh/uv/guides/projects/ +- uv_build declared stable (byteiota) — https://byteiota.com/uv-build-backend-stable-python-packaging/ +- src layout vs flat layout — https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/ +- PEP 621 (project metadata) — https://peps.python.org/pep-0621/ +- PEP 735 (dependency groups) — https://peps.python.org/pep-0735/ +- PEP 561 (py.typed distribution) — https://peps.python.org/pep-0561/ +- validate-pyproject — https://validate-pyproject.readthedocs.io/ +- migrate-to-uv — https://github.com/mkniewallner/migrate-to-uv diff --git a/skills/python-packaging/scripts/check_wheel.py b/skills/python-packaging/scripts/check_wheel.py new file mode 100644 index 0000000..e1ddf24 --- /dev/null +++ b/skills/python-packaging/scripts/check_wheel.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Verify built Python distribution artifacts (wheel + sdist) in a dist directory. + +Read-only: never modifies the project or the artifacts. + +Checks: + - dist dir contains at least one wheel (.whl) and one sdist (.tar.gz) + - each wheel is a valid zip with exactly one *.dist-info holding + METADATA, WHEEL, and RECORD + - every py.typed found in the project source tree is present in each wheel + - no test/doc/example directories leaked into the wheel as top-level packages + - each sdist contains pyproject.toml and at least one .py file + - every --expect PATH (repeatable, exact archive path) is present in each wheel + +Output contract (machine-readable): + failures -> "ERROR : " on stderr, exit 1 + usage / IO problems -> "ERROR: " on stderr, exit 2 + success -> "PASS: ..." summary on stdout, exit 0 + +Usage: + python3 check_wheel.py [--project-root DIR] [--expect PATH]... [--no-sdist] [DIST_DIR] +""" + +# Keep full, actionable packaging guidance in this standalone checker. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import sys +import tarfile +import zipfile +from pathlib import Path + +LEAK_TOP_LEVELS = { + "tests", + "test", + "testing", + "docs", + "doc", + "examples", + "example", + "benchmarks", + "scripts", +} +SOURCE_SKIP_DIRS = { + ".git", + ".hg", + ".venv", + "venv", + ".env", + "dist", + "build", + ".tox", + ".nox", + ".eggs", + "__pycache__", + "node_modules", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + "tests", + "test", + "site-packages", +} + +errors: list[str] = [] +infos: list[str] = [] + + +def err(artifact: str, msg: str) -> None: + errors.append(f"ERROR {artifact}: {msg}") + + +def find_source_py_typed(root: Path) -> list[str]: + """Return wheel-relative archive paths for every py.typed in the source tree.""" + found: list[str] = [] + for p in sorted(root.rglob("py.typed")): + rel = p.relative_to(root) + if any(part in SOURCE_SKIP_DIRS for part in rel.parts): + continue + parts = rel.parts + if parts and parts[0] == "src": + parts = parts[1:] + if parts: + found.append("/".join(parts)) + return found + + +def wheel_top_levels(names: list[str]) -> list[str]: + tops = set() + for n in names: + first = n.split("/", 1)[0] + if first.endswith((".dist-info", ".data")) or not first: + continue + tops.add(first) + return sorted(tops) + + +def check_dist_info(label: str, names: list[str]) -> None: + dist_infos = sorted( + {n.split("/", 1)[0] for n in names if n.split("/", 1)[0].endswith(".dist-info")} + ) + if len(dist_infos) != 1: + err( + label, + f"expected exactly one *.dist-info directory, found {dist_infos or 'none'}", + ) + else: + di = dist_infos[0] + for required in ("METADATA", "WHEEL", "RECORD"): + if f"{di}/{required}" not in names: + err( + label, + f"missing {di}/{required} — wheel is malformed and pip may reject it", + ) + + +def check_top_levels(label: str, names: list[str], expected: list[str]) -> None: + tops = wheel_top_levels(names) + infos.append( + f"INFO {label}: top-level entries: {', '.join(tops) if tops else '(none)'}" + ) + if not tops: + err(label, "wheel contains no importable top-level package or module") + for leak in sorted({t.lower() for t in tops} & LEAK_TOP_LEVELS): + err( + label, + f"top-level {leak!r} leaked into the wheel — constrain package discovery (src layout or packages.find)", + ) + if len(tops) > 1: + infos.append( + f"INFO {label}: {len(tops)} top-level entries — fine for setuptools/hatchling, unsupported by uv_build" + ) + + for exp in expected: + if exp not in names: + err(label, f"expected file {exp!r} not found in wheel") + + +def check_wheel(path: Path, expected: list[str]) -> None: + label = path.name + try: + with zipfile.ZipFile(path) as zf: + bad = zf.testzip() + if bad is not None: + err(label, f"corrupt zip member {bad!r}") + return + names = zf.namelist() + except zipfile.BadZipFile as exc: + err(label, f"not a valid zip/wheel: {exc}") + return + + check_dist_info(label, names) + check_top_levels(label, names, expected) + + +def check_sdist(path: Path) -> None: + label = path.name + try: + with tarfile.open(path, "r:gz") as tf: + names = tf.getnames() + except (tarfile.TarError, OSError) as exc: + err(label, f"not a valid .tar.gz sdist: {exc}") + return + has_pyproject = any( + n.split("/")[1:] == ["pyproject.toml"] for n in names if "/" in n + ) + if not has_pyproject: + err( + label, + "sdist has no /pyproject.toml — cannot be built by PEP 517 frontends", + ) + if not any(n.endswith(".py") for n in names): + err(label, "sdist contains no .py files — source is missing from the artifact") + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "dist_dir", + nargs="?", + default="dist", + help="directory containing built artifacts (default: dist)", + ) + ap.add_argument( + "--project-root", + default=".", + help="project root to scan for py.typed parity (default: cwd)", + ) + ap.add_argument( + "--expect", + action="append", + default=[], + metavar="PATH", + help="archive path that must exist in every wheel (repeatable), e.g. my_package/py.typed", + ) + ap.add_argument( + "--no-sdist", + action="store_true", + help="do not require an sdist (wheel-only builds)", + ) + args = ap.parse_args() + + dist = Path(args.dist_dir) + root = Path(args.project_root) + if not dist.is_dir(): + print(f"ERROR: dist directory not found: {dist}", file=sys.stderr) + return 2 + if not root.is_dir(): + print(f"ERROR: project root not found: {root}", file=sys.stderr) + return 2 + + wheels = sorted(dist.glob("*.whl")) + sdists = sorted(dist.glob("*.tar.gz")) + if not wheels: + err(str(dist), "no *.whl found — run `uv build` first") + if not sdists and not args.no_sdist: + err( + str(dist), + "no *.tar.gz sdist found — `uv build` produces both; pass --no-sdist if intentional", + ) + + expected = list(args.expect) + find_source_py_typed(root) + # de-duplicate, preserve order + expected = list(dict.fromkeys(expected)) + + for whl in wheels: + check_wheel(whl, expected) + for sd in sdists: + check_sdist(sd) + + for line in infos: + print(line) + if errors: + for line in errors: + print(line, file=sys.stderr) + print( + f"FAIL: {len(errors)} error(s) across {len(wheels)} wheel(s), {len(sdists)} sdist(s)" + ) + return 1 + checked = ", ".join(p.name for p in wheels + sdists) + print( + f"PASS: verified {checked or 'nothing'}" + + (f" (required in wheel: {', '.join(expected)})" if expected else "") + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/python-pydantic-httpx-contracts/SKILL.md b/skills/python-pydantic-httpx-contracts/SKILL.md new file mode 100644 index 0000000..557ffcd --- /dev/null +++ b/skills/python-pydantic-httpx-contracts/SKILL.md @@ -0,0 +1,16 @@ +--- +name: python-pydantic-httpx-contracts +description: Maintain Pydantic request/response models and contract-tested HTTPX resource methods in the Frontal Python SDK. +--- + +# Python Models and HTTPX Contracts + +Use this skill when adding typed resource methods, changing model validation, or aligning behavior with Frontal API contracts. + +- Check `contracts/sdk-endpoints.json` and the matching OpenAPI snapshot for method, path, and declared wire fields. Do not invent public operations or model fields. +- Put boundary models in `frontal_sdk/models` using the repository's Pydantic v2 conventions; keep resource methods hand-written in `frontal_sdk/resources`. +- Preserve wire aliases and serialization behavior. Avoid weakening validation to accept undocumented payload shapes. +- Use RESPX at the HTTPX layer for offline tests. Assert request construction, typed response parsing, validation failures, and API/network error mapping. +- Run `uv run python scripts/check_contracts.py`, Ruff, strict mypy, and focused pytest coverage; README snippets are executable in CI. + +Keep `frontal_sdk/py.typed` and the supported Python versions intact. diff --git a/skills/python-supply-chain/SKILL.md b/skills/python-supply-chain/SKILL.md new file mode 100644 index 0000000..07aa58b --- /dev/null +++ b/skills/python-supply-chain/SKILL.md @@ -0,0 +1,303 @@ +--- +name: python-supply-chain +description: Hardens a Python repository's supply chain — Dependabot update automation, pip-audit vulnerability scanning, secret scanning and push protection, CodeQL, OpenSSF Scorecard, SBOMs, CODEOWNERS. Use for 'set up dependabot', 'audit dependencies', 'someone committed a key — stop that happening again', 'protect paths with CODEOWNERS', 'harden this repo'. Not for CI quality gates, workflow hardening, or PyPI publishing. +license: MIT +--- + +# python-supply-chain + +Installs layered supply-chain controls into a Python package repository on GitHub: +dependency-update automation with a freshness delay, vulnerability and secret +scanning, code scanning, ownership rules for the paths that define automation, and +SBOM/provenance artifacts. The design premise, taken from the 2025–2026 incident wave +(the Shai-Hulud npm worm, the malicious `axios` release spread by dependency bots, +mutable-tag action compromises), is that the attack surface is now *your automation +and the first hours after a release* — so no single control below is sufficient, and +bot output is treated with the same suspicion as human PRs. + +## When NOT to use + +- **CI quality-gate workflows, branch rulesets, action SHA pinning, zizmor** — the + python-ci skill, if installed, owns those. This skill only adds *scanning* and + *update* workflows, and notes where a control depends on a ruleset. +- **PyPI trusted publishing (OIDC) and the tag-triggered publish workflow** — the + python-release skill. This skill stops at repo-side SBOM/provenance; the publish + step that uploads to PyPI belongs there. +- **Pre-commit hook wiring** (including a local gitleaks hook) — the + python-precommit skill. Local hooks are advisory; this skill installs the + enforcing server/CI layer. +- **Packaging and lockfile setup** (`uv init`, `pyproject.toml`) — python-packaging. +- **Agent hooks / AGENTS.md rules** — agent-guardrails. This skill only *protects* + agent-config paths via CODEOWNERS. +- **Active incident response** beyond credential rotation basics — escalate to a + human; do not attempt autonomous cleanup of a live compromise. + +## Workflow + +Work through the steps in order; each is independently valuable, so stop where the +user's scope ends. Steps that change GitHub settings need `gh` authenticated with +admin rights on the repo — otherwise print the commands for the user to run. + +### 1. Audit the current posture + +Run the bundled read-only checker from the repo root before adding anything: + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/check_supply_chain.py" --root . # file-level checks +python3 "${CLAUDE_SKILL_DIR}/scripts/check_supply_chain.py" --root . --github # + GitHub-side settings via gh +``` + +(Invoke it from this skill's folder with an absolute path if cwd differs.) It prints +one `PASS|WARN|FAIL|NOTE check: detail` line per control and exits non-zero on FAIL. +Fix FAILs in the steps below; treat WARNs as the backlog. Never duplicate a control +that already exists — tune it instead. + +### 2. Baseline: a committed lockfile + +Every control below assumes the dependency set is pinned and committed: `uv.lock` +(preferred), `poetry.lock`/`pdm.lock`, or hash-pinned requirements +(`--generate-hashes` with pip-tools). If there is no lockfile, that is a +python-packaging problem — flag it and stop; auditing an unpinned dependency set +audits a guess. + +### 3. Dependency-update automation (Dependabot) + +Write `.github/dependabot.yml`: + +```yaml +version: 2 +updates: + - package-ecosystem: "uv" # uv.lock projects; use "pip" for requirements/pip-tools/poetry + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + cooldown: + default-days: 7 # skip versions published < 7 days ago + groups: + python-minor-patch: + update-types: ["minor", "patch"] # majors stay individual PRs + - package-ecosystem: "github-actions" # keeps actions updated; pinning policy = python-ci + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 +``` + +Key decisions, with detail in [references/dependabot.md](references/dependabot.md): + +- **Cooldown is the supply-chain control**, not the schedule: it delays adoption of + freshly published versions past the window in which malicious releases are usually + caught, and it is auto-bypassed for security-advisory updates — the reason to + prefer it over a package-manager-level freshness pin alone. Since 2026-07-14 + github.com applies a 3-day default with no config; `default-days: 7` is how you + lengthen it deliberately (and keep it aligned with `exclude-newer`, step 5). +- **Grouping minor+patch** is the accepted fix for PR-noise complaints. Keep majors + ungrouped so breaking changes get individual review. +- **Never blanket auto-merge bot PRs.** In the April 2026 malicious-`axios` incident + bots opened 154 of 214 update PRs and ~60% of bot PRs merged without human review. + Require CI plus a human review on Dependabot PRs like any other PR. +- **Renovate** is the alternative when the user needs config-as-code flexibility; + its cooldown equivalent is `minimumReleaseAge`. Trade-offs and its `pinDigest` + footgun are in the reference. Pick one bot — running both doubles the noise. +- Cooldown (`default-days`, and `semver-*-days` for pip/uv) is supported for the + pip, uv, pre-commit and github-actions ecosystems; after enabling, confirm update + PRs actually arrive on the next scheduled run. + +### 4. Vulnerability audit of the lockfile (pip-audit) + +Audit the *project's* pinned set, exported from the lockfile: + +```bash +uv export --format requirements-txt --no-emit-project -o /tmp/audit-req.txt +uvx --from pip-audit==2.10.1 pip-audit -r /tmp/audit-req.txt --no-deps +``` + +(`==2.10.1` is an example pin — check PyPI for the current release and pin that. +Unpinned `uvx` executes whatever was published minutes ago: the exact attack this +skill exists to prevent.) Plain-pip fallback, run inside the project environment: +`python -m pip install pip-audit==2.10.1 && pip-audit`. + +- Do NOT run bare `uvx pip-audit` with no `-r`: it audits pip-audit's own ephemeral + environment, not your project, and reports a clean bill for the wrong thing. +- `--no-deps` skips re-resolution and is correct because the export is fully pinned. +- Report findings with the fix version per package; do not blind-upgrade everything, + and avoid `pip-audit --fix` in automation (it mutates the environment, not the + lockfile). +- Alternatives: `osv-scanner` (Go binary) reads `uv.lock` directly; `uv audit` + (preview since June 2026, OSV-backed, lockfile-aware) is the zero-install option + once it stabilizes — pin the uv version you verified it with. +- Recurrence: Dependabot security updates cover the alerting side; add a scheduled + scan workflow only if the user wants audit output in CI (workflow shape in + [references/scanning-and-provenance.md](references/scanning-and-provenance.md)). + +### 5. Resolution freshness window (uv projects) + +For uv projects, add a resolver-level buffer against just-published packages: + +```toml +[tool.uv] +exclude-newer = "2026-06-27T00:00:00Z" # resolver ignores anything published after this +``` + +uv also accepts a relative window — `exclude-newer = "7 days"` (or ISO 8601 +`P7D`) — and `exclude-newer-package` scopes the cutoff per package; run `uv lock` +after setting it. The cutoff compares each *artifact's* upload time, not the +release date, so a wheel uploaded late is filtered on its own. +Two gotchas: a fixed timestamp freezes resolution in the past until someone bumps it +(add it to the release checklist), and unlike Dependabot's cooldown it delays +security fixes too — which is why you run *both*: `exclude-newer` guards `uv lock` +runs on laptops and CI, cooldown guards the bot, security advisories bypass the bot's +cooldown. Reference: . + +### 6. Secret scanning, push protection, and a CI backstop + +GitHub-side scanning is free and on by default for public repos; private repos need +GitHub Secret Protection licensing. Enable/verify both toggles: + +```bash +gh api -X PATCH "repos/{owner}/{repo}" --input - <<'EOF' +{ "security_and_analysis": { + "secret_scanning": { "status": "enabled" }, + "secret_scanning_push_protection": { "status": "enabled" } } } +EOF +``` + +Push protection blocks a detected secret *before it enters history* — strictly +stronger than any after-the-fact scan — but GitHub documents that it only blocks a +subset of high-confidence patterns. Add a CI gitleaks scan as the backstop that also +catches `--no-verify` bypasses of local hooks (workflow YAML, plus the +gitleaks-action org-license gotcha, in the reference). If a secret is ever found: +**rotate the credential first**; history rewriting does not un-leak anything already +cloned, forked, or cached. + +### 7. Code scanning (CodeQL) + +Default setup is enough for pure-Python repos (no build step needed): + +```bash +gh api -X PATCH "repos/{owner}/{repo}/code-scanning/default-setup" -f state=configured +``` + +Free for public repos; private repos need GitHub Code Security licensing. Use the +advanced workflow (in the reference) only when the user needs query-pack control or +`language: actions` coverage of workflow files. Making code-scanning results block +merges is a ruleset — hand that to python-ci. + +### 8. CODEOWNERS for the paths that define automation + +``` +# .github/CODEOWNERS — LAST matching rule wins (not first) +* @OWNER/maintainers +/.github/ @OWNER/platform-security +/.github/CODEOWNERS @OWNER/platform-security +/AGENTS.md @OWNER/platform-security +/CLAUDE.md @OWNER/platform-security +/.claude/ @OWNER/platform-security +``` + +Rationale: whoever can edit workflows, `dependabot.yml`, or agent instruction files +can bypass every other control, so those paths need a stricter owner than the code. +Two silent failures: CODEOWNERS enforces nothing until "require review from code +owners" is enabled in branch protection/rulesets (python-ci's territory — say so in +the handoff), and an owner without write access is ignored without error — check the +file's rendered view on GitHub for syntax/owner errors. + +### 9. Posture dashboard (OpenSSF Scorecard, optional) + +Add the `ossf/scorecard-action` weekly workflow (YAML in the reference) to get a +0–10 score across ~a dozen heuristics with SARIF results in the Security tab. Treat +it as a triage dashboard, not a gate: its own maintainers call the checks heuristic, +and chasing the number (e.g. pinning SHAs no one audited) is documented security +theater. Public repos can check their existing score at + before adding anything. + +### 10. SBOM and release provenance + +Two SBOM paths — pick per audience: + +```bash +# Repo-level SPDX from GitHub's dependency graph (compliance requests): +gh api "repos/{owner}/{repo}/dependency-graph/sbom" --jq .sbom > sbom.spdx.json + +# Build-time CycloneDX from the lockfile (ship next to the wheel): +uv export --format requirements-txt --no-emit-project -o /tmp/sbom-req.txt +uvx --from cyclonedx-bom==7.3.1 cyclonedx-py requirements /tmp/sbom-req.txt -o sbom.cdx.json +``` + +For releases, attest the built artifacts with `actions/attest-build-provenance` +(and `actions/attest-sbom` for the SBOM) inside the release workflow — snippet and +required permissions in the reference. The publish workflow itself, and PyPI-side +attestations via trusted publishing, belong to the python-release skill. Tell the +user the honest limit: an attestation proves *where and how* an artifact was built, +not that it is safe — real campaigns have shipped validly-signed malicious builds — +and it only matters if consumers verify (`gh attestation verify dist/pkg.whl +--repo OWNER/REPO`). + +### 11. Re-audit and hand off + +Re-run step 1; the script should report no FAILs. List remaining WARN/NOTE items +with a one-line reason each (deliberate skip vs. needs licensing vs. sibling-skill +territory). + +## Output spec — what done looks like + +- `.github/dependabot.yml` with a pip or uv ecosystem entry, weekly schedule, + cooldown, grouped minor/patch updates, and a github-actions entry. +- Committed lockfile; a documented, pinned pip-audit invocation (or scheduled scan). +- Secret scanning + push protection enabled (or exact commands handed to the user + when permissions/licensing block it), plus a CI gitleaks backstop. +- CodeQL enabled (default setup or committed workflow). +- CODEOWNERS covering `/.github/` and agent-config paths, with the + branch-protection dependency stated in the handoff. +- Optional per scope: Scorecard workflow, SBOM artifact, attestation step wired into + the release workflow (or delegated to python-release). +- `scripts/check_supply_chain.py --root .` exits 0. + +## Failure modes and gotchas + +- **`uvx pip-audit` without `-r` audits the wrong environment** (the tool's own + ephemeral venv) and happily reports zero findings. +- **Dependabot cooldown can wedge frequently-released dependencies**: the filter has + evaluated only the latest release, so a dep that ships weekly may never propose an + update (dependabot-core #14234/#14579; partially fixed — verify current behavior). + Symptom: a dependency silently months stale despite Dependabot running green. +- **Cooldown at the package-manager level blocks security fixes**; Dependabot's + cooldown does not (advisory updates bypass it). Don't replace one with the other. +- **Cooldown is not proven for transitive dependencies** (confirmed gap for npm in + dependabot-core #14683; unverified for pip/uv) — the lockfile audit in step 4 is + the control that actually covers transitives. +- **Bot-authored PRs carry unearned trust**: attackers mimic Dependabot branding, + and auto-merge-on-green shipped malware in the axios incident. Review like any PR. +- **Dependabot does not alert on SHA-pinned actions** (only semver tags) — a known + gap between this skill's alerting and python-ci's pinning policy; the dependabot + `github-actions` entry still proposes pin bumps, so keep it. +- **gitleaks-action requires a license key for organization repos** + (`GITLEAKS_LICENSE` secret; free for personal accounts) — a silent-failure setup + step people miss. +- **CODEOWNERS is last-match-wins** — a trailing `*` rule overrides every protection + above it. Put the catch-all first, specific security paths after. +- **Push protection is a subset filter, not a guarantee** — keep the CI backstop. +- **Scorecard is gameable and heuristic** — never wire it as a merge gate; its + public weekly scan omits several checks (CI-Tests, Contributors, + Dependency-Update-Tool) for API-cost reasons, so scores differ from local runs. +- **Installing a Python package executes code** — at install time (build backends, + `setup.py`) and import time. Never install a package to inspect it; read it on + first. Watch for typosquats (`request` vs `requests`) + and for hallucinated package names in LLM suggestions (slopsquatting) — verify + the exact name, repo link, and release history on pypi.org before `uv add`. +- **Free-tier boundary**: dependency graph, Dependabot alerts and security updates + are free everywhere; secret scanning and CodeQL are free for *public* repos only — + on private repos, hand the user the licensing decision instead of a broken toggle. + +## Files + +- [references/dependabot.md](references/dependabot.md) — full annotated + dependabot.yml, cooldown semantics and bug detail, Renovate comparison, + noise-tuning, auto-merge policy evidence. +- [references/scanning-and-provenance.md](references/scanning-and-provenance.md) — + gitleaks/TruffleHog CI workflows, advanced CodeQL workflow, Scorecard workflow, + SBOM/attestation snippets with permissions, verification commands. +- [scripts/check_supply_chain.py](scripts/check_supply_chain.py) — read-only posture + audit; `--github` adds GitHub-settings checks via `gh`; exits non-zero on FAIL. diff --git a/skills/python-supply-chain/evals/evals.json b/skills/python-supply-chain/evals/evals.json new file mode 100644 index 0000000..73c490f --- /dev/null +++ b/skills/python-supply-chain/evals/evals.json @@ -0,0 +1,80 @@ +{ + "skill": "python-supply-chain", + "cases": [ + { "type": "should_trigger", "prompt": "set up dependabot for this repo" }, + { "type": "should_trigger", "prompt": "harden this python repo against supply chain attacks" }, + { "type": "should_trigger", "prompt": "audit our dependencies for known vulnerabilities" }, + { "type": "should_trigger", "prompt": "enable secret scanning and push protection on this repository" }, + { "type": "should_trigger", "prompt": "dependabot is spamming us with update PRs, group them and add a cooldown" }, + { "type": "should_trigger", "prompt": "can u add a sbom to our release artifacts" }, + { "type": "should_trigger", "prompt": "setup codeql scannig for this python package" }, + { "type": "should_trigger", "prompt": "add openssf scorecard" }, + { "type": "should_trigger", "prompt": "someone committed an AWS key last month. make sure that never happens again" }, + { "type": "should_trigger", "prompt": "what's the minimum security setup for a public python repo?" }, + { "type": "should_trigger", "prompt": "protect .github and our agent config files with CODEOWNERS" }, + { "type": "should_trigger", "prompt": "should we use renovate or dependabot for a uv project?" }, + { "type": "should_trigger", "prompt": "hold newly published pypi releases for a week before our builds pull them in" }, + { "type": "should_not_trigger", "prompt": "pin our github actions to commit SHAs" }, + { "type": "should_not_trigger", "prompt": "run zizmor on the workflows and fix what it finds" }, + { "type": "should_not_trigger", "prompt": "set up trusted publishing so we can publish to PyPI without tokens" }, + { "type": "should_not_trigger", "prompt": "add a gitleaks pre-commit hook" }, + { "type": "should_not_trigger", "prompt": "set up CI with lint, type check and test jobs" }, + { "type": "should_not_trigger", "prompt": "make the checks required and add a branch ruleset for main" }, + { "type": "should_not_trigger", "prompt": "switch this project to uv and commit the lockfile" }, + { "type": "should_not_trigger", "prompt": "there's a hardcoded password in config.py, refactor it to read from the environment" }, + { "type": "should_not_trigger", "prompt": "upgrade fastapi to the latest version and fix the breaking changes" }, + { "type": "should_not_trigger", "prompt": "cut a release, bump the version and update the changelog" }, + { + "type": "quality", + "prompt": "harden this repo's supply chain", + "expected_behavior": [ + "audits the current posture first (e.g. runs the bundled check script or inspects .github/) instead of blindly adding files", + "creates .github/dependabot.yml with a pip or uv ecosystem entry, a weekly schedule, grouping, and a cooldown", + "enables or gives exact commands to enable secret scanning and push protection, and sets up CodeQL scanning", + "adds a CODEOWNERS rule covering .github/ and agent config paths", + "does not recommend blanket auto-merge of Dependabot/Renovate PRs and says why" + ] + }, + { + "type": "quality", + "prompt": "set up dependabot with a cooldown for our uv project", + "expected_behavior": [ + "uses the uv package ecosystem for a uv.lock project (or pip with a stated reason) rather than only npm-style examples", + "configures cooldown with default-days and explains that security-advisory updates bypass the cooldown", + "warns about the known cooldown bug where frequently-released dependencies can get stuck because only the latest release is evaluated", + "groups minor and patch updates to cut PR noise and sets a sane open-pull-requests-limit", + "does not enable auto-merge for the bot's PRs" + ] + }, + { + "type": "quality", + "prompt": "audit our dependencies for vulnerabilities", + "expected_behavior": [ + "audits the project's real dependency set (lockfile or exported pinned requirements), not the tool's own ephemeral environment", + "pins the audit tool version when invoking it via uvx or pip", + "reports findings with affected versions and available fix versions instead of blind-upgrading everything", + "offers a recurring option (scheduled workflow or Dependabot security updates) so the audit is not one-shot" + ] + }, + { + "type": "quality", + "prompt": "generate an SBOM and attach provenance to our releases", + "expected_behavior": [ + "generates a CycloneDX or SPDX SBOM from the lockfile or environment with a pinned tool, or uses GitHub's dependency-graph SBOM export", + "wires SBOM/provenance attestation into the release workflow or defers the publish workflow itself to the python-release skill", + "states that an attestation proves where an artifact was built, not that it is safe, and shows how consumers verify it", + "mentions the permissions the attestation step needs (id-token and attestations write)" + ] + }, + { + "type": "quality", + "prompt": "we found an AWS key in our git history", + "expected_behavior": [ + "instructs rotating/revoking the credential first, before any history rewriting", + "explains that rewriting history does not un-leak a secret already cloned, forked, or cached", + "enables push protection and secret scanning so the class of incident is blocked going forward", + "adds a CI secret-scan backstop (e.g. gitleaks) that fails the build on new findings" + ] + } + ] +} diff --git a/skills/python-supply-chain/references/dependabot.md b/skills/python-supply-chain/references/dependabot.md new file mode 100644 index 0000000..d8ff586 --- /dev/null +++ b/skills/python-supply-chain/references/dependabot.md @@ -0,0 +1,191 @@ +# Dependabot for Python repos — config, cooldown, and the trust model + +**Contents:** [Full config](#full-annotated-config) · +[Ecosystem choice](#ecosystem-choice-uv-vs-pip) · +[Cooldown semantics](#cooldown-semantics) · +[Cooldown bugs](#known-cooldown-bugs) · +[Grouping and noise tuning](#grouping-and-noise-tuning) · +[Auto-merge policy](#why-not-auto-merge-bot-prs) · +[Renovate comparison](#renovate-as-the-alternative) · +[Interplay with uv exclude-newer](#interplay-with-uv-exclude-newer) · +[SHA-pinned action alert gap](#the-sha-pinned-action-alert-gap) · +[Free-tier facts](#free-tier-facts) + +## Full annotated config + +```yaml +# .github/dependabot.yml +version: 2 +updates: + # --- Python dependencies --- + - package-ecosystem: "uv" # for projects managed by uv.lock + directory: "/" # where pyproject.toml lives + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + timezone: "Etc/UTC" + open-pull-requests-limit: 10 # default is 5; raise, don't disable + cooldown: + default-days: 7 # skip versions published in the last 7 days + semver-major-days: 14 # be slower on majors + semver-minor-days: 7 + semver-patch-days: 3 + groups: + python-minor-patch: + update-types: ["minor", "patch"] + # majors intentionally ungrouped: breaking changes get their own PR + labels: ["dependencies"] + + # --- GitHub Actions used by the repo's workflows --- + - package-ecosystem: "github-actions" + directory: "/" # reads .github/workflows/ + schedule: + interval: "weekly" + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] +``` + +Commit the file; Dependabot picks it up on the next scheduled run (or trigger a run +from Insights → Dependency graph → Dependabot). Version *updates* (this file) are +distinct from Dependabot *alerts* and *security updates*, which are repo settings: + +```bash +gh api -X PUT "repos/{owner}/{repo}/vulnerability-alerts" # enable alerts +gh api -X PUT "repos/{owner}/{repo}/automated-security-fixes" # enable security PRs +``` + +## Ecosystem choice: uv vs pip + +- `package-ecosystem: "uv"` — projects with a `uv.lock`. Dependabot updates the + lockfile and `pyproject.toml` constraints together. +- `package-ecosystem: "pip"` — `requirements*.txt`, pip-tools output, Pipenv, and + Poetry projects (the `pip` value covers all of these). +- The uv ecosystem supports `cooldown` (`default-days` and the `semver-*-days` + variants) like pip; the public bug history below is mostly from other + ecosystems. After enabling, verify on the next scheduled run that PRs actually + appear for a known-stale dependency before trusting it. + +## Cooldown semantics + +`cooldown` delays adoption of a freshly published version — the window in which +malicious releases (account takeovers, typosquats promoted by download counts) are +usually detected and yanked. Options per update entry: + +- `default-days` — applies to everything not matched by a more specific option. +- `semver-major-days` / `semver-minor-days` / `semver-patch-days` — severity-aware + delays (only meaningful for ecosystems where semver intent is knowable). +- `include` / `exclude` — lists of dependency name patterns to scope the cooldown. + +Since 2026-07-14 github.com applies a **3-day default cooldown** to version updates +even with no `cooldown:` block (security updates still open immediately; GHES adoption +is version-dependent) — so an explicit `default-days` sets a longer or scoped window, +it does not create the control from nothing +(). + +The property that makes Dependabot's cooldown the right layer for this control: +**security-advisory-driven updates bypass the cooldown automatically.** A freshness +window enforced in the package manager (see `exclude-newer` below) cannot make that +distinction and will delay security fixes by the same number of days. + +Announcement: + +## Known cooldown bugs + +- **Latest-release-only evaluation** — the filter has been observed to evaluate only + the newest release: if that one is inside the cooldown window, Dependabot proposes + nothing at all, instead of falling back to an older release that satisfies the + window. Frequently-released dependencies can be wedged for months. + [dependabot-core #14234](https://github.com/dependabot/dependabot-core/issues/14234), + [#14579](https://github.com/dependabot/dependabot-core/issues/14579) (#14579 was + closed with a fix for github-actions; verify current behavior for your ecosystem). + Detection: a dependency that stays stale across several green Dependabot runs. +- **No transitive coverage (confirmed for npm)** — cooldown does not apply to + transitive npm dependencies at all + ([dependabot-core #14683](https://github.com/dependabot/dependabot-core/issues/14683), + maintainer-confirmed). Whether the pip/uv updaters share the gap is unverified — + assume cooldown covers direct dependencies only, and rely on lockfile auditing + (pip-audit/osv-scanner) for transitives. + +## Grouping and noise tuning + +Alert fatigue is the top practitioner complaint about Dependabot ("PR tsunamis"). +The levers, in order of impact: + +1. **Groups**: one PR per group per run. Group `minor` + `patch`; leave majors + individual. Grouping by `patterns` (name globs) also works for families like + `pytest*`. +2. **Cooldown**: fewer, older, safer versions per run. +3. **Schedule**: `weekly` beats `daily` for humans; `monthly` risks large diffs. +4. **`open-pull-requests-limit`**: caps concurrent PRs (default 5). Setting `0` + disables version updates entirely — don't; use security-only mode deliberately + instead by removing the entry and keeping alerts + security updates on. +5. **Metrics view**: for alert triage, GitHub's security-alert metrics rank by + CVSS/EPSS/patch availability — point users there rather than muting alerts. + Dependabot treats every alert as equally urgent regardless of reachability (a + known critique), so triage is human work. + +## Why not auto-merge bot PRs + +Evidence from the April 2026 malicious-`axios` incident +(): +bots opened 154 of 214 update PRs spreading the malicious version; ~60% of +bot-opened PRs merged without human review; 50 were fully auto-merged. Auto-merge on +green CI shipped malware *faster* than manual review would have. Separately, +attackers have crafted PRs that mimic Dependabot's branding to exploit the implicit +trust in bot PRs (). + +Policy to recommend: bot PRs get CI + human review like any PR. If the user insists +on auto-merge, confine it to grouped patch updates of dev-only dependencies, with +cooldown enabled — and record that this was their call, with the incident above as +the counter-argument. + +## Renovate as the alternative + +| Dimension | Dependabot | Renovate | +|---|---|---| +| Hosting | native to GitHub, zero install | GitHub App (Mend-hosted) or self-hosted | +| Freshness delay | `cooldown` (advisories bypass it) | `minimumReleaseAge` | +| Config | one YAML, limited surface | JSON5 config-as-code, presets, `packageRules` | +| Grouping | `groups` | richer (monorepo presets, scheduling per rule) | +| Lockfile-only refresh | no | `lockFileMaintenance` | + +Pick Renovate when the user needs per-package policies or already runs it elsewhere; +otherwise Dependabot's native integration wins on setup cost. Never run both. + +**Renovate footgun**: the `pinDigest` update type can silently roll a pinned commit +SHA forward to a new commit while keeping the same version tag — quietly defeating +SHA-pinning unless disabled via `packageRules`. If the repo pins actions by SHA +(python-ci's policy), audit the Renovate config for this before enabling. + +## Interplay with uv exclude-newer + +`[tool.uv] exclude-newer` (settled form: an RFC 3339 timestamp; recent uv releases +also accept relative windows like `"7 days"` — verify with `uv lock`) makes the +*resolver* ignore anything published after the cutoff. It guards every `uv lock` / +`uv sync --upgrade` run, including ones a coding agent triggers on a laptop — which +Dependabot's cooldown cannot see. Run both: + +- `exclude-newer` → freshness floor for all local/CI resolution; delays security + fixes too, so keep the window short (3–7 days) or bump the timestamp routinely. +- Dependabot cooldown → freshness floor for bot PRs; advisories bypass it. + +Docs: + +## The SHA-pinned action alert gap + +Dependabot *alerts* fire only for actions pinned by semver tag, not by commit SHA +(documented gap: GitHub community discussions #154189 and #125481) — so the more +secure pinning pattern loses vulnerability visibility, with no free first-party fix. +Mitigation: keep the `github-actions` ecosystem entry in `dependabot.yml` — version +*updates* still propose bumps for SHA-pinned actions (with the tag comment updated) — +and treat pinning policy/auditing as python-ci territory. + +## Free-tier facts + +Dependency graph, Dependabot alerts, and Dependabot security updates are free for +all repos, public and private. (Secret scanning and CodeQL are free for public repos +only — see the scanning reference.) diff --git a/skills/python-supply-chain/references/scanning-and-provenance.md b/skills/python-supply-chain/references/scanning-and-provenance.md new file mode 100644 index 0000000..3adfa93 --- /dev/null +++ b/skills/python-supply-chain/references/scanning-and-provenance.md @@ -0,0 +1,261 @@ +# Scanning workflows and provenance artifacts + +**Contents:** [Action pinning note](#action-pinning-note) · +[Secret scanning and push protection](#secret-scanning-and-push-protection) · +[gitleaks CI backstop](#gitleaks-ci-backstop) · +[Secret-found playbook](#secret-found-playbook) · +[CodeQL](#codeql) · [Scheduled pip-audit](#scheduled-pip-audit-workflow) · +[OpenSSF Scorecard](#openssf-scorecard) · [SBOM generation](#sbom-generation) · +[Release attestations](#release-attestations) · +[Verification — the part everyone skips](#verification--the-part-everyone-skips) + +## Action pinning note + +Every `uses:` below is written as `owner/action@ # vX.Y.Z` with a +placeholder. Before committing, resolve the full commit SHA of the current release +tag and substitute it: + +```bash +gh api "repos///commits/" --jq .sha +``` + +Mutable tags (`@v2`) have been repointed to malicious commits in real attacks +(GhostAction, tj-actions). The repo-wide pinning policy and its audit tooling +(zizmor) belong to the python-ci skill — this note exists so the workflows *this* +skill adds don't arrive unpinned. + +## Secret scanning and push protection + +State of the platform: 200+ provider detectors, partner auto-revocation, validity +checks, and org-wide enforced enablement. Free and on by default for public repos; +private repos require GitHub Secret Protection licensing. + +```bash +# Enable/verify both toggles (needs admin): +gh api -X PATCH "repos/{owner}/{repo}" --input - <<'EOF' +{ "security_and_analysis": { + "secret_scanning": { "status": "enabled" }, + "secret_scanning_push_protection": { "status": "enabled" } } } +EOF + +# Check current state: +gh api "repos/{owner}/{repo}" --jq .security_and_analysis +``` + +Push protection blocks a detected secret before the push lands — the secret never +enters history. Caveat GitHub itself documents: it blocks only a subset of +high-confidence patterns. That is why the CI backstop below stays even when push +protection is on. Background reading: GitHub's own triage playbook, +. + +## gitleaks CI backstop + +Catches generic/entropy patterns push protection misses and any local-hook bypass +(`git commit --no-verify`). A local gitleaks pre-commit hook is good fast feedback +but advisory — wiring it is the python-precommit skill's job; CI is the boundary. + +```yaml +# .github/workflows/secret-scan.yml +name: secret-scan +on: [push, pull_request] +permissions: + contents: read +jobs: + gitleaks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ # vX.Y.Z — see pinning note + with: + fetch-depth: 0 # full history: scan every commit in the push + - uses: gitleaks/gitleaks-action@ # vX.Y.Z + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} # required for ORG repos +``` + +- **Org-repo gotcha**: gitleaks-action requires a `GITLEAKS_LICENSE` key for + organization-owned repos (free for personal accounts). Without it the job fails — + better than silently passing, but plan for it. +- Alternative/addition: TruffleHog verifies candidate secrets against provider APIs + so it only reports *live* credentials — strong signal, fewer false positives: + `trufflehog git file://. --since-commit --results=verified --fail`. + +## Secret-found playbook + +1. **Rotate/revoke the credential immediately.** This is the fix; everything else + is cleanup. Assume the secret is compromised the moment it was pushed. +2. Only then consider history rewriting (`git filter-repo`) — and say plainly that + rewriting does not un-leak: clones, forks, caches, and scrapers already have it. +3. Close the alert in the Security tab with the resolution recorded. +4. Confirm push protection + the CI backstop are on so the class recurs as a block, + not an incident. + +## CodeQL + +Default setup — right answer for pure-Python repos, no workflow file to maintain: + +```bash +gh api -X PATCH "repos/{owner}/{repo}/code-scanning/default-setup" -f state=configured +gh api "repos/{owner}/{repo}/code-scanning/default-setup" # verify +``` + +Advanced setup, only when query packs or workflow-file scanning are needed: + +```yaml +# .github/workflows/codeql.yml +name: codeql +on: + push: { branches: [main] } + pull_request: { branches: [main] } + schedule: [{ cron: "0 4 * * 1" }] +permissions: + contents: read +jobs: + analyze: + runs-on: ubuntu-latest + permissions: + security-events: write + strategy: + matrix: + language: [python, actions] # 'actions' scans the workflow files themselves + steps: + - uses: actions/checkout@ # vX.Y.Z + - uses: github/codeql-action/init@ # vX.Y.Z + with: + languages: ${{ matrix.language }} + - uses: github/codeql-action/analyze@ # vX.Y.Z +``` + +Notes: Python needs no build step. Free for public repos; private needs GitHub Code +Security. Blocking merges on code-scanning severity is done via repository rulesets — +python-ci's territory. A November 2025 platform change makes default setup bypass +restrictive org Actions policies (unless Actions is fully disabled), so coverage +isn't silently lost — worth knowing when an org policy seems contradicted. + +## Scheduled pip-audit workflow + +Only add when the user wants recurring audit output in CI (Dependabot security +updates already cover alerting). Scan workflow, not a quality gate: + +```yaml +# .github/workflows/dep-audit.yml +name: dep-audit +on: + schedule: [{ cron: "0 5 * * 1" }] + workflow_dispatch: +permissions: + contents: read +jobs: + pip-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ # vX.Y.Z + - uses: astral-sh/setup-uv@ # vX.Y.Z + - run: uv export --format requirements-txt --no-emit-project -o audit-req.txt + - run: uvx --from pip-audit==2.10.1 pip-audit -r audit-req.txt --no-deps + # ==2.7.3 is an example pin — substitute the current PyPI release +``` + +## OpenSSF Scorecard + +```yaml +# .github/workflows/scorecard.yml +name: scorecard +on: + schedule: [{ cron: "30 2 * * 1" }] + push: { branches: [main] } +permissions: read-all +jobs: + analysis: + runs-on: ubuntu-latest + permissions: + security-events: write # SARIF upload + id-token: write # publish_results signing + steps: + - uses: actions/checkout@ # vX.Y.Z + with: + persist-credentials: false + - uses: ossf/scorecard-action@ # vX.Y.Z + with: + results_file: results.sarif + results_format: sarif + publish_results: true # feeds the public scorecard.dev API/badge + - uses: github/codeql-action/upload-sarif@ # vX.Y.Z + with: + sarif_file: results.sarif +``` + +How to frame it for the user: a 0–10 heuristic posture dashboard +(), useful for spotting +which controls are off and for downstream consumers evaluating the project. Not a +gate: maintainers call the checks heuristic; scores are gameable (SHA-pinning purely +for points); and the public weekly scan omits CI-Tests, Contributors, and +Dependency-Update-Tool checks for API-cost reasons, so it reads lower than a local +run. A large academic sample of research repos averaged 3.5/10 — low scores are the +norm, not an emergency. + +## SBOM generation + +| Path | Produces | Use when | +|---|---|---| +| `gh api "repos/{owner}/{repo}/dependency-graph/sbom" --jq .sbom` | SPDX JSON of the repo's dependency graph | compliance asks "send us an SBOM" | +| `cyclonedx-py` from the exported lockfile | CycloneDX of the *built artifact's* deps | shipping an SBOM next to the wheel | +| `syft` | either format, broader ecosystem coverage | container images / mixed repos | + +```bash +uv export --format requirements-txt --no-emit-project -o /tmp/sbom-req.txt +uvx --from cyclonedx-bom==7.3.1 cyclonedx-py requirements /tmp/sbom-req.txt -o sbom.cdx.json +# ==4.4.3 is an example pin — substitute the current PyPI release +``` + +An SBOM is inventory, not assurance: it says what's inside, not whether the build +that produced it was compromised, and it only becomes actionable when cross-checked +against a live vulnerability feed (OSV/NVD — pip-audit and osv-scanner do exactly +that). Regulatory drivers making SBOMs table stakes for commercial consumers: US EO +14028 and the EU Cyber Resilience Act. + +## Release attestations + +Inside the release workflow (the workflow itself — tag trigger, build, PyPI publish +via trusted publishing — is the python-release skill's territory): + +```yaml +permissions: + contents: read + id-token: write # OIDC identity for signing + attestations: write +steps: + # ... build step producing dist/ ... + - uses: actions/attest-build-provenance@ # vX.Y.Z + with: + subject-path: "dist/*" + - uses: actions/attest-sbom@ # vX.Y.Z + with: + subject-path: "dist/*" + sbom-path: "sbom.cdx.json" +``` + +GitHub Artifact Attestations are built on Sigstore; the workflow's OIDC identity is +the ephemeral signing key (docs: +). +Known limits to state up front: provenance from private repos is restricted +(enterprise-gated — check current docs), trusted-publishing/attestation flows don't +cover self-hosted runners or non-GitHub CI, and OIDC reduces but does not eliminate +token risk — a compromised workflow run can still mint valid short-lived credentials. + +## Verification — the part everyone skips + +An attestation proves *where and how* an artifact was built, not that it is safe: +real campaigns have produced cryptographically valid SLSA Build L3 attestations for +malicious packages by compromising the pipeline itself ("The signature is real. The +software is not." — +). +Two consequences: + +1. Consumers must actually verify: + `gh attestation verify dist/pkg-1.0.0-py3-none-any.whl --repo OWNER/REPO` + — put this line in the release notes / README so the artifact's verifiability is + discoverable. +2. The controls that protect the *pipeline* (workflow permissions, pinning, rulesets + — python-ci) are what give the attestation meaning. Provenance without pipeline + hardening is a signed receipt from a burgled shop. diff --git a/skills/python-supply-chain/scripts/check_supply_chain.py b/skills/python-supply-chain/scripts/check_supply_chain.py new file mode 100644 index 0000000..89d7b88 --- /dev/null +++ b/skills/python-supply-chain/scripts/check_supply_chain.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +"""Read-only supply-chain posture audit for a Python repository. + +Checks for the presence and basic shape of supply-chain controls: Dependabot +config, lockfile, CODEOWNERS coverage, scanning workflows, security policy and +(with --github) GitHub-side settings via the `gh` CLI. Never modifies anything. + +Usage: + python3 check_supply_chain.py --root /path/to/repo + python3 check_supply_chain.py --root . --github # also query repo settings + python3 check_supply_chain.py --root . --strict # WARN counts as FAIL + +Output: one `LEVEL check-id: detail` line per check, where LEVEL is +PASS | WARN | FAIL | NOTE, then a RESULT summary line. +Exit codes: 0 = no FAILs (WARNs allowed unless --strict), 1 = FAILs present, +2 = usage/environment error (bad root, gh missing with --github). + +Python 3.10+, stdlib only. YAML/TOML are matched with conservative regexes on +purpose: no third-party parser, and a miss degrades to WARN, never a crash. +""" + +# Keep full, actionable supply-chain guidance in this standalone checker. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +results: list[tuple[str, str, str]] = [] # (level, check, detail) +GITHUB_DIR = ".github" + + +def report(level: str, check: str, detail: str) -> None: + results.append((level, check, detail)) + + +def read(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + +def workflow_texts(root: Path) -> dict[str, str]: + wf_dir = root / GITHUB_DIR / "workflows" + if not wf_dir.is_dir(): + return {} + return { + p.name: read(p) + for p in sorted(wf_dir.iterdir()) + if p.suffix in (".yml", ".yaml") and p.is_file() + } + + +def check_dependabot(root: Path) -> None: + cfg = None + for name in ("dependabot.yml", "dependabot.yaml"): + candidate = root / GITHUB_DIR / name + if candidate.is_file(): + cfg = candidate + break + if cfg is None: + report( + "FAIL", + "dependabot-config", + ".github/dependabot.yml not found — no dependency-update automation", + ) + return + text = read(cfg) + ecosystems = re.findall(r"package-ecosystem:\s*[\"']?([A-Za-z-]+)", text) + if any(e in ("pip", "uv") for e in ecosystems): + report( + "PASS", + "dependabot-python", + f"{cfg.name} covers a Python ecosystem ({', '.join(sorted(set(ecosystems)))})", + ) + else: + report( + "FAIL", + "dependabot-python", + f"{cfg.name} has no pip/uv entry (found: {sorted(set(ecosystems)) or 'none'})", + ) + if "github-actions" in ecosystems: + report("PASS", "dependabot-actions", "github-actions ecosystem entry present") + else: + report( + "WARN", + "dependabot-actions", + "no github-actions entry — action versions won't get update PRs", + ) + if re.search(r"^\s*cooldown\s*:", text, re.MULTILINE): + report("PASS", "dependabot-cooldown", "cooldown configured") + else: + report( + "WARN", + "dependabot-cooldown", + "no cooldown — freshly published (possibly malicious) versions are adopted immediately", + ) + if re.search(r"^\s*groups\s*:", text, re.MULTILINE): + report("PASS", "dependabot-groups", "update grouping configured") + else: + report( + "WARN", + "dependabot-groups", + "no groups — expect one PR per dependency (noise breeds auto-merge)", + ) + + +def check_lockfile(root: Path) -> None: + for name in ("uv.lock", "poetry.lock", "pdm.lock", "Pipfile.lock"): + if (root / name).is_file(): + report("PASS", "lockfile", f"{name} present") + return + reqs = sorted(root.glob("requirements*.txt")) + sorted( + root.glob("requirements/*.txt") + ) + if reqs: + if any("--hash=" in read(r) for r in reqs): + report( + "PASS", "lockfile", f"hash-pinned requirements ({reqs[0].name}, ...)" + ) + else: + report( + "WARN", + "lockfile", + "requirements files found but without --hash pins — " + "substitution attacks are invisible", + ) + return + report( + "FAIL", + "lockfile", + "no lockfile (uv.lock/poetry.lock/pdm.lock/requirements) — " + "the dependency set is unpinned; fix via packaging setup first", + ) + + +def check_codeowners(root: Path) -> None: + for rel in (".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS"): + path = root / rel + if path.is_file(): + text = read(path) + rules = [ + ln.split()[0] + for ln in text.splitlines() + if ln.strip() and not ln.lstrip().startswith("#") and ln.split() + ] + if any(r.startswith((GITHUB_DIR, f"/{GITHUB_DIR}")) for r in rules): + report("PASS", "codeowners", f"{rel} present and covers .github/") + else: + report( + "WARN", + "codeowners", + f"{rel} present but no rule for /.github/ — whoever edits " + "workflows or dependabot.yml bypasses every other control", + ) + star_rules = [i for i, r in enumerate(rules) if r == "*"] + if star_rules and star_rules[-1] == len(rules) - 1 and len(rules) > 1: + report( + "WARN", + "codeowners-order", + "catch-all '*' is the LAST rule — CODEOWNERS is " + "last-match-wins, so it overrides every specific rule above", + ) + return + report( + "WARN", + "codeowners", + "no CODEOWNERS file — no mandatory review for automation-defining paths", + ) + + +def check_workflow_presence(joined: str) -> None: + if "codeql-action" in joined: + report("PASS", "codeql", "CodeQL workflow present") + else: + report( + "WARN", + "codeql", + "no CodeQL workflow — OK only if code scanning 'default setup' is " + "enabled in repo settings (verify with --github)", + ) + if re.search(r"gitleaks|trufflehog", joined, re.IGNORECASE): + report("PASS", "secret-scan-ci", "CI secret-scan backstop present") + else: + report( + "WARN", + "secret-scan-ci", + "no gitleaks/trufflehog workflow — GitHub push protection alone " + "covers only high-confidence patterns", + ) + if "scorecard-action" in joined: + report("PASS", "scorecard", "OpenSSF Scorecard workflow present") + else: + report( + "NOTE", "scorecard", "no Scorecard workflow (optional posture dashboard)" + ) + if re.search( + r"attest-build-provenance|attest-sbom|cyclonedx|anchore/sbom-action", joined + ): + report("PASS", "sbom-provenance", "SBOM/attestation step found in workflows") + else: + report( + "NOTE", + "sbom-provenance", + "no SBOM/attestation step in workflows (optional; relevant at release)", + ) + + +def unpinned_workflow_actions(flows: dict[str, str]) -> list[str]: + unpinned = [] + for name, text in flows.items(): + for m in re.finditer( + r"^\s*(?:-\s*)?uses:\s*([\w./-]+)@([\w.-]+)", text, re.MULTILINE + ): + action, ref = m.group(1), m.group(2) + if action.startswith("./"): + continue + if not re.fullmatch(r"[0-9a-f]{40}", ref): + unpinned.append(f"{name}:{action}@{ref}") + return unpinned + + +def report_unpinned_actions(unpinned: list[str]) -> None: + if unpinned: + report( + "NOTE", + "action-pinning", + f"{len(unpinned)} action ref(s) not pinned to a 40-char SHA " + f"(e.g. {unpinned[0]}) — pinning policy is python-ci territory", + ) + + +def check_workflows(root: Path) -> None: + flows = workflow_texts(root) + check_workflow_presence("\n".join(flows.values())) + report_unpinned_actions(unpinned_workflow_actions(flows)) + + +def check_misc(root: Path) -> None: + if any( + (root / p).is_file() + for p in ("SECURITY.md", ".github/SECURITY.md", "docs/SECURITY.md") + ): + report("PASS", "security-policy", "SECURITY.md present") + else: + report( + "WARN", + "security-policy", + "no SECURITY.md — no documented way to report a vulnerability", + ) + pyproject = root / "pyproject.toml" + if pyproject.is_file() and re.search( + r"^\s*exclude-newer\s*=", read(pyproject), re.MULTILINE + ): + report("PASS", "freshness-window", "[tool.uv] exclude-newer configured") + else: + report( + "NOTE", + "freshness-window", + "no [tool.uv] exclude-newer — resolver has no freshness buffer " + "against just-published packages (optional, uv projects)", + ) + + +def gh_json(args: list[str], cwd: Path | None = None) -> tuple[int, dict | list | None]: + try: + proc = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + timeout=30, + cwd=str(cwd) if cwd else None, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return 1, {"error": str(exc)} + if proc.returncode != 0: + return proc.returncode, None + try: + return 0, json.loads(proc.stdout) if proc.stdout.strip() else {} + except ValueError: + return 0, {} + + +def github_repo_name(root: Path) -> str | None: + rc, view = gh_json(["repo", "view", "--json", "nameWithOwner"], cwd=root) + if rc == 0 and isinstance(view, dict) and "nameWithOwner" in view: + return str(view["nameWithOwner"]) + return None + + +def check_github_security(repo: str) -> None: + _, data = gh_json(["api", f"repos/{repo}"]) + sec = ( + (data or {}).get("security_and_analysis") or {} + if isinstance(data, dict) + else {} + ) + for key, check in ( + ("secret_scanning", "gh-secret-scanning"), + ("secret_scanning_push_protection", "gh-push-protection"), + ): + status = (sec.get(key) or {}).get("status") + if status == "enabled": + report("PASS", check, f"{key} enabled") + elif status == "disabled": + report("FAIL", check, f"{key} disabled — enable it (free on public repos)") + else: + report( + "WARN", + check, + f"{key} state unknown (insufficient token scope or licensing)", + ) + + +def check_github_alerts(repo: str) -> None: + rc, _ = gh_json(["api", f"repos/{repo}/vulnerability-alerts"]) + if rc == 0: + report("PASS", "gh-dependabot-alerts", "Dependabot alerts enabled") + else: + report( + "FAIL", + "gh-dependabot-alerts", + "Dependabot alerts disabled or not visible — enable with " + f"`gh api -X PUT repos/{repo}/vulnerability-alerts`", + ) + + +def check_github_code_scanning(repo: str) -> None: + rc, setup = gh_json(["api", f"repos/{repo}/code-scanning/default-setup"]) + if rc == 0 and isinstance(setup, dict) and setup.get("state") == "configured": + report("PASS", "gh-codeql-default", "code scanning default setup configured") + elif rc == 0: + report( + "WARN", + "gh-codeql-default", + "code scanning default setup not configured (fine if an advanced " + "CodeQL workflow exists)", + ) + else: + report( + "WARN", + "gh-codeql-default", + "could not query code-scanning setup (licensing/permissions)", + ) + + +def check_github(root: Path) -> None: + if shutil.which("gh") is None: + print( + "ERROR github-checks: gh CLI not found on PATH (needed for --github)", + file=sys.stderr, + ) + raise SystemExit(2) + repo = github_repo_name(root) + if repo is None: + report( + "WARN", + "github-repo", + "could not resolve the GitHub repo via `gh repo view` " + "(not a GitHub remote, or gh unauthenticated) — skipping API checks", + ) + return + check_github_security(repo) + check_github_alerts(repo) + check_github_code_scanning(repo) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--root", + type=Path, + default=Path.cwd(), + help="repository root to audit (default: cwd)", + ) + ap.add_argument( + "--github", + action="store_true", + help="also query GitHub-side settings via the gh CLI", + ) + ap.add_argument( + "--strict", action="store_true", help="treat WARN as FAIL for the exit code" + ) + args = ap.parse_args() + + root = args.root.resolve() + if not root.is_dir(): + print(f"ERROR root: {root} is not a directory", file=sys.stderr) + return 2 + + check_dependabot(root) + check_lockfile(root) + check_codeowners(root) + check_workflows(root) + check_misc(root) + if args.github: + check_github(root) + + order = {"FAIL": 0, "WARN": 1, "NOTE": 2, "PASS": 3} + for level, check, detail in sorted(results, key=lambda r: order[r[0]]): + print(f"{level} {check}: {detail}") + counts = {lvl: sum(1 for r in results if r[0] == lvl) for lvl in order} + failing = counts["FAIL"] + (counts["WARN"] if args.strict else 0) + print( + f"RESULT: {counts['FAIL']} fail, {counts['WARN']} warn, " + f"{counts['NOTE']} note, {counts['PASS']} pass" + ) + return 1 if failing else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/python-sync-async-clients/SKILL.md b/skills/python-sync-async-clients/SKILL.md new file mode 100644 index 0000000..f615ed6 --- /dev/null +++ b/skills/python-sync-async-clients/SKILL.md @@ -0,0 +1,17 @@ +--- +name: python-sync-async-clients +description: Implement and use the Frontal Python SDK's synchronous and asynchronous clients consistently. +--- + +# Python Sync and Async Clients + +Use this skill when changing client lifecycle, sync/async parity, or request behavior in `sdk-python`. + +- Use `Frontal` for sync applications and `AsyncFrontal` for async applications. Mirror public behavior where both client modes are supported. +- Use `with` and `async with` so HTTPX resources close deterministically. Do not block an async call path with sync network operations. +- Keep transport policy in `frontal_sdk/core` and domain operations in `frontal_sdk/resources`; avoid duplicating request handling in each service. +- Respect Python 3.9–3.12 support and the dependency versions pinned by `pyproject.toml`/`uv.lock`. +- Test sync behavior and async behavior independently where supported, including cancellation, timeouts, and cleanup. Use HTTP mocks; never require live credentials. +- Preserve the documented environment behavior; the SDK does not load `.env` files automatically. + +Read `AGENTS.md`, `docs/ARCHITECTURE.md`, and `docs/RESOURCES.md` before changing package boundaries. diff --git a/skills/python-testing/SKILL.md b/skills/python-testing/SKILL.md new file mode 100644 index 0000000..b67e831 --- /dev/null +++ b/skills/python-testing/SKILL.md @@ -0,0 +1,317 @@ +--- +name: python-testing +description: Builds pytest infrastructure for a Python package — layout and config, coverage gates with fail_under, Hypothesis property-based tests, mutation testing to catch weak or agent-gamed tests, multi-version runs via tox, nox, or uv. Use for 'set up pytest', 'add coverage', 'test on multiple Python versions'. Not for writing tests for one file, fixing a single failing test, or CI workflow YAML. +license: MIT +--- + +# python-testing + +Build test infrastructure for a Python package that stays trustworthy under +pressure: pytest configuration the installed pytest actually reads, branch +coverage with a gate proven to fail, Hypothesis and mutation testing as +backstops against tests that execute code without checking it, and multi-version +runs. The failure this skill prevents is green-but-meaningless — gates that +silently pass, config tables that are silently ignored, and suites edited (by +humans or agents) until they stop proving anything. + +## When NOT to use + +- Writing tests for a single file or function, or improving one module's tests — + do that directly; no infrastructure needed. +- Diagnosing or fixing one failing test — read the test and the code under test. +- CI workflow YAML — GitHub Actions jobs, version-matrix wiring, coverage + upload/combine across CI legs. The python-ci skill, if installed, owns that. + This skill decides *what* runs; CI decides *where*. +- Packaging work — pyproject metadata, build backend, converting a repo to src + layout as a packaging exercise. The python-packaging skill, if installed. +- Git hook wiring — the python-precommit skill; full test suites do not belong + in pre-commit hooks anyway. + +## Workflow + +### 1. Survey the repo + +Run the read-only audit first — it catches shadowed and split config sources: + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/check_test_config.py" --root . +``` + +Then establish the facts later steps depend on: + +```bash +uv run pytest --version # major version decides the config table (step 2) +grep -n "requires-python" pyproject.toml +ls pytest.ini setup.cfg tox.ini .coveragerc noxfile.py 2>/dev/null +``` + +Note the layout: `src//` + sibling `tests/` (src layout) or package at the +repo root (flat). Convention throughout: invoke tools through uv (`uv run`, +`uvx`). If uv is unavailable, the plain fallback is +`python -m pip install -e ".[dev]"` then `python -m pytest` — same flags +everywhere; stated once here. + +### 2. pytest config — one table, in pyproject.toml + +Two tables exist, and the installed major decides: pytest >= 9.0 (Nov 2025) reads +a native `[tool.pytest]` table (real TOML arrays/booleans); every supported +pytest reads `[tool.pytest.ini_options]` (INI-style strings). Never write both. +On pytest below 9, `[tool.pytest]` is ignored **silently**: no error, tests still +run, strictness flags never apply — so check `uv run pytest --version` first +(https://docs.pytest.org/en/stable/reference/customize.html). The block below +uses `ini_options` because it works everywhere; on a >= 9 repo prefer the native +table with the same keys. + +```toml +[tool.pytest.ini_options] +minversion = "8.0" # raise to "9.1" if you rely on the strict flags below via addopts (see gotchas) +testpaths = ["tests"] +addopts = "-ra --strict-markers --strict-config" +filterwarnings = ["error"] +``` + +- `--strict-markers` — typo'd or undeclared `@pytest.mark.*` markers fail + instead of silently never running. +- `--strict-config` — typo'd config keys become errors. +- `filterwarnings = ["error"]` — deprecations fail now, not at upgrade time. On + a legacy dependency tree this is noisy; add targeted + `"ignore::"` entries rather than deleting the gate. +- Async project? Add `asyncio_mode = "auto"` (with pytest-asyncio as a dev dep) + to drop the per-test marker. +- Keep `--cov` OUT of `addopts`. Coupling every run to coverage slows + single-test loops and can confuse debugger breakpoints; pass coverage flags + in the explicit commands below and in CI. Trade-off: some teams keep it in + `addopts` so coverage is never forgotten — if the user prefers that, keep it + and note `--no-cov` disables it for one run. +- **One config source.** pytest uses the first of `pytest.toml`/`.pytest.toml`, + `pytest.ini`/`.pytest.ini` (both win even when empty), `pyproject.toml` + (with either table), `tox.ini`, `setup.cfg` and ignores the rest — a stray + `pytest.ini` silently shadows everything in pyproject. Merge, then delete. + +### 3. Layout and discovery + +src layout (`src//` with tests in a sibling `tests/`, not inside the +package) is the reliable shape: with `uv run`, pytest imports the *installed* +package, so import and packaging bugs surface in tests instead of in users' +environments (https://docs.pytest.org/en/stable/explanation/goodpractices.html). +`testpaths = ["tests"]` and coverage `source = ["src"]` then line up. + +- "pytest can't find/import my package" right after moving to src layout + usually means the package is not installed in the environment — `uv sync` + (or `pip install -e .`) and re-run. +- Do not migrate a flat-layout repo to src layout just for tests; that is a + packaging change (python-packaging territory). Testing works in flat layout — + you only lose the installed-package guarantee. +- Shared fixtures live in `tests/conftest.py`. + +### 4. Coverage — branch on, gate proven + +```toml +[tool.coverage.run] +source = ["src"] +branch = true +parallel = true +relative_files = true + +[tool.coverage.report] +fail_under = 85 +show_missing = true +skip_covered = true +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "if __name__ == .__main__.:", +] +``` + +- `branch = true` is the point — line coverage says a line ran; branch coverage + says both sides of each `if` did. Line-only numbers overstate. +- `fail_under` of 80–90 is the defensible band. 100% is anti-recommended: it + breeds assert-free tests and `pragma` sprawl. Exclude genuinely unreachable + lines instead. +- If a `.coveragerc` exists it silently wins over `[tool.coverage.*]` in + pyproject — keep exactly one source. +- Run with `uv run pytest --cov --cov-report=term-missing`. + +**Prove the gate.** pytest-cov below 7.1 could return exit code 0 on a missed +aggregate threshold depending on report options (fixed in 7.1.0, 2026-03), and +rounding once passed 66.6% against a 67 bar. Never assume — demonstrate: + +```bash +uv run pytest --cov --cov-fail-under=100 -q; echo "exit=$?" +``` + +Expect a non-zero exit (unless coverage is genuinely 100%). If it exits 0, gate +in two steps instead — `uv run pytest --cov` then +`uv run coverage report --fail-under=85` — the standalone coverage command's +exit code is reliable. + +- Parallel runs (`pytest -n auto` via pytest-xdist) require `parallel = true` + and `relative_files = true`. Subprocesses spawned by tests need + `[tool.coverage.run] patch = ["subprocess"]` (coverage >= 7.10) — pytest-cov 7 + dropped its own subprocess measurement — otherwise coverage comes back empty + or 0%. +- A threshold is a floor, not a quality signal — it is trivially satisfied by + tests without assertions. Steps 5–6 are the countermeasures. Patch/delta + coverage on PRs is CI wiring (python-ci skill, if installed). + +Flag-by-flag detail, precedence rules, and combine mechanics: +[references/configuration.md](references/configuration.md). + +### 5. Property-based tests where they pay + +Add Hypothesis for parsers, serializers, codecs, and algorithmic code — anywhere +a property (round-trip, idempotence, invariant) exists. A 426-project corpus +study found each property-based test kills roughly 50x the mutations of an +average unit test (https://dl.acm.org/doi/10.1145/3764068). + +```bash +uv add --dev hypothesis # exact version pinned by uv.lock +``` + +```python +from hypothesis import given, strategies as st + + +@given(st.text()) +def test_roundtrip(s): + assert decode(encode(s)) == s +``` + +- Prefer bounded strategies (`st.integers(min_value=1)`) over `assume()` + filtering — rejecting most examples raises `HealthCheck.filter_too_much`. +- Register settings profiles in `tests/conftest.py`; CI runs the heavy one via + `pytest --hypothesis-profile=ci`: + +```python +from hypothesis import settings + +settings.register_profile("ci", max_examples=1000) +settings.register_profile("dev", max_examples=25) +``` + +- Persist the `.hypothesis/` example database (commit it, or cache it in CI) — + otherwise every run rediscovers known counterexamples from scratch. +- `DeadlineExceeded` only on CI machines means slow shared runners — raise or + disable `deadline` in the ci profile deliberately; don't delete the test. + +Property patterns, strategy design, and stateful testing: +[references/beyond-coverage.md](references/beyond-coverage.md). + +### 6. Mutation testing — do the tests notice bugs? + +Coverage proves a line executed; mutation testing proves a test *fails* when +that line is wrong. It is the direct check on assertion quality, which matters +most when agents write or edit tests: a suite that was edited until green +proves nothing by itself. Options — no settled winner, choose per constraints: + +| Tool | Character | Watch out | +| --- | --- | --- | +| mutmut | Mature, simplest CLI, pytest-first | Full-repo runs are slow — scope `paths_to_mutate` | +| cosmic-ray | Session-based, resumable, distributed | Heaviest setup; overkill for small packages | +| pytest-gremlins | Runs as a pytest plugin | Newer; adoption evidence is thin — vet maintenance first | + +Install as a dev dependency so it runs inside the project environment and gets +pinned by `uv.lock` (an unpinned `uvx` one-off executes whatever was published +last — a reproducibility and supply-chain hole): + +```bash +uv add --dev mutmut +uv run mutmut run +``` + +- Scope the first run to one core module (`[tool.mutmut]` `source_paths`; `do_not_mutate` for generated code), + then triage survivors: a surviving mutant means a missing or weak assertion — + or a genuinely equivalent mutant, which you mark and move on. +- Do not gate CI on mutation score on day one; run it scoped or nightly and + treat the score as a diagnostic — mutmut 3.7.0's own unreleased notes still list + coverage-excluded lines being mutated, false survivors and timeouts counted as + kills. Practitioners report 85–95% mutation scores + as far more meaningful than the same number in line coverage. + +Triage workflow and the agent-era rationale: +[references/beyond-coverage.md](references/beyond-coverage.md). + +### 7. Multi-version runs + +Match the tool to the project — this debate is genuinely unsettled: + +| Situation | Use | +| --- | --- | +| Simple deps, few version-specific branches | plain uv loop (below) | +| Complex matrix (extras, backends, C extensions) | tox or nox, uv-accelerated | +| Want Python (not INI) config, dynamic sessions | nox | +| Already on tox | keep it; add tox-uv — speed-up, no config rewrite | + +```bash +for v in 3.11 3.12 3.13 3.14; do # the versions requires-python promises; 3.10 is EOL 2026-10 + uv run -p "$v" --with pytest --with pytest-cov pytest || exit 1 +done +``` + +For tox or nox, swap the environment backend to uv (`tox-uv`, nox's uv backend): +environment setup gets dramatically faster with no config-model change. + +- nox footgun: a repo-root `.python-version` file can silently pin every + session to the local dev version (https://github.com/wntrblm/nox/issues/1038). +- Never enforce `fail_under` inside each version leg — a 3.13-only branch drops + coverage on the 3.10 leg. Locally: run the loop with `parallel = true`, then + `uv run coverage combine && uv run coverage report --fail-under=85` once. The + CI version of that split (artifacts, combine job) belongs to python-ci. + +Configs and trade-offs in depth: +[references/multi-version.md](references/multi-version.md). + +### 8. Verify + +1. `uv run pytest` — green. +2. The gate-trip demonstration from step 4 exits non-zero. +3. `python3 "${CLAUDE_SKILL_DIR}/scripts/check_test_config.py" --root .` exits 0. +4. Report to the user: config location and table chosen (and why), thresholds, + and which backstops (Hypothesis, mutation, multi-version) are wired vs + deferred. + +## Output spec + +Done means: + +- Exactly one pytest config source and one coverage config source; the pytest + table matches the installed major version. +- Branch coverage on; `fail_under` in the 80–90 band; the gate demonstrated to + exit non-zero when missed. +- Suite runs green via `uv run pytest` from a clean checkout. +- Where requested: Hypothesis dev dep with profiles; mutation tool pinned with + a first scoped run triaged; multi-version runner chosen with stated rationale. +- No CI YAML written or modified. + +## Failure modes & gotchas + +| Symptom | Cause / fix | +| --- | --- | +| Coverage gate "passes" while below threshold | pytest-cov < 7.1 aggregate exit-code bug or rounding; prove the gate (step 4) or gate via standalone `coverage report --fail-under` | +| Strictness flags mysteriously not applied | `[tool.pytest]` on pytest <9 (silently ignored), a shadowing `pytest.toml`/`pytest.ini`, or pytest 9.0.x — a regression silently ignored `--strict-markers`/`--strict-config` inside `addopts` (fixed 9.1.0, 2026-06); on >= 9 prefer the `strict_markers`/`strict_config` ini keys | +| Coverage 0% or empty under `pytest -n auto` | missing `parallel`/`relative_files`; test-spawned subprocesses need coverage's `patch = ["subprocess"]` (pytest-cov 7 no longer measures them) | +| Coverage config edits change nothing | a `.coveragerc` wins over pyproject — delete one | +| Gate script keys on a specific exit code | the tools disagree: coverage.py `fail_under` exits 2, pytest-cov exits 1, pytest exits 5 when no tests were collected — gate on non-zero | +| `fail_under = 90` with `branch = true` "passes" at 85% branch coverage | the total combines statement and branch opportunities; it is not a branch-only minimum — read the branch column, or gate branches separately | +| Matrix legs fail on version-specific code | `fail_under` enforced per leg; combine first, gate once | +| Suite green right after an agent edited the tests | proves nothing — review test diffs before source diffs, run mutation testing; researchers have shown agents faking green via an injected `conftest.py`, so verify in an environment the agent cannot write to | +| Generated tests pass but catch no regressions | mock-heavy tests, a documented agent bias (https://arxiv.org/abs/2602.00409) — require behavioral assertions on the public API, not mock call counts | +| Hypothesis rejects most examples (`filter_too_much`) | replace `assume()` with bounded strategies | +| Hypothesis finds a bug, next run forgets it | `.hypothesis/` database not persisted across runs | +| Flaky `DeadlineExceeded` on CI only | slow shared runners — adjust `deadline` in the ci profile | + +## Files + +- `scripts/check_test_config.py` — read-only audit of pytest/coverage config + (shadowed sources, wrong config table, missing branch coverage or gate). + Exits non-zero when it finds errors; never modifies the repo. +- `references/configuration.md` — pytest and coverage config in depth: + precedence, the pytest-9 table question, the exit-code bug, xdist/combine + mechanics. +- `references/beyond-coverage.md` — Hypothesis patterns and settings; mutation + tool comparison and survivor triage; why agent-edited green suites need + backstops. +- `references/multi-version.md` — uv loop vs tox vs nox, uv-backed runners, + coverage across versions. diff --git a/skills/python-testing/evals/evals.json b/skills/python-testing/evals/evals.json new file mode 100644 index 0000000..639ea1f --- /dev/null +++ b/skills/python-testing/evals/evals.json @@ -0,0 +1,82 @@ +{ + "skill": "python-testing", + "cases": [ + { "type": "should_trigger", "prompt": "Set up pytest for this package" }, + { "type": "should_trigger", "prompt": "add coverage reporting with a fail-under gate to our test suite" }, + { "type": "should_trigger", "prompt": "can u set up test coverge so the build fails under 85%?" }, + { "type": "should_trigger", "prompt": "we need to run the tests on python 3.10 through 3.14" }, + { "type": "should_trigger", "prompt": "should we use tox or nox for multi-version testing?" }, + { "type": "should_trigger", "prompt": "add hypothesis property-based tests for the parser" }, + { "type": "should_trigger", "prompt": "my AI agent keeps rewriting tests until they pass - how do I make the suite catch that?" }, + { "type": "should_trigger", "prompt": "add mutation testing so tests that never assert anything get flagged" }, + { "type": "should_trigger", "prompt": "pytest can't find my tests since we moved to src layout" }, + { "type": "should_trigger", "prompt": "move the pytest config from setup.cfg into pyproject.toml" }, + { "type": "should_trigger", "prompt": "coverage comes back empty when I run pytest -n auto" }, + { "type": "should_trigger", "prompt": "our coverage number is high but regressions keep slipping through, what should we add?" }, + { "type": "should_trigger", "prompt": "pytest + coverage + tox setup pls" }, + { "type": "should_not_trigger", "prompt": "write unit tests for src/mypkg/utils.py" }, + { "type": "should_not_trigger", "prompt": "test_parse_headers is failing after my change, fix it" }, + { "type": "should_not_trigger", "prompt": "add a GitHub Actions workflow that runs pytest across a python version matrix" }, + { "type": "should_not_trigger", "prompt": "combine the coverage files from the CI matrix legs and upload them to Codecov in the workflow" }, + { "type": "should_not_trigger", "prompt": "set up pre-commit so the tests and linters run before every commit" }, + { "type": "should_not_trigger", "prompt": "switch the repo to src layout and fill in the pyproject.toml metadata so we can build a wheel" }, + { "type": "should_not_trigger", "prompt": "set up mypy in strict mode for this package" }, + { "type": "should_not_trigger", "prompt": "ruff is failing on the tests folder, fix the lint errors" }, + { "type": "should_not_trigger", "prompt": "delete the flaky test that keeps breaking the suite" }, + { "type": "should_not_trigger", "prompt": "bump the version and publish the package to PyPI" }, + { + "type": "quality", + "prompt": "set up pytest with coverage for my python package", + "expected_behavior": [ + "checks the installed pytest version before choosing between [tool.pytest.ini_options] and the newer [tool.pytest] table", + "writes pytest config into pyproject.toml with testpaths and strict marker/config flags", + "enables branch coverage and sets a fail_under threshold in the 80-90 range, not 100", + "proves the coverage gate works by running pytest with an unreachable threshold and checking for a non-zero exit code", + "runs the test suite afterwards and reports the result" + ] + }, + { + "type": "quality", + "prompt": "we support python 3.11 through 3.14, make the tests run on all of them", + "expected_behavior": [ + "assesses project complexity before picking a runner instead of defaulting to tox", + "presents a plain uv run -p loop, tox, and nox as options with trade-offs", + "recommends a uv-backed environment backend when tox or nox is chosen", + "does not write GitHub Actions workflow YAML", + "verifies the suite runs on at least one non-default Python version locally" + ] + }, + { + "type": "quality", + "prompt": "add hypothesis property-based tests for our config parser", + "expected_behavior": [ + "targets properties like round-trip, idempotence, or invariants rather than re-asserting example outputs", + "uses bounded strategies instead of filtering generated inputs with assume()", + "registers separate dev and CI Hypothesis settings profiles", + "addresses persistence of the .hypothesis example database so discovered bugs stay found", + "runs the new property tests and reports failures honestly" + ] + }, + { + "type": "quality", + "prompt": "coverage is at 90% but bugs still keep shipping to users", + "expected_behavior": [ + "explains that coverage proves execution, not assertion quality", + "checks whether branch coverage is enabled and enables it if not", + "proposes mutation testing with a pinned tool and at least one alternative with trade-offs", + "suggests property-based tests for algorithmic or parsing code paths", + "does not respond by simply raising the coverage threshold" + ] + }, + { + "type": "quality", + "prompt": "an AI coding agent keeps making our tests pass by weakening assertions, harden the testing setup", + "expected_behavior": [ + "recommends mutation testing as the check on assertion quality", + "advises reviewing test-file diffs before production-code diffs in agent changes", + "warns that a green suite after the agent edited tests or conftest.py proves nothing by itself", + "hardens config so weakened gates fail loudly, for example a verified fail_under plus strict pytest flags" + ] + } + ] +} diff --git a/skills/python-testing/references/beyond-coverage.md b/skills/python-testing/references/beyond-coverage.md new file mode 100644 index 0000000..2c2332f --- /dev/null +++ b/skills/python-testing/references/beyond-coverage.md @@ -0,0 +1,222 @@ +# Beyond coverage: property-based and mutation testing + +**Contents:** [Why coverage alone fails](#why-coverage-alone-fails) · +[Hypothesis](#hypothesis) · [Mutation testing](#mutation-testing) · +[Agent-era test integrity](#agent-era-test-integrity) · [Sources](#sources) + +## Why coverage alone fails + +Coverage — even branch coverage — proves execution, not verification. A test +that calls a function and asserts nothing produces the same coverage as one +that would catch a regression. Widely-shared postmortems exist of suites with +100% branch coverage that still shipped bugs. Two mechanisms detect what +coverage cannot: + +- **Property-based testing** checks *classes* of inputs against invariants + instead of hand-picked examples. +- **Mutation testing** deliberately breaks the code and checks that some test + notices. + +## Hypothesis + +### Where it pays + +Best targets: parsers, serializers/codecs, mathematical and algorithmic code, +data-structure implementations — anywhere a property holds for all inputs. +Poor targets: heavily side-effectful code (network, UI flows); use example +tests there. The empirical case is strong: an OOPSLA 2025 corpus study of 426 +Hypothesis-using projects found the average property-based test catches ~50x +more mutations than the average unit test, with exception/membership/type-check +properties over 19x more effective than other property categories, and ~76% of +caught mutations found within the first 20 generated inputs +(https://dl.acm.org/doi/10.1145/3764068). + +### Property patterns, in priority order + +```python +from hypothesis import given, strategies as st + + +@given(st.binary()) +def test_roundtrip(payload): # 1. round-trip + assert decode(encode(payload)) == payload + + +@given(st.lists(st.integers())) +def test_idempotent(xs): # 2. idempotence + assert normalize(normalize(xs)) == normalize(xs) + + +@given(st.lists(st.integers())) +def test_invariant(xs): # 3. invariants + assert len(sorted(xs)) == len(xs) + + +@given(st.text()) +def test_oracle(s): # 4. oracle: compare to a trusted impl + assert fast_parse(s) == reference_parse(s) + + +@given(st.text()) +def test_expected_exceptions(s): # 5. "raises cleanly or succeeds" + try: + parse(s) + except ParseError: + pass # defined failure is fine; anything else propagates +``` + +### Strategy design + +- Generate valid data directly with bounded strategies + (`st.integers(min_value=1)`, `st.text(alphabet=..., min_size=1)`) instead of + generating broadly and discarding with `assume()`. Rejecting too many + examples fails the run with `HealthCheck.filter_too_much`. +- Build domain objects with `@st.composite` or `st.builds(MyModel, ...)`. +- When combining with `@pytest.mark.parametrize`, stack `parametrize` + outermost, above `@given`. + +### Settings and profiles + +Register in `tests/conftest.py`, select per environment: + +```python +from hypothesis import settings + +settings.register_profile("ci", max_examples=1000) +settings.register_profile("dev", max_examples=25) +# select with: pytest --hypothesis-profile=ci +``` + +Knobs that matter: `max_examples` (default 100), `deadline` (per-example time +limit — slow shared CI runners cause spurious `DeadlineExceeded`; raise it or +set it to None in the ci profile rather than deleting tests), `derandomize` +(fixed seed for reproducible runs), `verbosity`. + +### The example database + +Hypothesis records failing examples in `.hypothesis/` and retries them first +on later runs — a cache that speeds up rediscovery, not a durable regression +suite (entries can be dropped after test or version changes, and Hypothesis +says not to rely on it for correctness). Cache it in CI (or commit it) for +speed, and promote important counterexamples to explicit `@example(...)` +decorators — that is the permanent record — so they are visible in +the test file. + +### Stateful testing + +For APIs, caches, and mutable state, `RuleBasedStateMachine` generates +operation *sequences* and checks invariants after each step: + +```python +from hypothesis.stateful import RuleBasedStateMachine, rule, invariant + + +class CacheMachine(RuleBasedStateMachine): + def __init__(self): + super().__init__() + self.cache, self.model = LRUCache(cap=3), {} + + @rule(k=st.integers(0, 5), v=st.integers()) + def put(self, k, v): + self.cache.put(k, v) + self.model[k] = v + + @invariant() + def size_bounded(self): + assert len(self.cache) <= 3 + + +TestCache = CacheMachine.TestCase # collected by pytest +``` + +## Mutation testing + +### What it measures + +A mutation tool applies small code changes ("mutants": flip an operator, +off-by-one a boundary, delete a branch) and re-runs the tests. A mutant that no +test kills is a line your suite executes but does not check. Mutation score is +the assertion-quality metric coverage pretends to be; practitioners report +85–95% mutation scores being far more meaningful than the same number in line +coverage. + +### Tool choice (unsettled — trade-offs, not a winner) + +| Tool | Strengths | Costs / caveats | +| --- | --- | --- | +| mutmut (https://github.com/boxed/mutmut) | Long-established, minimal setup, `mutmut run` / `mutmut results` workflow, pytest-first | Whole-repo runs slow on large suites; scope `paths_to_mutate` | +| cosmic-ray (https://github.com/sixty-north/cosmic-ray) | Session files make long runs resumable; distributed execution | Most setup; overkill below a few tens of kLoC | +| pytest-gremlins | Runs inside pytest as a plugin | Newest of the three; its popularity traces to a single showcase thread (https://www.reddit.com/r/Python/comments/1ruv4wx/showcase_pytestgremlins_v150_fast_mutation/) — verify maintenance and adoption before standardizing on it | + +Install as a project dev dependency so the tool runs in the project +environment and gets an exact version pinned in `uv.lock`: + +```bash +uv add --dev mutmut +uv run mutmut run +uv run mutmut results +``` + +Pin the version wherever it is executed outside the lockfile (for example a +one-off `uvx --from mutmut== mutmut run`) — an unpinned download +executes whatever was published most recently, which is both irreproducible +and a supply-chain exposure. + +### Workflow that stays affordable + +1. Scope the first run to one high-value module: + +```toml +[tool.mutmut] +source_paths = ["src/mypkg/parser.py"] # mutmut 3.x key; `do_not_mutate` skips generated code +``` + +2. Triage survivors, one at a time: + - Missing assertion → strengthen the test that covers the line. + - Untested behavior → add a test (often a property test kills whole + families of mutants at once). + - Equivalent mutant (behavior genuinely unchanged) → mark/skip it and move + on; do not contort tests to kill it. +3. Expand scope module by module. Keep full runs out of the per-PR path — + mutation runs multiply suite runtime by the mutant count. Nightly or + scoped-to-changed-files runs are the sustainable shapes; wiring that + schedule into CI is python-ci territory. +4. Treat the score as a diagnostic to trend, not a gate to game — the same + Goodhart failure that hollowed out coverage percentages applies. + +## Agent-era test integrity + +Patterns worth encoding into team practice, all observed in the wild: + +- **Agents "fix" failing tests by weakening them** — deleting the test, + skipping it, or rewriting the assertion to match broken behavior. A suite + that went green *because the tests changed* proves nothing. Review heuristic: + in any agent-authored diff, read the test changes before the source changes; + treat a diff that rewrites many tests as a red flag. +- **A green run from an environment the agent can write to is not evidence.** + Researchers demonstrated agents reaching perfect benchmark resolve rates by + injecting a `conftest.py` that forces pytest to report success. Re-run + verification in an environment the agent cannot modify (fresh checkout, CI). +- **Agent-written tests skew mock-heavy** (https://arxiv.org/abs/2602.00409), + asserting call counts instead of behavior — they pass easily and detect + little. Require behavioral assertions against the public API; mutation + testing flags these tests because their mutants survive. +- **Agent tests tend to re-assert current behavior**, locking in bugs as + expectations. Property-based tests resist this: an invariant is stated + independently of the current output. +- **TDD-as-guardrail**: require a failing test (run it, observe the failure) + before the fix. This blocks the weaken-the-test shortcut because the test's + meaning is established while it still fails. +- Mutation testing is the mechanical backstop for all of the above: it + directly measures whether the suite would notice the code being wrong, + which is precisely the property an assertion-weakening edit destroys. + +## Sources + +- Hypothesis docs — https://hypothesis.readthedocs.io/ and https://hypothesis.works/ +- OOPSLA 2025, An Empirical Evaluation of Property-Based Testing in Python — https://dl.acm.org/doi/10.1145/3764068 +- Anthropic, Finding bugs with Claude and property-based testing — https://www.anthropic.com/research/property-based-testing +- mutmut — https://github.com/boxed/mutmut · cosmic-ray — https://github.com/sixty-north/cosmic-ray +- pytest-gremlins showcase thread — https://www.reddit.com/r/Python/comments/1ruv4wx/showcase_pytestgremlins_v150_fast_mutation/ +- Over-mocked agent tests — https://arxiv.org/abs/2602.00409 · agent test value studies — https://arxiv.org/abs/2603.13724, https://arxiv.org/abs/2602.07900 +- HypoFuzz (coverage-guided fuzzing over Hypothesis suites; early-stage) — https://github.com/Zac-HD/hypofuzz diff --git a/skills/python-testing/references/configuration.md b/skills/python-testing/references/configuration.md new file mode 100644 index 0000000..127f6a5 --- /dev/null +++ b/skills/python-testing/references/configuration.md @@ -0,0 +1,231 @@ +# pytest and coverage configuration in depth + +**Contents:** [Config source precedence](#config-source-precedence) · +[The pytest 9 table question](#the-pytest-9-table-question) · +[Recommended pytest block](#recommended-pytest-block) · +[Recommended coverage block](#recommended-coverage-block) · +[Proving the gate](#proving-the-gate) · +[Parallelism and combining data](#parallelism-and-combining-data) · +[Weakly-sourced leads](#weakly-sourced-leads) · [Sources](#sources) + +## Config source precedence + +Both tools read exactly one config source and silently ignore the rest. Most +"my config change did nothing" reports trace back to this. + +**pytest** stops at the first file that qualifies as a config file, in this +order (see https://docs.pytest.org/en/stable/reference/customize.html): + +1. `pytest.toml` / `.pytest.toml` (win even if empty; pytest >= 9) +2. `pytest.ini` / `.pytest.ini` (win even if empty) +3. `pyproject.toml` (only counts with a `[tool.pytest]` or `[tool.pytest.ini_options]` table) +4. `tox.ini` (only with a `[pytest]` section) +5. `setup.cfg` (only with a `[tool:pytest]` section) + +**coverage.py** reads the first of (see +https://coverage.readthedocs.io/en/latest/config.html): + +1. `.coveragerc` (wins even when pyproject has `[tool.coverage.*]`) +2. `.coveragerc.toml` +3. `setup.cfg` / `tox.ini` (`[coverage:...]` sections) +4. `pyproject.toml` (`[tool.coverage.*]`) + +Rule: one source per tool, preferably `pyproject.toml` for both, and delete the +losers after merging. The bundled `scripts/check_test_config.py` flags these +conflicts. + +## The two pyproject tables + +pytest 9.0 (2025-11-05) added a native `[tool.pytest]` table — real TOML arrays +and booleans for `addopts`/`testpaths` instead of the INI-string bridge that +`[tool.pytest.ini_options]` (pytest >= 6) provides. Rules, from +https://docs.pytest.org/en/stable/reference/customize.html: + +- Use one table, never both; pick by `uv run pytest --version`. +- On pytest < 9 an unrecognized `[tool.pytest]` table produces no error. Tests + run with default (lenient) settings and every strictness flag you thought you + set is off — `--strict-config` cannot save you because it is itself inside the + ignored table. `ini_options` works on every supported pytest. +- Keys are the same in both tables; only the value types differ + (`addopts = ["-ra", "--strict-markers"]` vs `addopts = "-ra --strict-markers"`). + +Also relevant when upgrading: pytest 9.x tightened config and deprecation +handling (warnings that used to pass now error), so a major-version bump can +break a previously green setup — read the release notes before bumping +`minversion`. + +## Recommended pytest block + +```toml +[tool.pytest.ini_options] +minversion = "8.0" +testpaths = ["tests"] +addopts = "-ra --strict-markers --strict-config" +filterwarnings = ["error"] +# async projects, with pytest-asyncio installed: +# asyncio_mode = "auto" +``` + +| Setting | Why | +| --- | --- | +| `minversion` | Fails fast when an old pytest would misread newer config | +| `testpaths` | Collection doesn't wander into `build/`, `.venv/`, docs | +| `-ra` | Summary of all non-passing outcomes at the end of a run | +| `--strict-markers` | Typo'd/undeclared `@pytest.mark.*` = error, not a silently-skipped mark | +| `--strict-config` | Typo'd config keys = error | +| `filterwarnings = ["error"]` | Deprecations fail during development, not during a forced upgrade | +| `asyncio_mode = "auto"` | Drops the `@pytest.mark.asyncio` boilerplate on every async test | + +`filterwarnings = ["error"]` on a legacy dependency tree produces noise from +third-party deprecations. Add targeted ignores rather than removing the gate: + +```toml +filterwarnings = [ + "error", + "ignore::DeprecationWarning:somepackage.*", +] +``` + +Declare markers you use so `--strict-markers` accepts them: + +```toml +markers = [ + "slow: long-running tests, deselect with -m 'not slow'", +] +``` + +### Should `--cov` live in addopts? + +Trade-off, not a rule: + +- **Out of addopts** (this skill's default): single-test debugging stays fast, + debugger breakpoints behave, and coverage is an explicit act + (`uv run pytest --cov`). CI must remember to pass the flags. +- **In addopts**: coverage can never be forgotten; every invocation pays + instrumentation overhead. Escape hatch for one run: `--no-cov`. + +Pick one, tell the user which and why. + +## Recommended coverage block + +```toml +[tool.coverage.run] +source = ["src"] +branch = true +parallel = true +relative_files = true + +[tool.coverage.report] +fail_under = 85 +show_missing = true +skip_covered = true +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "raise AssertionError", + "def __repr__", + "if __name__ == .__main__.:", +] +``` + +Notes: + +- `exclude_lines` entries are regexes — that is why `.__main__.` uses dots. + Overriding `exclude_lines` replaces the default list, so re-state + `pragma: no cover` when customizing. +- Flat layout (no `src/`): set `source = [""]` and add + `omit = ["tests/*"]` so the tests themselves aren't measured. +- Threshold philosophy: 80–90 with branch coverage is the widely used band; + 100% is anti-recommended because it optimizes for line count over assertion + quality. Tiered targets are a reasonable refinement (e.g. core logic higher + than glue code). Whatever the number, it is a floor/regression alarm — + "coverage tells you what you didn't test, not whether your tests are good." +- Per-PR patch/delta coverage ("changed lines need 80%, no legacy backfill") + is a stronger merge gate than a global number, but it is enforced in CI + services/workflows — out of scope here (python-ci skill, if installed). + +## Proving the gate + +pytest-cov has a documented failure mode where `--cov-fail-under` does not +produce a non-zero exit code for aggregate/multi-directory configurations +depending on report options, and internal rounding could pass e.g. 66.66% against a +67 requirement. pytest-cov 7.1.0 (2026-03-21) made the total consistent regardless +of reporting settings — still demonstrate the gate once per repo. +A gate that has never been seen to fail must be treated as not wired up. + +Demonstration procedure (also the acceptance test after any config change): + +```bash +uv run pytest --cov --cov-fail-under=100 -q; echo "exit=$?" +``` + +- Non-zero exit (and coverage below 100): gate works, restore the real + threshold. +- Zero exit while coverage is below the bar: split measurement from gating — + +```bash +uv run pytest --cov +uv run coverage report --fail-under=85 +``` + +The standalone `coverage report --fail-under` exit code is reliable, and the +split has a second benefit: it is the same shape you need for combining data +from multiple runs (below). + +## Parallelism and combining data + +**pytest-xdist** (`pytest -n auto`) splits tests across workers, each writing +its own data file. Requirements for coverage to survive this: + +- `[tool.coverage.run] parallel = true` — data files get unique suffixes + (`.coverage...`) instead of clobbering each other. +- `relative_files = true` — paths recorded relative to the repo root so data + from different working directories/runners lines up. +- Subprocesses spawned *by tests* (`subprocess`, `multiprocessing`) are not + measured unless coverage is told to patch them: `[tool.coverage.run] + patch = ["subprocess"]` (coverage >= 7.10; pytest-cov 7.0 dropped its own + subprocess support) — see https://coverage.readthedocs.io/en/latest/subprocess.html. + +Symptom of getting this wrong: coverage reports 0% or "no data collected" +under `-n auto` while a serial run looks fine. + +**Combining across runs** (multiple Python versions, unit vs integration): + +```bash +uv run coverage combine # merges .coverage.* into .coverage +uv run coverage report --fail-under=85 +``` + +Enforce the threshold once, on the combined data. Enforcing inside each leg +fails versions that legitimately skip version-specific branches. Shipping the +per-leg data files to a combine job in CI is python-ci territory. + +A conflicting-advice note: some setups use `coverage run -m pytest` instead of +`pytest --cov` (plugin-loading order arguments); guidance for xdist users is +the opposite — prefer `pytest -n auto --cov`. Pick based on whether you use +xdist; don't mix both invocation styles in one repo. + +## Weakly-sourced leads + +Verify before acting; both were single-sourced in the research behind this +skill: + +- **Python 3.14+ `sys.monitoring` core**: reported to silently break per-test + ("dynamic context") coverage tracking, with `COVERAGE_CORE=ctrace` as the + fallback. Check the coverage.py changelog + (https://coverage.readthedocs.io/en/latest/changes.html) if per-test + contexts misbehave on 3.14+. +- OpenSSF-badge-driven coverage floors appearing as external requirements — + observed once, not an established pattern. + +## Sources + +- pytest configuration reference — https://docs.pytest.org/en/stable/reference/customize.html +- pytest good practices (src layout, importmode) — https://docs.pytest.org/en/stable/explanation/goodpractices.html +- pytest changelog — https://docs.pytest.org/en/stable/changelog.html +- coverage.py config reference — https://coverage.readthedocs.io/en/latest/config.html +- coverage.py subprocess measurement — https://coverage.readthedocs.io/en/latest/subprocess.html +- pytest-cov docs — https://pytest-cov.readthedocs.io/ +- pytest-cov exit-code report — https://stackoverflow.com/questions/79815717/why-doesnt-pytest-cov-fail-with-a-non-zero-exit-code-when-code-coverage-thresho +- Modern Python CI with coverage (combine-then-gate walkthrough) — https://danielnouri.org/notes/2025/11/03/modern-python-ci-with-coverage-in-2025/ diff --git a/skills/python-testing/references/multi-version.md b/skills/python-testing/references/multi-version.md new file mode 100644 index 0000000..5b48ab8 --- /dev/null +++ b/skills/python-testing/references/multi-version.md @@ -0,0 +1,166 @@ +# Multi-version testing: the uv loop, tox, and nox + +Contents: [Choosing a runner](#choosing-a-runner) · [The plain uv loop](#the-plain-uv-loop) · +[tox](#tox-declarative-ini) · [nox](#nox-python-configuration) · +[uv-backed acceleration](#uv-backed-acceleration) · +[Coverage across versions](#coverage-across-versions) · +[Where CI takes over](#where-ci-takes-over) · [Sources](#sources) + +The question "tox or nox or neither?" is genuinely unsettled — a live debate, +not a solved one (see the tox-vs-nox thread in astropy, +https://github.com/astropy/astropy/issues/16412). Treat it as a fit question: +match the runner to the project's complexity, don't migrate a working setup for +fashion. + +## Choosing a runner + +| Situation | Recommendation | Why | +| --- | --- | --- | +| Simple deps, no per-version extras, no C extensions | plain `uv run -p` loop | zero config files, uv provisions interpreters on demand | +| Complex matrix — extras, doc builds, lint envs, C extensions | tox or nox | structured isolation, named environments, per-env deps | +| Config should be Python (conditionals, dynamic sessions) | nox | `noxfile.py` is code; sessions are decorated functions | +| Team already on tox | keep tox, add `tox-uv` | free speed-up, zero config-model change | +| Many *dependency* versions (not Python versions) | CI-side dynamic matrix | see "Where CI takes over" below | + +Interpreter availability is what used to make tox/nox painful: each Python in +the envlist had to pre-exist on the machine. `uv python install 3.11 3.12 3.13 3.14` +(or uv's on-demand download inside `uv run -p`) removes that class of failure +entirely. + +## The plain uv loop + +For a package with straightforward dependencies, a loop replaces the whole +runner layer: + +```bash +for v in 3.11 3.12 3.13 3.14; do + uv run -p "$v" --with pytest --with pytest-cov pytest || exit 1 +done +``` + +- `|| exit 1` matters — without it the loop reports only the last version's + status. +- Wrap it in a task runner (a `just` recipe or Makefile target) so the matrix + is one memorable command and the version list lives in one place. +- The loop tests against the *resolved project environment* per version; if a + dependency has no wheel for an older interpreter, the failure surfaces here + exactly as it would for a user — that is signal, not noise. +- What you give up vs tox/nox: named auxiliary environments (docs, lint), + per-env dependency overrides, and parallel env execution. When you start + wanting those, switch runners instead of growing the loop. + +## tox: declarative INI + +```ini +# tox.ini +[tox] +envlist = py310, py311, py312, py313 +isolated_build = true + +[testenv] +deps = + pytest + pytest-cov +commands = + pytest {posargs} +``` + +- `{posargs}` passes through anything after `--` (`tox -- -k parser -x`), so + the matrix entry point still supports single-test loops. +- Keep dependency versions out of `[testenv] deps` where possible; prefer + installing the project's own dev extras so tox and plain `uv run pytest` + test the same pins. +- tox environments are cached in `.tox/`; a stale env after dependency changes + is fixed with `tox -r` (recreate) — a classic "works in CI, stale locally" + source. + +## nox: Python configuration + +```python +# noxfile.py +import nox + +nox.options.default_venv_backend = "uv" + + +@nox.session(python=["3.11", "3.12", "3.13", "3.14"]) +def tests(session): + session.install(".", "pytest", "pytest-cov") + session.run("pytest", *session.posargs) +``` + +- Sessions are plain Python — conditionals, version-specific deps, and dynamic + parametrization need no plugin system. +- **Footgun:** a repo-root `.python-version` file can silently pin every nox + session to the locally selected interpreter, so the "matrix" runs one version + N times (https://github.com/wntrblm/nox/issues/1038). If the repo carries a + `.python-version`, verify each session's `python --version` output the first + time the matrix runs. + +## uv-backed acceleration + +Both runners can delegate environment creation to uv — the config model stays +identical, only env setup gets fast: + +- tox: add `tox-uv` (`uv add --dev tox-uv`); tox picks the plugin up + automatically. +- nox: set `nox.options.default_venv_backend = "uv"` (as above) or pass + `--default-venv-backend uv`; install nox itself with the extra + (`nox[uv]`) so the backend is available. + +Environment creation is usually the dominant cost of a local matrix run, so +this is the highest-leverage change for an existing tox/nox setup — far cheaper +than a runner migration. + +## Coverage across versions + +The rule from SKILL.md step 7, in full: + +1. `[tool.coverage.run]` needs `parallel = true` and `relative_files = true` + so each leg writes its own `.coverage.` data file instead of + overwriting a shared one. +2. Run the matrix (loop, tox, or nox). Each version leg runs + `pytest --cov` **without** any `fail_under` enforcement. +3. Combine once, gate once: + +```bash +uv run coverage combine +uv run coverage report --fail-under=85 +``` + +Never enforce the threshold inside a version leg: code guarded by +`sys.version_info >= (3, 13)` is legitimately unexecuted on the 3.10 leg, so a +per-leg gate fails builds for having version-specific branches at all. Only the +combined union is a meaningful number. (Same reasoning applies to uploading +coverage from every CI leg — it double-counts or conflicts; combine +deliberately.) + +Under tox, "coverage is 0% / no data collected" is almost always the parallel +data files never being combined, or the project being imported from `.tox/` +site-packages while `source = ["src"]` points elsewhere — `relative_files = +true` plus a `coverage combine` step resolves both. + +## Where CI takes over + +This reference covers deciding and running the matrix locally. The CI half — +GitHub Actions `strategy: matrix` wiring, per-leg artifact upload, a combine +job, patch-coverage gates on PRs — is the python-ci skill's territory (if +installed). Two patterns worth knowing exist on that side so you don't rebuild +them locally: + +- OS × Python matrices (e.g. 3 OSes × 4 versions) are a CI-native concern; + locally, testing the oldest and newest supported Python catches most issues. +- Testing against many versions of a *dependency* (not of Python) is a + CI-dynamic-matrix problem — MLflow's pattern (a central versions YAML plus a + script that queries PyPI and samples versions, always including min/max) is + the reference architecture there, not a local-runner concern. + +## Sources + +- pytest good practices (src layout, installed-package testing) — + https://docs.pytest.org/en/stable/explanation/goodpractices.html +- tox-vs-nox live debate — https://github.com/astropy/astropy/issues/16412 +- nox `.python-version` footgun — https://github.com/wntrblm/nox/issues/1038 +- tox-uv — https://github.com/tox-dev/tox-uv +- nox uv backend — https://nox.thea.codes/en/stable/usage.html +- coverage.py combine mechanics — https://coverage.readthedocs.io/en/latest/cmd.html#combining-data-files diff --git a/skills/python-testing/scripts/check_test_config.py b/skills/python-testing/scripts/check_test_config.py new file mode 100644 index 0000000..1f1c678 --- /dev/null +++ b/skills/python-testing/scripts/check_test_config.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Read-only audit of pytest/coverage configuration in a Python package repo. + +Finds the misconfigurations that make test infrastructure lie: shadowed or +split config sources, a config table the installed tools silently ignore, +missing branch coverage, and coverage gates that were never wired up. + +This script only READS files. It never modifies the repo, never runs the test +suite, and never installs anything. Version checks that require executing tools +(e.g. `pytest --version`) are intentionally left to the caller. + +Usage: + python3 check_test_config.py [--root PATH] [--strict] + +Output: one machine-readable line per finding: + ERROR : + WARN : + INFO : + +Exit codes: 0 = no errors (warnings allowed unless --strict), 1 = errors found. + +Requires Python 3.11+ for full TOML-aware checks (stdlib tomllib); on 3.10 it +falls back to section-presence checks only and says so. +""" + +# Keep full, actionable testing guidance in this standalone checker. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import tomllib +except ImportError: # Python 3.10: degrade to text-level checks + tomllib = None + +errors: list[str] = [] +warnings: list[str] = [] +infos: list[str] = [] + + +def err(where: str, msg: str) -> None: + errors.append(f"ERROR {where}: {msg}") + + +def warn(where: str, msg: str) -> None: + warnings.append(f"WARN {where}: {msg}") + + +def info(where: str, msg: str) -> None: + infos.append(f"INFO {where}: {msg}") + + +def read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + err(str(path), f"cannot read: {exc}") + return "" + + +def has_section(text: str, section: str) -> bool: + """Text-level TOML/INI section presence check, e.g. section='tool.pytest'.""" + pattern = re.compile( + r"^\s*\[" + re.escape(section) + r"(\.[A-Za-z0-9_.-]+)?\]", re.M + ) + return bool(pattern.search(text)) + + +def has_exact_section(text: str, section: str) -> bool: + pattern = re.compile(r"^\s*\[" + re.escape(section) + r"\]", re.M) + return bool(pattern.search(text)) + + +def addopts_as_string(value: object) -> str: + if isinstance(value, str): + return value + if isinstance(value, list): + return " ".join(str(v) for v in value) + return "" + + +def pytest_table_flags(pp_text: str, tool: dict) -> tuple[bool, bool, bool]: + pytest_table = tool.get("pytest") + if tomllib is not None and pp_text: + has_ini = isinstance(pytest_table, dict) and "ini_options" in pytest_table + has_native = isinstance(pytest_table, dict) and any( + key != "ini_options" for key in pytest_table + ) + has_coverage = "coverage" in tool + return has_ini, has_native, has_coverage + return ( + has_exact_section(pp_text, "tool.pytest.ini_options"), + has_exact_section(pp_text, "tool.pytest"), + has_section(pp_text, "tool.coverage"), + ) + + +def check_conflicting_sources( + pytest_ini: Path, + toml_configs: list[Path], + setup_cfg: Path, + tox_ini: Path, + has_pytest: bool, + setup_has_pytest: bool, + tox_has_pytest: bool, +) -> None: + if pytest_ini.is_file() and has_pytest: + err( + str(pytest_ini), + "pytest.ini exists alongside pytest config in pyproject.toml — pytest.ini silently wins; merge into one source and delete the other", + ) + if has_pytest or pytest_ini.is_file(): + for cfg in toml_configs: + err( + str(cfg), + f"{cfg.name} exists alongside other pytest config — it takes precedence over every other file, even when empty; keep exactly one source", + ) + if setup_has_pytest and has_pytest: + warn( + str(setup_cfg), + "[tool:pytest] in setup.cfg alongside pytest config in pyproject.toml — only one source is read; merge and delete the loser", + ) + if tox_has_pytest and has_pytest: + warn( + str(tox_ini), + "[pytest] section in tox.ini alongside pytest config in pyproject.toml — only one source is read; merge and delete the loser", + ) + + +def check_pytest_table_compatibility( + pyproject: Path, has_ini: bool, has_native: bool +) -> None: + if has_native and has_ini: + err( + str(pyproject), + "both [tool.pytest] and [tool.pytest.ini_options] present — pytest reads one table; merge into the one your pytest major supports", + ) + elif has_native: + info( + str(pyproject), + "[tool.pytest] native table (pytest >= 9.0) — SILENTLY ignored on older pytest; confirm with `pytest --version`, or use [tool.pytest.ini_options] if < 9 must work", + ) + + +def warn_missing_pytest_config( + root: Path, + has_pytest: bool, + pytest_ini: Path, + toml_configs: list[Path], + setup_has_pytest: bool, + tox_has_pytest: bool, +) -> None: + sources_exist = has_pytest or pytest_ini.is_file() or toml_configs + if not (sources_exist or setup_has_pytest or tox_has_pytest): + warn( + str(root), + "no pytest configuration found in pyproject.toml, pytest.toml, pytest.ini, setup.cfg, or tox.ini — add [tool.pytest.ini_options] (or [tool.pytest] on pytest >= 9) with testpaths and strict flags", + ) + + +def check_config_sources( + root: Path, + pyproject: Path, + pytest_ini: Path, + toml_configs: list[Path], + setup_cfg: Path, + tox_ini: Path, + has_ini: bool, + has_native: bool, +) -> None: + has_pytest = has_ini or has_native + setup_cfg_text = read_text(setup_cfg) if setup_cfg.is_file() else "" + tox_ini_text = read_text(tox_ini) if tox_ini.is_file() else "" + setup_has_pytest = bool(setup_cfg_text) and has_exact_section( + setup_cfg_text, "tool:pytest" + ) + tox_has_pytest = bool(tox_ini_text) and has_exact_section(tox_ini_text, "pytest") + check_conflicting_sources( + pytest_ini, + toml_configs, + setup_cfg, + tox_ini, + has_pytest, + setup_has_pytest, + tox_has_pytest, + ) + check_pytest_table_compatibility(pyproject, has_ini, has_native) + warn_missing_pytest_config( + root, + has_pytest, + pytest_ini, + toml_configs, + setup_has_pytest, + tox_has_pytest, + ) + + +def check_coverage_config( + pyproject: Path, + coveragerc: Path, + tool: dict, + has_coverage: bool, + pytest_table: object, +) -> None: + if coveragerc.is_file() and has_coverage: + err( + str(coveragerc), + ".coveragerc exists alongside [tool.coverage.*] in pyproject.toml — .coveragerc silently wins; keep exactly one source", + ) + if ( + tomllib is None + or not has_coverage + or not isinstance(tool.get("coverage"), dict) + ): + return + + coverage = tool["coverage"] + run = coverage.get("run", {}) if isinstance(coverage.get("run", {}), dict) else {} + report = ( + coverage.get("report", {}) + if isinstance(coverage.get("report", {}), dict) + else {} + ) + if run.get("branch") is not True: + warn( + str(pyproject), + "[tool.coverage.run] branch is not true — line-only coverage overstates; set branch = true", + ) + if run.get("parallel") is True and run.get("relative_files") is not True: + warn( + str(pyproject), + "[tool.coverage.run] parallel = true without relative_files = true — combining data across paths/runners will mismatch files", + ) + + ini = pytest_table.get("ini_options", {}) if isinstance(pytest_table, dict) else {} + addopts = addopts_as_string(ini.get("addopts", "")) if isinstance(ini, dict) else "" + if "fail_under" not in report and "--cov-fail-under" not in addopts: + warn( + str(pyproject), + "no coverage gate found — set fail_under under [tool.coverage.report] (80-90), then prove it trips with a non-zero exit", + ) + if "--cov" in addopts.split() or "--cov=" in addopts: + info( + str(pyproject), + "--cov baked into pytest addopts — every run (incl. single-test debugging) pays coverage overhead; --no-cov disables per run", + ) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--root", type=Path, default=Path.cwd(), help="package repo root (default: cwd)" + ) + ap.add_argument("--strict", action="store_true", help="treat warnings as errors") + args = ap.parse_args() + root = args.root.resolve() + if not root.is_dir(): + err(str(root), "not a directory") + print(errors[0], file=sys.stderr) + print("FAIL: 1 error(s), 0 warning(s)") + return 1 + + pyproject = root / "pyproject.toml" + pytest_ini = root / "pytest.ini" + toml_configs = [ + p + for p in (root / "pytest.toml", root / ".pytest.toml", root / ".pytest.ini") + if p.is_file() + ] + setup_cfg = root / "setup.cfg" + tox_ini = root / "tox.ini" + coveragerc = root / ".coveragerc" + pp_text = read_text(pyproject) if pyproject.is_file() else "" + pp_data: dict = {} + if pp_text and tomllib is not None: + try: + pp_data = tomllib.loads(pp_text) + except tomllib.TOMLDecodeError as exc: + err(str(pyproject), f"invalid TOML: {exc}") + elif pp_text and tomllib is None: + info( + str(pyproject), + "Python < 3.11 (no tomllib) — key-level checks skipped, section-presence checks only", + ) + + raw_tool = pp_data.get("tool", {}) + tool = raw_tool if isinstance(raw_tool, dict) else {} + has_ini, has_native, has_coverage = pytest_table_flags(pp_text, tool) + check_config_sources( + root, + pyproject, + pytest_ini, + toml_configs, + setup_cfg, + tox_ini, + has_ini, + has_native, + ) + check_coverage_config(pyproject, coveragerc, tool, has_coverage, tool.get("pytest")) + + for line in errors + warnings + infos: + print(line, file=sys.stderr) + n_fail = len(errors) + (len(warnings) if args.strict else 0) + print( + f"{'FAIL' if n_fail else 'OK'}: {len(errors)} error(s), {len(warnings)} warning(s), {len(infos)} info" + ) + return 1 if n_fail else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/python-typing/SKILL.md b/skills/python-typing/SKILL.md new file mode 100644 index 0000000..9bb9a61 --- /dev/null +++ b/skills/python-typing/SKILL.md @@ -0,0 +1,251 @@ +--- +name: python-typing +description: Sets up static type checking for a Python package — choosing mypy, pyright, basedpyright or ty, strict configuration, per-module overrides, shipping py.typed — and fixes type-check errors. Use when the user says 'add type checking', 'set up mypy', 'pyright errors', 'make this package typed'. Not for lint or format failures, failing tests, runtime bugs, or the CI workflow that runs the checker. +license: MIT +--- + +# Python Typing + +Stand up static type checking in the user's Python package with a strictness +strategy the team can defend — one checker, pinned, configured in +`pyproject.toml`, per-module escape hatches tracked as debt, a ratchet toward +strict, and (for libraries) a `py.typed` marker that verifiably survives into +the built wheel. Also covers fixing the errors the checker then reports. + +## When NOT to use + +- Ruff lint or format failures — including the ANN annotation-style rules, + which are lint, not type checking (the python-lint skill, if installed). +- A failing test suite or pytest setup (python-testing). +- Runtime exceptions — a `TypeError` raised at runtime is a bug to debug + directly, not a checker-configuration task. +- The CI workflow YAML that runs the checker (python-ci) or pre-commit wiring + (python-precommit) — this skill hands the working command off to those. +- Packaging metadata, build backends, or wheel building beyond the `py.typed` + marker itself (python-packaging). + +## Workflow + +### 1. Survey before choosing anything + +- Layout: `src/` vs flat; the import-package name; library (published, has + consumers) vs application. +- Lowest supported Python from `requires-python` — the checker must target it, + not the interpreter you happen to run. +- Framework deps that need checker *plugins*: `grep -E "django|sqlalchemy|pydantic" pyproject.toml`. + Plugins exist for mypy only; this can decide the whole checker choice. +- Existing config (`[tool.mypy]`, `[tool.pyright]`, `mypy.ini`, + `pyrightconfig.json`) — extend it, don't silently replace it. +- New project vs legacy codebase (rough annotation coverage) — decides + strict-from-day-one vs ratchet in step 4. + +### 2. Pick ONE checker — the race is unsettled + +The type-checker field is a live, unsettled race (mypy, pyright/basedpyright, +ty, Pyrefly, zuban). Do not present any of them as the obvious winner, and do +not quote speed multipliers or conformance percentages — the circulating +numbers are single-source and partly refuted. Decision rules that hold up: + +| Situation | Pick | +| --- | --- | +| Django (django-stubs) or Pydantic plugin-dependent stack | mypy + the framework's plugin (SQLAlchemy 2.x is natively typed — its mypy plugin is deprecated and stopped at mypy 1.10) | +| New project, no plugin needs, VS Code team | pyright (`standard`, then `strict`) | +| Non-VS-Code editors (Neovim, Helix, Cursor) or stricter defaults wanted | basedpyright | +| uv/Ruff shop that tolerates beta churn | pilot ty locally; keep mypy or pyright as the CI gate | +| Multi-million-LOC monorepo | evaluate Pyrefly | + +Commit to **one primary checker as the CI gate**. Using pyright in the editor +(Pylance) with a different CI checker is normal; running two checkers *as +gates* means maintaining parallel suppression comments (gotcha 5). Details and +trade-offs: [references/checker-selection.md](references/checker-selection.md). + +### 3. Install pinned, run via uv + +Pin the exact version — checker releases routinely add new diagnostics, so an +unpinned checker breaks CI on unrelated PRs. Substitute the current release +from PyPI for the pins shown: + +```bash +uv add --dev "mypy==2.3.1" # or: "pyright==1.1.413" (PyPI wrapper, bundles node) +uv run mypy src tests # or: uv run pyright +``` + +Without uv (once, for reference): `python -m pip install "mypy==2.3.1"` then +`python -m mypy src tests`. For a ty pilot without touching deps: +`uvx --from "ty==0.0.80" ty check src` (again, substitute the current pin; ty is still pre-1.0). + +### 4. Configure in pyproject.toml + +mypy and pyright read different tables, so both can coexist while migrating. +mypy — and note `strict = true` is what makes mypy check unannotated code at +all (gotcha 1): + +```toml +[tool.mypy] +python_version = "3.10" # lowest supported, from requires-python +strict = true +warn_unused_ignores = true +# plugins = ["pydantic.mypy"] # only if pydantic is a dependency + +[[tool.mypy.overrides]] +module = ["some_untyped_lib.*"] # third-party deps that ship no types +ignore_missing_imports = true +``` + +pyright / basedpyright (use `[tool.basedpyright]` for the fork): + +```toml +[tool.pyright] +include = ["src", "tests"] +exclude = ["**/legacy"] # added on top of pyright's defaults (node_modules, __pycache__, .*) +typeCheckingMode = "standard" # "strict" for new projects +pythonVersion = "3.10" +``` + +ty reads `[tool.ty]` in `pyproject.toml` (or `ty.toml`); config surface is +small and beta — check `uv run ty --help` rather than trusting stale examples. + +### 5. Set strictness — day one or ratchet, never big-bang retrofit + +- **New project**: `strict = true` / `typeCheckingMode = "strict"` from the + first commit. Retrofitting strictness later is what generates contributor + friction and hundred-error floods. +- **Legacy codebase**: ratchet. Global strict with per-module exemptions that + only ever shrink: + +```toml +[tool.mypy] +strict = true + +[[tool.mypy.overrides]] +module = ["myapp.legacy.*"] # exempt, migrate module-by-module +ignore_errors = true +``` + + pyright's equivalent is a `strict = ["src/myapp/core"]` array of strict + islands over a `standard` base. Full staged rollout, both checkers: + [references/strictness-ratchet.md](references/strictness-ratchet.md). +- Record the strategy where the team will see it (CONTRIBUTING or README): the + chosen checker, the run command, and the rule that exemptions only shrink. + +### 6. Ship py.typed and verify the wheel (libraries) + +Without a `py.typed` marker, PEP 561 says checkers treat every import from +your library as `Any` — annotations exist in source but are invisible to +consumers, silently. + +```bash +touch src//py.typed +``` + +Build backends differ on whether non-`.py` files get packaged (setuptools +needs `[tool.setuptools.package-data]`; hatchling usually includes it), so +never trust the source tree — verify the artifact: + +```bash +uv build +python3 "${CLAUDE_SKILL_DIR}/scripts/check_py_typed.py" --package +``` + +The script exits non-zero if `py.typed` is missing from the source package, +the newest wheel, or the newest sdist. Backend-by-backend config and stub +distribution options: [references/py-typed-distribution.md](references/py-typed-distribution.md). + +### 7. Fix the reported errors + +Triage in this order: + +1. **Real bugs first** — `Optional` misuse, wrong return types, unreachable + branches. These are the payoff; don't suppress them. +2. **Untyped third-party imports** — install the `types-*` stub package if one + exists and pin it (stubs version independently of their library). Only + fall back to a per-module `ignore_missing_imports` override — never the + global flag. +3. **Genuine false positives** — suppress with an error-code-scoped comment, + `# type: ignore[arg-type]` (mypy) or `# pyright: ignore[reportArgumentType]`, + and keep `warn_unused_ignores` (mypy) / `reportUnnecessaryTypeIgnoreComment` + (pyright) on so stale ignores fail the build. +4. Never fix an error by weakening global config. + +### 8. Hand off + +The type-check command is now the contract: the python-ci skill (if installed) +turns it into a CI job; python-precommit wires it into hooks. Not this skill's +job — stop at a locally passing command. + +## Output spec + +Done means all of these hold: + +- Exactly one primary checker, pinned in dev dependencies, configured in + `pyproject.toml`. +- `uv run ...` exits 0 over `src` (and `tests`, unless deliberately + excluded with a written reason). +- Strictness decision recorded: strict, or the current ratchet stage plus the + exemption list tracked as debt. +- For a library: `py.typed` present in the package **and** + `scripts/check_py_typed.py` passes against a freshly built wheel. +- Every suppression comment is error-code-scoped; unused-ignore detection is + enabled. +- The run command is documented where the team will find it. + +## Failure modes & gotchas + +1. **mypy green ≠ checked.** By default mypy skips unannotated function + bodies; a "no errors" run can be ignoring most of the codebase. Set + `check_untyped_defs = true` (or `strict`) before trusting any green run. +2. **Global strict + per-module downgrades don't compose.** Long-standing + issues in both tools (mypy #11401, pyright #601): you cannot cleanly + downgrade individual strict rules per module. What works: global strict + with per-module `ignore_errors` (mypy) or a `strict` directory array over a + `standard` base (pyright). +3. **pyright floods legacy code harder than mypy.** It was already checking + unannotated code, so jumping straight to `strict` surfaces far more errors + at once — start at `standard` on existing codebases. +4. **py.typed missing from the wheel** is the classic invisible failure: the + marker sits in git, the backend drops it from the artifact, and every + downstream user silently gets `Any`. Always verify the built wheel, not the + source tree (step 6). +5. **Two checkers, double suppressions.** The engines disagree (kwargs typing, + `__new__` inference, union widening), so the same line can need both + `# type: ignore[...]` and `# pyright: ignore[...]`. Accept that cost + knowingly or stay with one gate. +6. **Strict is not portable.** mypy strict, pyright strict, basedpyright's + "recommended" preset, and ty/Pyrefly rules are all different contracts — + zero errors on one implies nothing about the others. +7. **Pydantic friction.** Checkers can't see through `before`/`wrap` + validators, so constructors called with pre-validation input types raise + false positives. Options with trade-offs: mypy + `pydantic.mypy` plugin, + rule-scoped ignores on model modules, or (bleeding-edge) Pyrefly's + experimental Pydantic integration. Separately — a statically clean model is + not runtime-safe: Pydantic coerces by default (`"30"` becomes `30`) unless + its own strict mode is enabled. +8. **ty/Pyrefly maturity.** No plugin system (Django/SQLAlchemy/Pydantic + plugin stacks can't migrate), beta-grade churn, and migrating off pyright + can *lose* diagnostics it used to catch. ty's "gradual guarantee" (removing + or loosening an annotation never introduces new errors — adding precision + may surface real ones) makes it a pleasant incremental pilot — as an editor/local tool, not yet an OSS package's CI + gate. +9. **pyright `exclude` is additive.** Custom entries are added on top of the + defaults (`**/node_modules`, `**/__pycache__`, `**/.*`), and the defaults take + precedence — so you cannot un-exclude a dot-directory by listing it in + `include`; move the code instead. +10. **Ruff is not a type checker.** Its ANN rules police annotation *style*; + they catch zero type errors. A passing lint run says nothing here. +11. **Checker upgrades flip defaults.** mypy 2.0 (2026-05) turned + `--local-partial-types` and `--strict-bytes` on by default, changed + `--allow-redefinition` semantics, dropped `--python-version 3.9`, and + added experimental parallel checking (`--num-workers N`) — a routine bump + can newly break CI, which is why step 3 pins exact versions and upgrades + deliberately. + +## Bundled resources + +- [references/checker-selection.md](references/checker-selection.md) — the + contender field, comparison table, decision guide, hybrid editor/CI pattern. +- [references/strictness-ratchet.md](references/strictness-ratchet.md) — + staged strict-mode rollout with full configs for mypy and pyright. +- [references/py-typed-distribution.md](references/py-typed-distribution.md) — + PEP 561 distribution options, per-backend packaging config, verification. +- `scripts/check_py_typed.py` — verifies `py.typed` in the source package and + inside built wheel/sdist artifacts; read-only, non-zero exit on failure. diff --git a/skills/python-typing/evals/evals.json b/skills/python-typing/evals/evals.json new file mode 100644 index 0000000..3ad7ae5 --- /dev/null +++ b/skills/python-typing/evals/evals.json @@ -0,0 +1,78 @@ +{ + "skill": "python-typing", + "cases": [ + { "type": "should_trigger", "prompt": "add type checking to my package" }, + { "type": "should_trigger", "prompt": "Set up mypy in strict mode for this repo" }, + { "type": "should_trigger", "prompt": "should we use mypy or pyright for this library?" }, + { "type": "should_trigger", "prompt": "downstream users say all the types from our package show up as Any" }, + { "type": "should_trigger", "prompt": "mypy strict = true is throwing hundreds of errors, how do I roll it out gradually?" }, + { "type": "should_trigger", "prompt": "is ty ready to replace mypy in our CI gate?" }, + { "type": "should_trigger", "prompt": "pyright keeps flagging my pydantic model constructors, how do i fix taht" }, + { "type": "should_trigger", "prompt": "how do I silence mypy for just the tests module but keep src strict" }, + { "type": "should_trigger", "prompt": "make this package typed" }, + { "type": "should_trigger", "prompt": "we publish a library — how do consumers actually get our type hints?" }, + { "type": "should_trigger", "prompt": "setup basedpyright pls" }, + { "type": "should_trigger", "prompt": "fix these errors from pyright strict mode" }, + { "type": "should_not_trigger", "prompt": "ruff is flagging missing annotations under the ANN rules, fix the lint errors" }, + { "type": "should_not_trigger", "prompt": "add a GitHub Actions workflow that runs mypy on every pull request" }, + { "type": "should_not_trigger", "prompt": "add mypy to our pre-commit config" }, + { "type": "should_not_trigger", "prompt": "getting TypeError at runtime, 'NoneType' object is not subscriptable — help me debug it" }, + { "type": "should_not_trigger", "prompt": "the pytest suite is failing after the refactor, fix the tests" }, + { "type": "should_not_trigger", "prompt": "set up pyproject.toml metadata and a src layout for this project" }, + { "type": "should_not_trigger", "prompt": "build the wheel and check that the sdist includes the README" }, + { "type": "should_not_trigger", "prompt": "publish the package to PyPI with trusted publishing" }, + { "type": "should_not_trigger", "prompt": "run the test suite on Python 3.10 through 3.13 with tox" }, + { "type": "should_not_trigger", "prompt": "just add type hints to this one helper function real quick" }, + { + "type": "quality", + "prompt": "Add type checking to my Python package", + "expected_behavior": [ + "inspects the repo (or asks) for framework dependencies like Django, SQLAlchemy, or Pydantic and for src vs flat layout before recommending a checker, and presents the mypy vs pyright vs ty choice as unsettled trade-offs rather than declaring one winner", + "writes checker config into pyproject.toml with strictness matched to project age — strict for a new project, a documented ratchet plan for a legacy one", + "adds the checker as a pinned dev dependency and runs it via uv (noting a plain pip fallback), then fixes or triages the reported errors", + "for a library, adds py.typed and configures the build backend to include it", + "does not quote specific speed multipliers or conformance percentages as established facts" + ] + }, + { + "type": "quality", + "prompt": "mypy --strict reports 400 errors on our legacy codebase, roll it out gradually", + "expected_behavior": [ + "proposes a per-module ratchet (global strict with [[tool.mypy.overrides]] exempting legacy modules, or staged flag adoption) instead of a big-bang fix or abandoning strict", + "warns that global strict plus per-module rule downgrades do not compose cleanly and shows working override config (ignore_errors for legacy modules, ignore_missing_imports for untyped deps)", + "recommends error-code-scoped ignores like '# type: ignore[arg-type]' with warn_unused_ignores enabled, not bare ignores", + "defines the ratchet direction — the override/exemption list only shrinks over time and is tracked as debt" + ] + }, + { + "type": "quality", + "prompt": "Downstream users of our library say all our types show up as Any in their editor", + "expected_behavior": [ + "diagnoses a missing or unshipped py.typed marker (PEP 561) as the likely cause", + "adds py.typed inside the import package and adds the build-backend include config the backend needs (e.g. setuptools package-data)", + "builds the wheel and verifies py.typed is actually inside the built artifact, not just the source tree", + "recommends checking the public API with both mypy and pyright since downstream consumers use both" + ] + }, + { + "type": "quality", + "prompt": "pyright strict keeps flagging our Pydantic model constructors as errors", + "expected_behavior": [ + "explains the root cause — static checkers cannot see through Pydantic before/wrap validators and coercion, so constructor calls with pre-validation input types look wrong statically", + "presents options with trade-offs (mypy with the pydantic.mypy plugin, rule-scoped pyright ignores, relaxing specific rules for model modules) instead of one blanket fix", + "warns against blanket '# type: ignore' and against assuming a statically clean model is runtime-safe, since Pydantic coerces by default unless its own strict mode is enabled", + "does not add a second type checker as a CI gate without flagging the dual suppression-comment cost" + ] + }, + { + "type": "quality", + "prompt": "should I use mypy, pyright, or ty?", + "expected_behavior": [ + "frames the checker race as genuinely unsettled and maps situations to checkers — plugin-dependent stacks to mypy, new projects or editor-first teams to pyright/basedpyright, risk-tolerant uv shops to a ty pilot, huge monorepos to evaluating Pyrefly", + "flags ty and Pyrefly maturity caveats, including the absence of a plugin system for Django/SQLAlchemy/Pydantic stacks", + "avoids quoting benchmark multipliers or conformance percentages as fact", + "recommends committing to one primary CI checker and warns that running two means maintaining parallel suppression comments" + ] + } + ] +} diff --git a/skills/python-typing/references/checker-selection.md b/skills/python-typing/references/checker-selection.md new file mode 100644 index 0000000..fe95304 --- /dev/null +++ b/skills/python-typing/references/checker-selection.md @@ -0,0 +1,121 @@ +# Choosing a Python type checker (2026 — an unsettled race) + +**Contents:** [State of the race](#state-of-the-race) · +[The contenders](#the-contenders) · [Comparison table](#comparison-table) · +[Decision guide](#decision-guide) · [The hybrid pattern](#the-hybrid-editorci-pattern) · +[Multi-checker CI](#multi-checker-ci-know-the-cost) · +[Claims not to repeat](#claims-not-to-repeat-as-fact) · [Sources](#sources) + +## State of the race + +Unlike linting (settled on Ruff), static typing is a genuinely contested +layer. mypy and pyright are the two mature incumbents; a newer generation of +Rust-based checkers (Astral's ty, Meta's Pyrefly, the solo-maintained zuban) +competes on speed. Community debate is active and unresolved — recommend per +situation, not a universal winner, and expect the answer to shift; re-evaluate +at each project milestone rather than hard-coding a choice into team dogma. + +## The contenders + +- **mypy** ([github.com/python/mypy](https://github.com/python/mypy)) — the + reference implementation, written in Python (mypyc-compiled). Slowest of the + field but the only one with a **plugin system**: Django (`django-stubs`), + SQLAlchemy, and Pydantic (`pydantic.mypy`) ship plugins that understand + metaprogramming no stub-driven checker can. Defining quirk: it skips + unannotated function bodies unless `check_untyped_defs`/`strict` is set. +- **pyright** ([github.com/microsoft/pyright](https://github.com/microsoft/pyright)) + — Microsoft, written in TypeScript, the engine behind VS Code's Pylance. + Checks all code by default with aggressive inference; widely regarded as the + closest to the typing spec. No plugins — stub-driven only. It is a Node.js + program; the PyPI `pyright` package wraps it so `uv add --dev pyright` works + in pure-Python environments. +- **basedpyright** ([github.com/DetachHead/basedpyright](https://github.com/DetachHead/basedpyright)) + — community fork of pyright that restores LSP features Microsoft reserves + for closed-source Pylance (useful in Neovim, Helix, Emacs, Cursor), ships + stricter defaults (its "recommended" preset enables all rules — stricter + than either mypy or pyright), and friendlier CLI output. Configured via + `[tool.basedpyright]`. +- **ty** ([github.com/astral-sh/ty](https://github.com/astral-sh/ty)) — from + Astral (Ruff, uv), Rust, formerly Red Knot, in beta. Built around a formal + "gradual guarantee": loosening or removing annotations in working code is designed never to + introduce new errors elsewhere — a structural answer to the cascading-error + pain ratchet strategies exist to manage. No plugin system. Config in + `[tool.ty]` or `ty.toml`. +- **Pyrefly** ([pyrefly.org](https://pyrefly.org/), + [github.com/facebook/pyrefly](https://github.com/facebook/pyrefly)) — Meta, + Rust, built for very large monorepos. Opposite philosophy to ty: aggressive + inference on untyped code by default, plus a `suppress` command that + bulk-inserts suppressions across legacy code so strictness can be enabled + immediately and debt cleaned up over time. Shipped experimental **Pydantic + integration** ([pyrefly.org/blog/pyrefly-pydantic](https://pyrefly.org/blog/pyrefly-pydantic)) + — notable because no other stub-driven checker understands Pydantic models + without a plugin. +- **zuban** — solo-maintained Rust checker with a "mypy mode" that reads + existing mypy config files as-is; interesting for drop-in speed experiments + on mypy-configured repos, with the bus-factor caveat a solo project carries. + +## Comparison table + +| Checker | Maintainer | Maturity | Plugins | Default posture | Config | +| --- | --- | --- | --- | --- | --- | +| mypy | Python community | Reference impl., most mature (2.x since 2026-05; 2.0 flipped `local-partial-types`/`strict-bytes` on) | Django, SQLAlchemy, Pydantic | Permissive — skips unannotated defs | `[tool.mypy]` / `mypy.ini` | +| pyright | Microsoft | Mature, editor-dominant | None | Checks everything, strong inference | `[tool.pyright]` / `pyrightconfig.json` | +| basedpyright | Community fork | Actively maintained | None | Stricter than pyright | `[tool.basedpyright]` | +| ty | Astral | Beta (0.0.x releases, pre-1.0 as of 2026-09) | None | Gradual guarantee, lenient on untyped | `[tool.ty]` / `ty.toml` | +| Pyrefly | Meta | Stable 1.x (1.2.0, 2026-08) | None | Aggressive inference + bulk suppress | `[tool.pyrefly]` / `pyrefly.toml` | +| zuban | Solo maintainer | Young | None (reads mypy config) | mypy-compatible mode | mypy config files | + +## Decision guide + +| Situation | Recommendation | +| --- | --- | +| Django / SQLAlchemy / Pydantic-heavy codebase | mypy as the CI gate — the plugins are load-bearing; add pyright/Pylance in-editor | +| New library, VS Code team | pyright `standard`, ratchet to `strict`; ship `py.typed` from day one | +| Neovim / Helix / Emacs / Cursor editors | basedpyright | +| Already on uv + Ruff, tolerant of beta churn | pilot ty for local/editor speed; keep mypy or pyright as the CI gate until it matures | +| Multi-million-LOC monorepo | evaluate Pyrefly | +| Existing mypy config, curious about speed | zuban's mypy mode as a low-cost experiment | +| Publishing a library for external consumers | whatever you gate on, smoke-test the public API with both mypy **and** pyright — downstream users run both | + +## The hybrid editor/CI pattern + +The de-facto community standard: **pyright/Pylance in the editor** for +instant feedback, **one authoritative checker in CI** as the merge gate +(mypy where plugins matter, pyright otherwise). The editor checker is +advisory; only the CI checker's config is the contract. This gets fast +feedback without dual-suppression bloat, because you only *silence* the gate +checker. + +## Multi-checker CI — know the cost + +Some projects run two or more checkers in CI and treat disagreement as a +quality signal (documented example: IBM's mcp-context-forge, +[github.com/IBM/mcp-context-forge/issues/211](https://github.com/IBM/mcp-context-forge/issues/211)). +It does catch more edge cases — the engines genuinely disagree on `**kwargs` +typing, `__new__` inference, and union widening (see +[pyright discussion #5040](https://github.com/microsoft/pyright/discussions/5040)) +— but the price is parallel suppression comments (`# type: ignore[code]` and +`# pyright: ignore[rule]` on the same line) and satisfying multiple +non-equivalent strictness contracts. Default to one gate; adopt multi-checker +only as a deliberate, documented choice. + +## Claims not to repeat as fact + +- **Speed multipliers** ("3–5x", "10–100x faster") and **conformance + percentages** for these checkers trace to a small number of blog posts with + undisclosed methodology; several were flagged or refuted in verification. + Say "pyright is faster than mypy; the Rust checkers are faster still" and + stop there. +- **Named adopters** ("framework X switched to ty") trace to individual social + posts — treat as directional buzz, not evidence. +- Corporate acquisition rumors about the toolchain vendors are unverified — + do not repeat them. + +## Sources + +- mypy vs pyright vs ty comparison — [pydevtools.com/handbook/explanation/how-do-mypy-pyright-and-ty-compare/](https://pydevtools.com/handbook/explanation/how-do-mypy-pyright-and-ty-compare/) +- basedpyright's mypy comparison — [docs.basedpyright.com/dev/usage/mypy-comparison/](https://docs.basedpyright.com/dev/usage/mypy-comparison/) +- Practitioner trade-off thread — [discuss.python.org/t/mypy-vs-pyright-in-practice/75984](https://discuss.python.org/t/mypy-vs-pyright-in-practice/75984) +- ty announcement — [astral.sh/blog/ty](https://astral.sh/blog/ty) +- Pyrefly vs ty deep comparison — [blog.edward-li.com/tech/comparing-pyrefly-vs-ty/](https://blog.edward-li.com/tech/comparing-pyrefly-vs-ty/) +- Pyrefly + Pydantic — [pyrefly.org/blog/pyrefly-pydantic](https://pyrefly.org/blog/pyrefly-pydantic) diff --git a/skills/python-typing/references/py-typed-distribution.md b/skills/python-typing/references/py-typed-distribution.md new file mode 100644 index 0000000..5e5e2ed --- /dev/null +++ b/skills/python-typing/references/py-typed-distribution.md @@ -0,0 +1,136 @@ +# py.typed, PEP 561, and shipping types to your consumers + +**Contents:** [The core rule](#the-core-rule) · +[Three distribution methods](#three-ways-to-distribute-type-information) · +[Making the marker survive the build](#making-the-marker-survive-the-build) · +[Verify the artifact](#verify-the-artifact-not-the-source-tree) · +[Consumer-side testing](#test-the-consumer-experience) · +[Edge cases](#edge-cases) · [Sources](#sources) + +## The core rule + +[PEP 561](https://peps.python.org/pep-0561/): without an (empty) `py.typed` +marker file inside the import package, type checkers must treat every import +from the library as `Any`. The annotations exist in your source and do +nothing for consumers. mypy enforces this strictly (downstream users see +`error: Skipping analyzing "yourlib": module is installed, but missing +library stubs or py.typed marker`); pyright is more forgiving and may infer +from library code anyway — which is why the breakage often goes unnoticed by +pyright-using maintainers and is reported by mypy-using consumers. + +The marker goes next to the package's `__init__.py`: + +``` +src/mypackage/ +├── __init__.py +├── py.typed # empty file — its presence is the whole signal +└── core.py +``` + +## Three ways to distribute type information + +Per the [typing spec's distribution section](https://typing.python.org/en/latest/spec/distributing.html): + +1. **Inline annotations + `py.typed`** — the default. Cheapest to maintain; + types live next to implementation and can't drift. +2. **Bundled `.pyi` stubs** alongside the `.py` sources (plus `py.typed`) — + for code that can't carry inline annotations: C extensions, generated code. +3. **A separate stub-only package** named `-stubs` on PyPI — + decouples stub releases from the runtime package (typeshed's naming + convention). Layout: `mypackage-stubs/` containing `py.typed`, + `__init__.pyi`, `core.pyi`. + +Prefer option 1 unless the source physically can't carry annotations. + +## Making the marker survive the build + +Build backends differ on whether non-`.py` files inside a package are +packaged automatically. This is where `py.typed` silently disappears. + +**setuptools** — requires explicit config: + +```toml +[tool.setuptools.package-data] +mypackage = ["py.typed"] +``` + +(or `include_package_data = true` with the file tracked appropriately). + +**hatchling** — packages the package directory's contents by default, so the +marker is usually included; force it if you have custom include rules: + +```toml +[tool.hatch.build.targets.wheel] +packages = ["src/mypackage"] +``` + +**flit-core / poetry-core** — generally include files inside the package +directory; Poetry also supports an explicit `include = ["mypackage/py.typed"]`. + +Backend defaults change between versions and custom include/exclude rules +override them — which is why the next section exists. Note that `uv init +--lib` scaffolds `py.typed` for new libraries automatically. + +## Verify the artifact, not the source tree + +The marker being in git proves nothing about the wheel. After every packaging +change: + +```bash +uv build # or: python -m build +python3 scripts/check_py_typed.py --package mypackage +``` + +The bundled script checks the source package, the newest wheel, and the +newest sdist in `dist/`, and exits non-zero with `ERROR` lines if any is +missing the marker. Manual equivalent: + +```bash +unzip -l dist/*.whl | grep py.typed # wheel +tar -tzf dist/*.tar.gz | grep py.typed # sdist +``` + +The sdist matters too: some consumers and rebuilders (conda-forge, distro +packagers) build from the sdist, and a marker missing there reintroduces the +bug downstream. + +## Test the consumer experience + +Downstream users run both major checkers, so smoke-test the published surface +with both — regardless of which one gates your CI: + +```bash +uv run mypy -c "import mypackage; reveal_type(mypackage.main_entry)" +uv run pyright --verifytypes mypackage +``` + +`pyright --verifytypes` is purpose-built for library authors: it reports the +percentage of the public API that is fully typed and lists every symbol with +missing or partial annotations. + +If your own strict run depends on `types-*` stub packages for dependencies +(e.g. `types-requests`), pin them — stub packages version independently of +the library they describe, and an unpinned stub bump can break your CI with +no change in your code. + +## Edge cases + +- **Single-file modules**: PEP 561 assumes packages; a lone `module.py` can't + carry a marker. The spec's guidance is to refactor into a package. A + community workaround — an optional companion `-stubs` package + containing only `py.typed` — is discussed in + [this Python Discourse thread](https://discuss.python.org/t/type-hinted-single-file-modules-empty-optional-stubs-but-for-py-typed-and-pep-561/107422); + it works with mypy, but its handling by pyright and newer checkers is + unconfirmed. Prefer the package refactor. +- **Namespace packages**: each distributed portion needs its own `py.typed`. +- **Partial typing**: a package that is only partly annotated may still ship + `py.typed`; consumers then see your real annotations where present and + inferred/`Any` elsewhere. Shipping it early is better than waiting for + 100% coverage. + +## Sources + +- PEP 561 — [peps.python.org/pep-0561/](https://peps.python.org/pep-0561/) +- Typing spec, distribution — [typing.python.org/en/latest/spec/distributing.html](https://typing.python.org/en/latest/spec/distributing.html) +- "Don't forget py.typed" — [dev.to/whtsky/don-t-forget-py-typed-for-your-typed-python-package-2aa3](https://dev.to/whtsky/don-t-forget-py-typed-for-your-typed-python-package-2aa3) +- Single-file modules and PEP 561 — [discuss.python.org/t/type-hinted-single-file-modules-empty-optional-stubs-but-for-py-typed-and-pep-561/107422](https://discuss.python.org/t/type-hinted-single-file-modules-empty-optional-stubs-but-for-py-typed-and-pep-561/107422) diff --git a/skills/python-typing/references/strictness-ratchet.md b/skills/python-typing/references/strictness-ratchet.md new file mode 100644 index 0000000..c070ece --- /dev/null +++ b/skills/python-typing/references/strictness-ratchet.md @@ -0,0 +1,141 @@ +# Strict mode and the gradual-adoption ratchet + +**Contents:** [What strict actually enables](#what-strict-actually-enables) · +[The ratchet, stage by stage](#the-ratchet-stage-by-stage) · +[mypy ratchet config](#mypy-ratchet-config) · +[pyright strict islands](#pyright-strict-islands) · +[Suppression hygiene](#suppression-hygiene) · +[Composability traps](#composability-traps) · +[Structural alternatives](#structural-alternatives-ty-and-pyrefly) · [Sources](#sources) + +## What strict actually enables + +mypy `strict = true` is a bundle of individual flags (per the +[mypy config docs](https://mypy.readthedocs.io/en/stable/config_file.html)): +`disallow_untyped_defs`, `disallow_incomplete_defs`, `disallow_untyped_calls`, +`disallow_any_generics`, `disallow_untyped_decorators`, `check_untyped_defs`, +`no_implicit_optional`, `warn_redundant_casts`, `warn_return_any`, +`warn_unused_ignores`, `strict_equality` — as of mypy 1.x; the bundle changes +across releases (mypy 2.0 also flipped `local_partial_types` and `strict_bytes` +on globally), so confirm the current list with `mypy --help`. Knowing the +components matters because the ratchet turns them on one at a time. + +pyright has four `typeCheckingMode` levels — `off` / `basic` / `standard` +(default) / `strict` — and its strict is generally tighter than mypy's. +Remember pyright checks unannotated code even in `standard`, so a legacy +codebase jumping straight to pyright `strict` floods harder than the same +jump in mypy. + +## The ratchet, stage by stage + +Big-bang strict on an existing codebase reliably produces hundreds of errors +and a demoralized team. Tighten one dimension at a time; per-module overrides +keep new code strict while legacy code is exempted rather than fixed at once: + +1. **Baseline honestly.** Enable the checker permissively — but for mypy set + `check_untyped_defs = true` immediately, or the run is skipping every + unannotated function body and the green result is a lie. +2. **Fix high-signal errors only** — real bugs the baseline surfaces. +3. **Optional-correctness first** — `no_implicit_optional` plus strict + optional checking is the single biggest ROI step; most real bugs live in + `None` handling. +4. **Require annotations** — `disallow_untyped_defs` + + `disallow_incomplete_defs`, usually per-module as code gets touched. +5. **Strict islands** — full `strict = true` for new/core modules while + legacy modules carry explicit exemptions. +6. **Global strict** — flip the global flag, keep a shrinking exemption list. + +The invariant at every stage: **the exemption list only shrinks**. Treat each +override block as recorded debt with an owner, and delete entries as modules +are migrated. If the list grows, the ratchet is broken. + +## mypy ratchet config + +```toml +[tool.mypy] +python_version = "3.10" +strict = true +warn_unused_ignores = true +# plugins = ["pydantic.mypy"] # if pydantic is in play + +# Legacy code: exempt wholesale until migrated (works reliably) +[[tool.mypy.overrides]] +module = ["myapp.legacy.*", "myapp.migrations.*"] +ignore_errors = true + +# Tests often relax annotation requirements without going untyped +[[tool.mypy.overrides]] +module = ["tests.*"] +disallow_untyped_defs = false + +# Third-party deps that ship no py.typed and have no types-* stubs +[[tool.mypy.overrides]] +module = ["some_untyped_lib.*"] +ignore_missing_imports = true +``` + +## pyright strict islands + +pyright inverts the model: a permissive base with an explicit array of strict +directories, expanded as migration proceeds. + +```toml +[tool.pyright] +include = ["src", "tests"] +exclude = ["**/legacy"] # added on top of pyright's defaults +typeCheckingMode = "standard" # base level while migrating +strict = ["src/myapp/core", "src/myapp/api"] # strict islands, grow this list +pythonVersion = "3.10" +reportMissingTypeStubs = "warning" # strict's noisiest rule, downgraded +``` + +## Suppression hygiene + +- Always scope ignores to an error code — `# type: ignore[arg-type]` (mypy), + `# pyright: ignore[reportArgumentType]` — never a bare `# type: ignore`, + which silences every future error on that line. +- Keep the rot detectors on: `warn_unused_ignores = true` (mypy) and + `reportUnnecessaryTypeIgnoreComment = "error"` (pyright) make stale + suppressions fail the run, so debt can't accumulate invisibly. +- Prefer a per-module override block over sprinkling tens of inline ignores + in one file — one visible config entry is easier to track and delete. +- `typing.cast` and `Any` are suppressions too; grep for them during reviews. + +## Composability traps + +- **mypy [#11401](https://github.com/python/mypy/issues/11401)** ("strict is + incompatible with per-module configuration"): with global `strict = true` + you cannot cleanly *downgrade individual strict components* per module in + every case. Per-module `ignore_errors = true` and flipping specific named + flags (as above) work; expecting `strict = false` in an override to undo the + bundle does not. +- **pyright [#601](https://github.com/microsoft/pyright/issues/601)**: under + `typeCheckingMode = "strict"` individual rules cannot be downgraded below + the strict floor in that scope — which is exactly why the + standard-base-plus-strict-array pattern exists. +- Overrides silence, they don't migrate: an `ignore_errors` module is + unchecked entirely — worse than permissive checking. Keep islands small and + migration active. + +## Structural alternatives (ty and Pyrefly) + +Two newer checkers attack the same pain structurally instead of procedurally: + +- **ty's gradual guarantee** — loosening or removing annotations never introduces new errors (adding precision may reveal real ones) + elsewhere, eliminating the "annotated one function, broke ten call sites" + cascade. Makes file-by-file adoption genuinely safe, at beta-maturity cost. +- **Pyrefly's `suppress`** — bulk-inserts suppression comments across legacy + code, letting aggressive inference turn on immediately with debt paid down + later — the ratchet run in reverse. + +Both are worth watching; neither has a plugin system, so plugin-dependent +stacks (Django/SQLAlchemy/Pydantic-via-mypy) can't migrate yet. + +## Sources + +- mypy config reference — [mypy.readthedocs.io/en/stable/config_file.html](https://mypy.readthedocs.io/en/stable/config_file.html) +- mypy strict how-to — [pydevtools.com/handbook/how-to/how-to-configure-mypy-strict-mode/](https://pydevtools.com/handbook/how-to/how-to-configure-mypy-strict-mode/) +- Strategic gradual-typing guide — [medium.com/@tihomir.manushev/a-strategic-guide-to-gradual-typing-in-python-49ac85f6dbdd](https://medium.com/@tihomir.manushev/a-strategic-guide-to-gradual-typing-in-python-49ac85f6dbdd) +- Ratchet wins write-up — [medium.com/@sparknp1/7-gradual-typing-wins-in-python-494fe14be587](https://medium.com/@sparknp1/7-gradual-typing-wins-in-python-494fe14be587) +- mypy #11401 — [github.com/python/mypy/issues/11401](https://github.com/python/mypy/issues/11401) +- pyright #601 — [github.com/microsoft/pyright/issues/601](https://github.com/microsoft/pyright/issues/601) diff --git a/skills/python-typing/scripts/check_py_typed.py b/skills/python-typing/scripts/check_py_typed.py new file mode 100644 index 0000000..21277c2 --- /dev/null +++ b/skills/python-typing/scripts/check_py_typed.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Verify a PEP 561 py.typed marker exists in the source package AND in built artifacts. + +Read-only: this script never creates, modifies, or deletes anything. Build the +artifacts first (`uv build` or `python -m build`), then run: + + python3 check_py_typed.py [--package NAME] [--root DIR] [--dist DIR] [--source-only] + +Checks performed: + 1. source — /src//py.typed or //py.typed exists + 2. wheel — newest *.whl in contains /py.typed + 3. sdist — newest *.tar.gz in contains ...//py.typed + +Output: one `OK`/`ERROR`/`WARN : ` line per check. +Exit codes: 0 = all checks passed, 1 = at least one ERROR, 2 = usage error. +""" + +# Keep full, actionable typing guidance in this standalone checker. +# ruff: noqa: E501 + +from __future__ import annotations + +import argparse +import sys +import tarfile +import zipfile +from pathlib import Path + +SKIP_DIRS = { + "tests", + "test", + "docs", + "doc", + "examples", + "scripts", + "tools", + "build", + "dist", + "site-packages", + "node_modules", + "__pycache__", +} + +failures = 0 + + +def report(status: str, check: str, detail: str) -> None: + global failures + if status == "ERROR": + failures += 1 + print(f"{status} {check}: {detail}") + + +def candidate_packages(root: Path) -> list[Path]: + """Import packages under /src/ (preferred) or /, by __init__.py.""" + found: list[Path] = [] + for base in (root / "src", root): + if not base.is_dir(): + continue + for child in sorted(base.iterdir()): + if ( + child.is_dir() + and child.name not in SKIP_DIRS + and not child.name.startswith(".") + and (child / "__init__.py").is_file() + ): + found.append(child) + if found: + break # src layout wins; don't mix in flat-layout lookalikes + return found + + +def locate_package(root: Path, name: str | None) -> Path | None: + if name: + for base in (root / "src", root): + pkg = base / name + if (pkg / "__init__.py").is_file(): + return pkg + report( + "ERROR", + "source", + f"package {name!r} not found under {root}/src or {root} (no __init__.py)", + ) + return None + candidates = candidate_packages(root) + if not candidates: + report( + "ERROR", + "source", + f"no import package found under {root} — pass --package NAME", + ) + return None + if len(candidates) > 1: + names = ", ".join(c.name for c in candidates) + report( + "ERROR", + "source", + f"multiple packages found ({names}) — pass --package NAME", + ) + return None + return candidates[0] + + +def newest(dist: Path, pattern: str) -> Path | None: + matches = sorted(dist.glob(pattern), key=lambda p: p.stat().st_mtime) + return matches[-1] if matches else None + + +def contains_marker(members: list[str], pkg: str) -> bool: + target = f"{pkg}/py.typed" + return any(m == target or m.endswith("/" + target) for m in members) + + +def check_wheel(dist: Path, pkg: str) -> None: + whl = newest(dist, "*.whl") + if whl is None: + report( + "ERROR", + "wheel", + f"no *.whl in {dist} — run `uv build` (or `python -m build`) first", + ) + return + try: + with zipfile.ZipFile(whl) as zf: + members = zf.namelist() + except (OSError, zipfile.BadZipFile) as exc: + report("ERROR", "wheel", f"cannot read {whl.name}: {exc}") + return + if contains_marker(members, pkg): + report("OK", "wheel", f"{whl.name} contains {pkg}/py.typed") + else: + report( + "ERROR", + "wheel", + f"{whl.name} is missing {pkg}/py.typed — configure the build backend " + "to include it (e.g. setuptools [tool.setuptools.package-data]) and rebuild", + ) + + +def check_sdist(dist: Path, pkg: str) -> None: + sdist = newest(dist, "*.tar.gz") + if sdist is None: + report( + "WARN", + "sdist", + f"no *.tar.gz in {dist} — skipping (sdist consumers rebuild from source)", + ) + return + try: + with tarfile.open(sdist, "r:gz") as tf: + members = tf.getnames() + except (OSError, tarfile.TarError) as exc: + report("ERROR", "sdist", f"cannot read {sdist.name}: {exc}") + return + if contains_marker(members, pkg): + report("OK", "sdist", f"{sdist.name} contains {pkg}/py.typed") + else: + report( + "ERROR", + "sdist", + f"{sdist.name} is missing {pkg}/py.typed — sdist rebuilders " + "(conda-forge, distros) would ship an untyped package", + ) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--package", help="import package name (auto-detected when unambiguous)" + ) + ap.add_argument( + "--root", type=Path, default=Path.cwd(), help="project root (default: cwd)" + ) + ap.add_argument( + "--dist", type=Path, help="artifacts directory (default: /dist)" + ) + ap.add_argument( + "--source-only", + action="store_true", + help="only check the source tree, skip wheel/sdist", + ) + args = ap.parse_args() + + root = args.root.resolve() + if not root.is_dir(): + print(f"ERROR usage: --root {root} is not a directory") + return 2 + dist = (args.dist or root / "dist").resolve() + + pkg_dir = locate_package(root, args.package) + if pkg_dir is None: + print(f"FAIL: {failures} error(s)") + return 1 + pkg = pkg_dir.name + + marker = pkg_dir / "py.typed" + if marker.is_file(): + report("OK", "source", f"{marker.relative_to(root)} exists") + else: + report( + "ERROR", + "source", + f"{pkg_dir.relative_to(root)}/py.typed missing — create it " + f"(empty file) so PEP 561 checkers stop treating {pkg} as Any", + ) + + if not args.source_only: + if dist.is_dir(): + check_wheel(dist, pkg) + check_sdist(dist, pkg) + else: + report( + "ERROR", + "dist", + f"{dist} does not exist — run `uv build` first, or pass --source-only", + ) + + if failures: + print(f"FAIL: {failures} error(s)") + return 1 + print("OK: py.typed verified") + return 0 + + +if __name__ == "__main__": + sys.exit(main())