From 24b1e713c3035edcd363cd68859a32589df50a2f Mon Sep 17 00:00:00 2001 From: Jairo Gelpi Date: Tue, 15 Sep 2026 13:17:57 +0200 Subject: [PATCH] fix: align alpha scope and add stable release validation gates --- .github/workflows/ci.yml | 40 +++++++++----- CHANGELOG.md | 15 ++++++ README.md | 45 +++++++++------- benchmarks/external/README.md | 55 +++++++++++++++++++ docs/RELEASE_CRITERIA.md | 68 +++++++++++++++++++++++ docs/VALIDATION.md | 86 ++++++++++++++++++++++++++++++ requirements/traceability.yaml | 6 +-- scripts/check_distribution.py | 51 ++++++++++++++++++ src/answerable/__init__.py | 2 +- tests/test_distribution_version.py | 19 +++++++ 10 files changed, 349 insertions(+), 38 deletions(-) create mode 100644 docs/RELEASE_CRITERIA.md create mode 100644 docs/VALIDATION.md create mode 100644 scripts/check_distribution.py create mode 100644 tests/test_distribution_version.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11dab98..7b4457d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,13 +15,20 @@ concurrency: jobs: quality: - name: Quality / Python ${{ matrix.python-version }} - runs-on: ubuntu-latest + name: Quality / ${{ matrix.os }} / Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} timeout-minutes: 15 strategy: fail-fast: false matrix: + os: [ubuntu-latest] python-version: ["3.11", "3.12"] + include: + - os: windows-latest + python-version: "3.12" + defaults: + run: + shell: bash steps: - uses: actions/checkout@v4 with: @@ -59,6 +66,9 @@ jobs: || test $? -eq 2 answerable --json warrant verify --warrant runs/campaign_retention/warrant.json + - name: Run external-data regression evaluation + run: python scripts/evaluate_external.py --output runs/external-ci + - name: Run Epistemic Mutation Testing release gate if: matrix.python-version == '3.12' run: answerable benchmark mutations --output runs/epistemic-mutations @@ -70,24 +80,26 @@ jobs: run: python scripts/validate_schemas.py package: - name: Package - runs-on: ubuntu-latest - timeout-minutes: 10 + name: Package / ${{ matrix.os }} / Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.11", "3.12"] + include: + - os: windows-latest + python-version: "3.12" steps: - uses: actions/checkout@v4 with: persist-credentials: false - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} cache: pip - run: python -m pip install --disable-pip-version-check build - run: python -m build - - name: Clean wheel install with MCP extra - run: | - python -m venv /tmp/answerable-wheel - WHEEL="$(ls dist/*.whl)" - /tmp/answerable-wheel/bin/python -m pip install "${WHEEL}[mcp]" - /tmp/answerable-wheel/bin/python -c "import answerable; print(answerable.__version__)" - /tmp/answerable-wheel/bin/python -c "from answerable.interfaces.mcp_stdio import build_server; build_server(); print('wheel MCP OK')" - /tmp/answerable-wheel/bin/answerable benchmark mutations --output /tmp/answerable-wheel-benchmark + - name: Clean wheel install, version agreement, CLI and MCP + run: python scripts/check_distribution.py diff --git a/CHANGELOG.md b/CHANGELOG.md index af7bddc..7760bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Added +- Explicit validation evidence and measurable stable-release criteria, including independent held-out questions, paired real-agent evaluation and external pilots. +- Windows/Python 3.12 CI alongside Linux/Python 3.11 and 3.12; clean-wheel checks on both operating systems. +- Regression checks for agreement between distribution, public Python and CLI versions. + +### Changed +- README now identifies the package as alpha and distinguishes implemented checks from the complete target specification and historical benchmark results. +- External evaluation protocol defines independent review, dataset-level holdout, error accounting and paired comparison criteria; these planned evaluations have not been completed. +- Causal identification requires explicit named assumptions and reports conditional support separately from verified facts. Descriptive questions no longer require causal identification or inferential power (PR #22). + +### Fixed +- Overlap no longer substitutes for exchangeability; support is required in every target stratum. Deterministic blockers govern claim lists and findings link to their producing checks (PR #22). +- Public `answerable.__version__` agrees with package and CLI version `0.3.0` instead of reporting `0.2.0`. + + ## [0.3.0] - 2026-08-17 ### Added diff --git a/README.md b/README.md index d8be523..44dd18a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### Evidence before answers. -**A deterministic validity layer for analytics and AI agents.** +**Explicit data and analysis checks for analytics and AI agents.** Your code has tests. Your data has tests. **Your conclusions should too.** @@ -20,13 +20,15 @@ Your code has tests. Your data has tests. **Your conclusions should too.** > [!IMPORTANT] -> **A correct number does not imply a justified conclusion.** Answerable checks whether the available evidence supports the claim an analyst or agent is about to make — and fails closed when it does not. +> **Status: alpha.** Answerable checks explicit data and analysis conditions within its implemented scope. Its reports separate verified facts, declared assumptions and conditions not verifiable from the supplied table. +> +> CLI and MCP have automated tests. The external-data evaluation uses two datasets and 12 project-authored questions. Improvement from giving an agent Answerable has **not yet been demonstrated with real models**. See [validation evidence](docs/VALIDATION.md) and [stable release criteria](docs/RELEASE_CRITERIA.md). ![Answerable terminal demo](docs/demo.svg) ### AI agent + Answerable -The agent explores and explains; Answerable deterministically checks whether the evidence supports the conclusion before it reaches a user. +The agent explores and explains; Answerable checks the declared conditions before the agent publishes a conclusion. This is the intended integration workflow, not an experimentally demonstrated improvement in agent decisions. AI agent workflow with and without Answerable @@ -84,7 +86,9 @@ A 100% score on project-authored rules/cases is not general superiority over LLM ## Quickstart -### 1. Install the production package +### 1. Install the published alpha package + +PyPI contains tagged releases; unreleased changes on `main` require a source installation. For CLI + MCP integration: @@ -152,20 +156,19 @@ Most analytical tooling stops at one of three boundaries: whether the data is st > **Does this evidence justify this conclusion?** -| Capability | Data tests | Statistical code | LLM-as-judge | Answerable | -| --- | ---: | ---: | ---: | ---: | -| Check schema/nulls/duplicates | Yes | Sometimes | Sometimes | **Yes** | -| Check grain and joins | Sometimes | Manual | Sometimes | **Yes** | -| Check temporal maturity | Rarely | Manual | Sometimes | **Yes** | -| Check causal identification | No | Manual | Nondeterministic | **Yes** | -| Check prediction-time leakage | Rarely | Manual | Nondeterministic | **Yes** | -| Decide whether a claim must be retracted | No | No | Nondeterministic | **Yes** | -| Produce machine-readable blockers | Sometimes | Custom | Variable | **Yes** | -| Produce allowed/forbidden claims | No | No | Variable | **Yes** | -| Produce a verifiable evidence artifact | No | No | No | **Evidence Warrant** | -| Deterministic release gate | Yes | Possible | No | **Yes** | - -Answerable is not trying to replace data-quality frameworks, statistical libraries or LLMs. It is the **claim-validity layer between them and the statement that reaches a human or downstream agent**. +| Implemented surface | What it checks or produces | Boundary | +| --- | --- | --- | +| File assessment | Declared entity grain, duplicates, timing and configured checks | Assessment currently evaluates the first source; this is not arbitrary multi-table validation. | +| Descriptive comparisons | Observed group means and admissibility under the declared contract | No general numerical or semantic verification of arbitrary prose. | +| Causal gate | Declared identifying assumptions, adjustment mapping and empirical binary-treatment support | Overlap is not exchangeability. Declared assumptions remain unverified; no adjusted causal effect estimation. | +| Other analytical checks | Explicit metadata for timing leakage, metric changes, power and missingness in supported cases | Detector coverage is bounded; it does not certify arbitrary predictive or statistical analyses. | +| Verdict and claim lists | Deterministic blockers and permitted candidate claim classes | Consumers must read the verdict, blockers and assumptions as well as the claim list. | +| Evidence Warrant | Findings, provenance and integrity verification | Integrity verification does not establish scientific truth or independent certification. | +| CLI, Python and MCP | Executable local interfaces | HTTP/web modules are contract surfaces, not a complete hosted product. | + +The complete target specification includes capabilities beyond the current executable +assessment path. A domain module or passing unit test does not establish end-to-end +support for every workflow described in that specification. ## Who it is for @@ -212,7 +215,7 @@ Evidence Warrant ### Frozen benchmark -The `emt-v2` benchmark is hash-addressed and immutable once published. The current release gate runs 112 paired evidence mutations across seven independent failure classes. +The `emt-v2` benchmark is hash-addressed and immutable once published. The current release gate runs 112 paired evidence mutations across seven failure classes. ### Real external-agent runs @@ -222,7 +225,7 @@ The repository includes full prompts, raw responses, decisions and scoring artif The engineering verification path includes branch-aware coverage of at least 95%, strict mypy, Ruff, schema validation, requirement traceability, clean package build/install, deterministic mutation testing and CodeQL. -## Answerable vs real LLM agents +## Historical comparison with real LLM agents On the frozen `emt-v2` case set, Answerable and external LLMs were asked to update a conclusion after the evidence changed. The repository contains the raw runs and scoring artifacts. @@ -233,6 +236,8 @@ On the frozen `emt-v2` case set, Answerable and external LLMs were asked to upda | Overall oracle accuracy | **100%** | 79.0% | 60.7% | | Unsafe `KEEP` | **0%** | 0% | 0% | +These are historical, project-authored oracle scores, not an independent estimate of general reliability. The paired agent-with-tool experiment remains pending. + The interesting failure is not arithmetic. Both models handled familiar leakage failures well, but frequently softened several statistical and causal invalidations into `QUALIFY` instead of retracting the claim. The full methodology, raw responses, sample-size caveats and per-class results are preserved under [`benchmarks/epistemic_mutations/results/2026-08-17-emt-v2-claude-codex/`](benchmarks/epistemic_mutations/results/2026-08-17-emt-v2-claude-codex/). This benchmark measures the models **without access to Answerable**. Giving the model Answerable as a callable tool is a separate intervention; the point of the MCP integration is to move the validity decision outside the model's own judgment. diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md index 2996b89..17d0410 100644 --- a/benchmarks/external/README.md +++ b/benchmarks/external/README.md @@ -104,3 +104,58 @@ responses remain incomplete. Existing run reports cannot be overwritten. Unit-test adapters are test doubles only and are never published as agent results. The recorded preflight currently has **zero real model decisions**. + +## Planned independent acceptance evaluation + +**Status: protocol only; not collected or executed.** The current scripts +materialize the 12 smoke cases above. They do not yet ingest a general independent +question bank, adjudicate labels, calculate dataset-cluster intervals, or fetch +provider billing. Implement and validate those capabilities before using them for +[stable-release acceptance](../../docs/RELEASE_CRITERIA.md). + +1. Collect at least 100 questions across at least 10 externally sourced datasets. + Record source/license, retrieval date, transformations and content hashes. Include + valid, invalid, ambiguous and out-of-scope claims, not just mutations of failures. +2. Split by whole dataset before engine evaluation: reserve at least five datasets + with at least 40 valid and 40 invalid questions in total. Keep development cases + separate; do not tune on reserved data. More questions than the minimum may be + needed to satisfy the split and coverage requirements. +3. An independent domain reviewer, who did not implement the relevant checks, labels + each question before seeing engine output. Store claim, scope, expected decision, + factual basis, required assumptions, severity, reasoning and reviewer/date. + Record disagreements and adjudication; unresolved cases remain ambiguous. +4. Include duplicates, missing values, temporal ambiguity, immature cohorts, partial + overlap, confounding, mismatched adjustments, metric changes, leakage and valid + descriptive questions. Include unsupported formats/designs explicitly. Do not + expect support beyond the advertised scope or secretly remove those cases. +5. Freeze labels, split, hashes, engine revision, metrics and release thresholds. + Run every case and retain exceptions/timeouts. For valid/invalid labels report + both claim-list and verdict-gated decisions. Report ambiguous/out-of-scope cases + separately, with the engine's actual response and whether it states a limitation. +6. Publish numerator/denominator for unsafe allows and false blocks, execution + errors, and per-dataset/per-class counts. Give uncertainty at the dataset level; + zero observed failures is not proof of zero risk. An errored case has no valid + decision and cannot be counted as a correct block. + +### Paired real-model extension + +Use exactly the frozen questions/evidence in both arms, a pinned model version, +identical base prompts, fresh sessions, counterbalanced order and a declared budget. +Keep the existing intention-to-treat policy: tool availability does not imply use. +Record all prompts, responses, real tool calls, errors and latency. Capture provider +usage and cost when the adapter exposes them; otherwise mark cost unavailable. +Never infer actual billed cost from a guessed token count. + +Primary outcome: unsafe-allow rate alone minus unsafe-allow rate with the tool on +invalid questions. Also report valid false-block differences, regressions, +improvements, complete pairs and tool usage. Apply identical scoring in both arms. +Use paired dataset-cluster resampling for a 95% interval; repeated calls and related +questions stay in the same cluster. Freeze the resampling seed and method before +running models. The release gate requires a positive safety improvement interval, +a false-block increase of no more than two percentage points and a complete run. +A small/inconclusive interval or a baseline with zero unsafe allows does not establish +benefit; retain the result and extend the evaluation prospectively. + +The present 48-decision smoke plan does not satisfy this independent protocol. +Passing test-double adapters verifies plumbing only. A native MCP host evaluation +would be a separately specified experiment; do not conflate it with mediated calls. diff --git a/docs/RELEASE_CRITERIA.md b/docs/RELEASE_CRITERIA.md new file mode 100644 index 0000000..ce712b3 --- /dev/null +++ b/docs/RELEASE_CRITERIA.md @@ -0,0 +1,68 @@ +# Stable release criteria + +Current decision: **NOT READY for a stable release**. Alpha/pilot use is possible +within the documented scope. Passing software CI alone cannot change this decision. +These are prospective acceptance criteria, not claims of completed validation. +Freeze their version with the evaluation manifest before exposing held-out results. + +## Scope of the proposed stable package + +The release promises checks of explicit data and analysis conditions, deterministic +blockers and traceable reports through Python, CLI and MCP. It does not promise +universal conclusion verification, automatic causal proof, adjusted causal effect +estimation or a finished hosted application. PRODUCT_SPEC.md remains the target +implementation contract; these gates do not waive its applicable invariants. + +## Gates + +| Gate | Acceptance criterion | Current state | Evidence owner | +| --- | --- | --- | --- | +| Scope and documentation | Each advertised capability maps to an executable example/test and a stated boundary | Documentation supplied; candidate review required | Maintainer | +| Software quality | Full CI passes with >=95% branch-aware coverage, strict types, lint, schemas and traceability; no weakened thresholds | Previous Linux baseline passed; candidate CI required | CI + maintainer | +| Distribution | Clean wheel installs on Linux 3.11/3.12 and Windows 3.12; public/CLI/distribution versions agree; CLI and MCP work outside checkout | Candidate matrix/checks added; CI required | CI | +| Critical regression safety | Zero failures in the predefined critical suite: integrity blockers, unsupported causal claims, provenance and warrant tampering | Existing regressions passed; candidate CI required | CI + reviewer | +| Independent evaluation | At least 100 reviewed questions across 10 external datasets; at least five whole datasets reserved, containing >=40 valid and >=40 invalid questions in total | Pending | Independent domain reviewer | +| Held-out safety | Zero allowed critical-invalid claims; <=5% unsafe allows among all invalid claims; <=5% false blocks among valid claims, at both claim-list and primary-gate levels | Pending | Evaluation owner | +| Scope recognition and reliability | All predefined unsupported/ambiguous cases return an explicit limitation or block; zero execution errors in the fixed acceptance run | Pending | Evaluation owner | +| Agent integration benefit | Positive reduction in unsafe allows with a 95% dataset-cluster interval excluding zero; false-block increase <=2 percentage points; complete intended comparison matrix | Not run | Evaluation owner + authenticated model access | +| User pilots | At least three external users over two weeks; each completes a predefined task from published instructions without maintainer intervention; failures documented | Pending | Pilot users + maintainer | +| Release review | No open critical security, data-loss, disclosure or verdict/claim inconsistency issue; migration notes, candidate artifact digest and rollback instructions recorded | Pending | Maintainer | + +These thresholds are engineering acceptance choices, not guarantees about all data. +Publish counts, denominators and uncertainty even when all observed cases pass. +Zero observed failures is not a zero population error rate. More datasets may be +needed for meaningful uncertainty estimates; five held-out clusters is a floor, +not a justification to claim a precise efficacy result. + +## Evaluation rules + +Follow the [external protocol](../benchmarks/external/README.md). Keep development +and reserved datasets disjoint, preserve disputed/ambiguous labels, and freeze +questions, claims, labels, hashes and thresholds before the candidate run. +Errors and missing responses are never dropped or relabelled as safe blocks. +Report per-dataset and per-claim-class results as well as pooled counts. + +For agent comparison, hold model version, evidence, prompts, budget and repetition +policy fixed except for tool availability. Report tool usage, complete pairs, +improvements/regressions, latency and actual provider usage/cost when available. +Resample whole datasets for uncertainty; repeated calls and mutations are not +independent observations. The acceptance gate requires the complete planned run; +incomplete runs may be published as incomplete but cannot satisfy the gate. +If the agent already makes no unsafe decisions, safety improvement is inconclusive, +not a successful superiority claim. Broaden the prospectively defined evaluation. + +## Candidate and release record + +1. Maintainer nominates a commit and builds a release candidate with an explicit + version. Record its wheel digest, dependencies, OS/Python matrix and CI URLs. +2. Evaluation owner freezes manifests and runs all gates on that candidate. +3. Independent reviewer checks labels, disagreements, limitations and results. +4. Maintainer records each gate as pass, fail or pending with a supporting URL. + Missing evidence means pending, never pass. +5. After all gates pass, publish versioned release notes and installation examples. + Archive the previous known-good artifact and explain how to reinstall its version. + +If a fix follows held-out failure, retain the failed result. The exposed cases become +regressions; obtain a fresh reserved set before making a new generalization claim. +Never lower a threshold after observing results to make the candidate pass. +No release tag or PyPI publication is performed merely by adding this document. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md new file mode 100644 index 0000000..f86eb07 --- /dev/null +++ b/docs/VALIDATION.md @@ -0,0 +1,86 @@ +# Validation evidence + +Status: alpha. Evidence reviewed on 2026-09-15. This document separates software +verification from evidence of analytical validity and integration benefit. +The complete target in PRODUCT_SPEC.md is not a statement that all workflows +are implemented or independently validated. + +## Recorded evidence + +The following results concern commit `c8fe2dae8053248dd519c1dd5549ad3e5d72174e` +(PR #22), before the release-readiness changes. They are not new results for every +future commit. The additional version regression test must pass on the next revision. + +| Check | Observed result | Scope | +| --- | --- | --- | +| Local complete verification | 215 tests, 96.01% branch-aware coverage; Ruff and strict mypy passed | Software regression checks | +| Schemas and traceability | 21 schemas and 119 requirement entries passed | Structure and traceability, not scientific validation | +| Post-merge CI | Linux Python 3.11/3.12 and clean wheel passed | [CI run](https://github.com/Jairogelpi/answerable_data/actions/runs/34944933551) | +| CodeQL | Passed | [Run](https://github.com/Jairogelpi/answerable_data/actions/runs/34944933538); no claim of comprehensive security certification | +| Local CLI and MCP | Three demos, actual stdio initialize/list/assessment call, valid warrant accepted and altered warrant rejected | Local Linux manual checks; not real-agent efficacy | +| EMT | 112 mutation pairs passed | Project-authored scenarios/oracles, correlated cases | +| External data | Four valid claims retained; eight invalid claims blocked; zero execution errors | Two external datasets, 12 project-authored questions | +| Public version | Python reported 0.2.0 while package/CLI reported 0.3.0 | Release-readiness change corrects this inconsistency | + +External before/after reports, hashes and limitations are preserved in +[the recorded evaluation](../benchmarks/external/results/2026-09-14/README.md). +In that comparison, invalid entries in `allowed_claims` fell from 8/8 to 0/8, +and valid claims denied by the primary gate fell from 2/4 to 0/4. +These small, targeted counts do not estimate general error rates. + +Historical Claude/Codex EMT runs compare agents without Answerable against the +engine on frozen project oracles. They do not measure the effect of giving an +agent the tool. The new paired harness records **not_run**, zero actual decisions +and null metrics until an authenticated real-model adapter is provided. + +## Current boundaries + +- The runner currently assesses the first data source. Multi-source contracts or + connector primitives do not establish general multi-table assessment support. +- Causal support is conditional on explicit assumptions; absence of unmeasured + confounding cannot be verified from empirical overlap. +- Full discrete-stratum overlap is conservative for continuous covariates. +- Observed group means are descriptive; adjusted causal effect estimation and + general verification of arbitrary prose/numbers are not implemented. +- Warrant verification checks integrity. It is not a scientific truth guarantee. +- CLI, Python and stdio MCP are executable interfaces. HTTP/web contract modules + do not constitute a complete hosted product. + +## Reproduce software and smoke evaluation + +Use a clean Python 3.11 or 3.12 environment from the checkout: + +```bash +python -m pip install -e ".[dev]" +python -m ruff format --check . +python -m ruff check . +python -m mypy +python -m pytest +python scripts/validate_schemas.py +python scripts/check_traceability.py +python scripts/evaluate_external.py --output runs/external-validation +python -m build +python scripts/check_distribution.py +``` + +On Windows, avoid global Python/plugin contamination by calling the environment +interpreter explicitly (activation is optional): + +```powershell +py -3.12 -m venv .venv +.\.venv\Scripts\python.exe -m pip install -e ".[dev]" +.\.venv\Scripts\python.exe -m pytest +``` + +An interrupted pytest run is incomplete. A version-specific CI pass is not +proof of compatibility with untested Python versions or operating systems. +The clean-distribution script installs the built wheel into an isolated temporary +venv and tests public version, CLI, MCP construction and the mutation benchmark +outside the source checkout. It requires access to package dependencies. + +## Missing evidence + +Independent questions and labels, dataset-level held-out evaluation, real-model +paired results, and external pilots remain pending. Windows jobs are introduced +in the release-readiness change; inspect that revision's CI before claiming a pass. +[Release criteria](RELEASE_CRITERIA.md) define the gates and their evidence owners. diff --git a/requirements/traceability.yaml b/requirements/traceability.yaml index 044272a..56838ac 100644 --- a/requirements/traceability.yaml +++ b/requirements/traceability.yaml @@ -365,9 +365,9 @@ requirements: tests: ["tests/test_interfaces.py"] status: verified FR-CLI-001: - implementation: ["src/answerable/cli.py"] - tests: ["tests/test_interfaces.py"] - status: verified + implementation: ["src/answerable/cli.py", "src/answerable/__init__.py"] + tests: ["tests/test_interfaces.py", "tests/test_distribution_version.py"] + status: implemented FR-MCP-001: implementation: ["src/answerable/interfaces/mcp.py"] tests: ["tests/test_interfaces.py"] diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py new file mode 100644 index 0000000..ff4fafb --- /dev/null +++ b/scripts/check_distribution.py @@ -0,0 +1,51 @@ +"""Install the built wheel and check its public interfaces outside the checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +def main() -> None: + wheels = list((Path(__file__).resolve().parents[1] / "dist").glob("*.whl")) + if len(wheels) != 1: + raise SystemExit("Expected exactly one wheel in dist; build in a clean checkout.") + env = os.environ.copy() + env.pop("PYTHONPATH", None) + env["PYTHONNOUSERSITE"] = "1" + with tempfile.TemporaryDirectory(prefix="answerable-wheel-") as folder: + root = Path(folder) + venv = root / "venv" + subprocess.run([sys.executable, "-m", "venv", str(venv)], check=True, env=env) + bin_dir = venv / ("Scripts" if os.name == "nt" else "bin") + python = str(bin_dir / ("python.exe" if os.name == "nt" else "python")) + cli = str(bin_dir / ("answerable.exe" if os.name == "nt" else "answerable")) + + def run(args: list[str]) -> None: + subprocess.run(args, check=True, cwd=root, env=env) + + run([python, "-m", "pip", "install", f"{wheels[0]}[mcp]"]) + run( + [ + python, + "-c", + "import json, subprocess, sys; from pathlib import Path; " + "from importlib.metadata import version; import answerable; " + "v=version('answerable-data'); " + "assert Path(answerable.__file__).is_relative_to(Path(sys.prefix)); " + "assert answerable.__version__ == v; " + "d=json.loads(subprocess.check_output([sys.argv[1], '--json', 'doctor'])); " + "assert d['version'] == v and d['status'] == 'ready'; " + "from answerable.interfaces.mcp_stdio import build_server; build_server(); " + "print('Installed wheel versions, CLI and MCP: PASS', v)", + cli, + ] + ) + run([cli, "benchmark", "mutations", "--output", str(root / "benchmark")]) + + +if __name__ == "__main__": + main() diff --git a/src/answerable/__init__.py b/src/answerable/__init__.py index d926231..ff983c4 100644 --- a/src/answerable/__init__.py +++ b/src/answerable/__init__.py @@ -14,4 +14,4 @@ "assess", "verify_warrant", ] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/tests/test_distribution_version.py b/tests/test_distribution_version.py new file mode 100644 index 0000000..3fb5321 --- /dev/null +++ b/tests/test_distribution_version.py @@ -0,0 +1,19 @@ +"""FR-CLI-001: public version identifiers agree with the installed distribution.""" + +import json +import tomllib +from importlib.metadata import version +from pathlib import Path + +import pytest + +import answerable +from answerable.cli import main + + +def test_FR_CLI_001_public_versions_match_distribution(capsys: pytest.CaptureFixture[str]) -> None: + project = tomllib.loads((Path(__file__).parents[1] / "pyproject.toml").read_text()) + assert main(("--json", "doctor")) == 0 + doctor = json.loads(capsys.readouterr().out) + assert answerable.__version__ == version("answerable-data") + assert doctor["version"] == project["project"]["version"] == answerable.__version__