diff --git a/install.sh b/install.sh index 0fec5265..009fbe83 100755 --- a/install.sh +++ b/install.sh @@ -145,7 +145,7 @@ copy_file() { fi } -for script in super-board-skills-ready.py super-board-calibration.py super-board-verification-ready.py super-board-github-read.py super-board-collect-state.py super-board-run.sh super-board-gh-guard.sh super-board-card.sh super-board-status.py super-board-wave-plan.sh super-board-codex-lane.sh super-board-codex-wave.sh super-board-deps.sh super-board-preflight.sh super-board-merge-gate.sh super-board-merge-policy.py super-board-approval.py super-board-env-check.sh super-board-agents-md.py super-board-settings.py super-board-setup.py super-board-usage.sh super-board-throttle.sh super-board-host.sh super-board-pr-body.sh super-review-file-refactor.sh super-qa-file-bug.sh super-board-stop.sh; do +for script in super-board-project-verifier.py super-board-skills-ready.py super-board-calibration.py super-board-verification-ready.py super-board-github-read.py super-board-collect-state.py super-board-run.sh super-board-gh-guard.sh super-board-card.sh super-board-status.py super-board-wave-plan.sh super-board-codex-lane.sh super-board-codex-wave.sh super-board-deps.sh super-board-preflight.sh super-board-merge-gate.sh super-board-merge-policy.py super-board-approval.py super-board-env-check.sh super-board-agents-md.py super-board-settings.py super-board-setup.py super-board-usage.sh super-board-throttle.sh super-board-host.sh super-board-pr-body.sh super-review-file-refactor.sh super-qa-file-bug.sh super-board-stop.sh; do if [ -f "$REPO_ROOT/scripts/$script" ]; then copy_file "$REPO_ROOT/scripts/$script" "$TARGET/.claude/bin/$script" chmod +x "$TARGET/.claude/bin/$script" diff --git a/scripts/super-board-project-verifier.py b/scripts/super-board-project-verifier.py new file mode 100755 index 00000000..1ec18652 --- /dev/null +++ b/scripts/super-board-project-verifier.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Read-only project verifier map inspection and affected-feature selection. + +Exit 0: structurally valid (no scenarios executed); 9: missing/stale setup; +64: malformed input. Uses project files only, without git, shell or host access. +""" +import argparse +import fnmatch +from functools import lru_cache +import glob +import hashlib +import json +import ntpath +from pathlib import Path +import sys + + +class InputError(Exception): + def __init__(self, message, code=64): + super().__init__(message) + self.code = code + + +def relative(value, label, pattern=False): + if not isinstance(value, str) or not value.strip(): + raise InputError(f'{label}: expected a nonempty repo-relative path') + if ('\x00' in value or '\\' in value or ntpath.splitdrive(value)[0] + or value.startswith('/') or any(part in ('', '.', '..') for part in value.split('/'))): + raise InputError(f'{label}: absolute, traversal or noncanonical path') + if not pattern and glob.has_magic(value): + raise InputError(f'{label}: patterns are allowed only in owners') + return value + + +def contained(root, path, label): + try: + resolved = path.resolve() + resolved.relative_to(root) + except (ValueError, OSError, RuntimeError): + raise InputError(f'{label}: path escapes the project or has a broken symlink', 9) + return resolved + + +def read_input(root, name, label): + relative(name, label) + path = contained(root, root / name, label) + try: + if not path.is_file() or not path.stat().st_mode & 0o444: + raise OSError('not a readable file') + data = path.read_bytes() + if not data.strip(): + raise OSError('empty file') + except OSError: + raise InputError(f'{label}: missing, empty or unreadable file: {name}', 9) + return data, dict(path=path.relative_to(root).as_posix(), sha256=hashlib.sha256(data).hexdigest()) + + +def text(value, label): + if not isinstance(value, str) or not value.strip(): + raise InputError(f'{label}: expected nonempty text') + + +def behavior(value, label): + if isinstance(value, list): + if not value: + raise InputError(f'{label}: document behavior or explicit absence') + for item in value: + text(item, label) + else: + text(value, label) + + +def owner_matches(root, pattern, label): + """Expand only within the project; validate intermediate matches before descent.""" + components, visited = pattern.split('/'), set() + + def walk(path, index): + resolved = contained(root, path, label) + if index == len(components): + if path.is_file(): + yield path.relative_to(root).as_posix() + return + key = (resolved, index) + if key in visited or not path.is_dir(): + return + visited.add(key) + component = components[index] + if component == '**': + yield from walk(path, index + 1) + for child in sorted(path.iterdir()): + if component == '**': + contained(root, child, label) + if child.is_dir(): + yield from walk(child, index) + elif index + 1 == len(components) and child.is_file(): + yield child.relative_to(root).as_posix() + elif fnmatch.fnmatchcase(child.name, component): + # A matching directory symlink is checked even if the remaining + # pattern would find no files, so another owner cannot hide it. + yield from walk(child, index + 1) + + yield from walk(root, 0) + + +def load_map(root, name): + raw, map_input = read_input(root, name, 'map') + try: + data = json.loads(raw) + except (ValueError, UnicodeError): + raise InputError('map: expected valid UTF-8 JSON') + if not isinstance(data, dict) or type(data.get('version')) is not int or data['version'] != 1: + raise InputError('map: expected object with version 1') + text(data.get('app'), 'app') + _, skill_input = read_input(root, data.get('skill'), 'skill') + features = data.get('features') + if not isinstance(features, list) or not features: + raise InputError('features: expected a nonempty array') + ids, scenarios, owners = [], [], {} + for feature in features: + if not isinstance(feature, dict): + raise InputError('feature: expected object') + name = feature.get('id') + text(name, 'feature id') + if name in ids: + raise InputError(f'duplicate feature id: {name}') + ids.append(name) + for key in ['description', 'route', 'witness']: + text(feature.get(key), f'{name}.{key}') + for key in ['expected', 'side_effects']: + behavior(feature.get(key), f'{name}.{key}') + _, scenario = read_input(root, feature.get('scenario'), f'{name}.scenario') + if scenario not in scenarios: + scenarios.append(scenario) + patterns = feature.get('owners') + if not isinstance(patterns, list) or not patterns: + raise InputError(f'{name}.owners: expected nonempty path/glob array') + matches = set() + for pattern in patterns: + relative(pattern, f'{name}.owners', pattern=True) + matches.update(owner_matches(root, pattern, f'{name}.owners')) + if not matches: + raise InputError(f'{name}.owners: no existing file match', 9) + owners[name] = (patterns, matches) + return dict(valid=True, app=data['app'], features=ids, + inputs=dict(map=map_input, skill=skill_input, scenarios=scenarios)), owners + + +def matches_owner(path, pattern): + """Root-anchored glob matching, including deleted paths and ** zero directories.""" + parts, patterns = path.split('/'), pattern.split('/') + + @lru_cache(None) + def match(i, j): + if j == len(patterns): + return i == len(parts) + if patterns[j] == '**': + return match(i, j + 1) or (i < len(parts) and match(i + 1, j)) + return i < len(parts) and fnmatch.fnmatchcase(parts[i], patterns[j]) and match(i + 1, j + 1) + + return match(0, 0) + + +class Parser(argparse.ArgumentParser): + def error(self, message): + raise InputError(message) + + +def main(): + parser = Parser(description=__doc__) + parser.add_argument('command', choices=['inspect', 'select']) + parser.add_argument('--root', required=True) + parser.add_argument('--map', required=True) + parser.add_argument('--changed', help='JSON changed-path array file, or - for stdin') + parser.add_argument('--feature', action='append', default=[]) + try: + args = parser.parse_args() + try: + root = Path(args.root).resolve() + except (OSError, RuntimeError): + raise InputError('root: project path is inaccessible or has a symlink loop', 9) + if not root.is_dir(): + raise InputError('root: project directory is missing', 9) + result, owners = load_map(root, args.map) + if args.command == 'inspect' and (args.changed is not None or args.feature): + raise InputError('inspect does not accept --changed or --feature') + if args.command == 'select': + if args.changed is None: + raise InputError('select requires --changed (use [] for explicit-only selection)') + try: + raw = sys.stdin.read() if args.changed == '-' else read_input(root, args.changed, 'changed')[0] + changed = json.loads(raw) + except (ValueError, UnicodeError): + raise InputError('changed: expected a JSON path array') + if not isinstance(changed, list): + raise InputError('changed: expected a JSON path array') + for path in changed: + relative(path, 'changed path') + contained(root, root / path, 'changed path') + unknown = set(args.feature) - set(result['features']) + if unknown: + raise InputError('unknown feature id: ' + ', '.join(sorted(unknown))) + selected, covered = set(args.feature), set() + for feature, (patterns, matches) in owners.items(): + hits = {path for path in changed if path in matches or + any(matches_owner(path, pattern) for pattern in patterns)} + if hits: + selected.add(feature) + covered.update(hits) + result['selected'] = [name for name in result['features'] if name in selected] + result['uncovered_paths'] = list(dict.fromkeys(path for path in changed if path not in covered)) + print(json.dumps(result, indent=2)) + return 0 + except InputError as error: + print(json.dumps(dict(valid=False, error=str(error)))) + return error.code + except OSError: + print(json.dumps(dict(valid=False, error='unreadable project input'))) + return 9 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/super-board-setup.py b/scripts/super-board-setup.py index 6410edf4..acfab70f 100755 --- a/scripts/super-board-setup.py +++ b/scripts/super-board-setup.py @@ -71,7 +71,7 @@ } SKILLS = ["super-board", "super-build", "super-qa", "super-review", "super-collect", "visual", "git-sync", "ui-refine-loop"] OLD_SKILL_DIRS = ["super-refine", "cleanup-wt", "arch-loop"] -BIN = ["super-board-skills-ready.py", "super-board-host.sh", "super-board-codex-wave.sh", "super-board-codex-lane.sh", "super-board-calibration.py", "super-board-verification-ready.py", "super-board-github-read.py", "super-board-collect-state.py", "super-board-run.sh", "super-board-gh-guard.sh", "super-board-card.sh", "super-board-status.py", "super-board-wave-plan.sh", +BIN = ["super-board-project-verifier.py", "super-board-skills-ready.py", "super-board-host.sh", "super-board-codex-wave.sh", "super-board-codex-lane.sh", "super-board-calibration.py", "super-board-verification-ready.py", "super-board-github-read.py", "super-board-collect-state.py", "super-board-run.sh", "super-board-gh-guard.sh", "super-board-card.sh", "super-board-status.py", "super-board-wave-plan.sh", "super-board-deps.sh", "super-board-preflight.sh", "super-board-merge-gate.sh", "super-board-merge-policy.py", "super-board-approval.py", "super-board-env-check.sh", "super-board-agents-md.py", "super-board-settings.py", "super-board-setup.py", "super-board-usage.sh", "super-board-pr-body.sh", diff --git a/skills/super-board/references/onboard.md b/skills/super-board/references/onboard.md index 37c3163f..1d55e58e 100644 --- a/skills/super-board/references/onboard.md +++ b/skills/super-board/references/onboard.md @@ -140,6 +140,10 @@ Write it atomically (temp file + rename) after every answer. A halt never loses ├─ verification commands: preserve valid `verify_commands`; otherwise inspect the owning │ project's documented checks and stage those commands. Unresolved checks are a setup │ halt. Never save placeholders or invent a universal runner. + ├─ project verifier: detect `.agents/skills/verify-*/SKILL.md` and feature maps. + │ For creation or upkeep read `../../super-qa/references/project-verifier.md`; + │ stage real project checks only when source names accepted features. Optional + │ verifier gaps never replace or block the required verification commands. └─ machine time zone for config `timezone`: $TZ, else readlink /etc/localtime | sed 's#.*zoneinfo/##', else timedatectl show -p Timezone --value, else "UTC". Shown at step 8. @@ -347,6 +351,9 @@ Write it atomically (temp file + rename) after every answer. A halt never loses │ Board Ledgerly Roadmap #4 (+QA) Branch staging (new) │ Rules safe defaults Perms 7 lines │ AGENTS rules moved, CLAUDE.md→@ Sources sentry, github, prs, arch, linear + ├─ Existing or staged project verifier: list skill/map/runner paths and state + │ `starter, not run` or the actual selected-feature proof. Preserve human-owned + │ critical paths and flows; missing source leaves this optional helper absent. ├─ AskUserQuestion "Write everything?" (header "Save") [Write everything (Recommended) / │ Change one thing → pick the step, walk only it, back here / Cancel — nothing written]. ├─ Write, in order (each atomic; stop and report on the first failure — answers are kept): @@ -361,6 +368,8 @@ Write it atomically (temp file + rename) after every answer. A halt never loses │ │ super-board-settings.py hooks .claude/settings.json /hooks/settings-protect-main.json │ ├─ AGENTS.md / CLAUDE.md: super-board-agents-md.py backup, then write --src │ │ --dest AGENTS.md, pointer --tail --force, block, check + │ ├─ staged project verifier skill/map/scenarios/runner, if source-backed; then + │ │ inspect the saved map and resolve the local skill for the actual host. │ └─ docs/agents/issue-tracker.md; docs/super-board/PROJECT.md (a sub-agent drafts it from │ manifests + README + tree, no question; listed in the table as "PROJECT.md drafted") └─ Delete onboard-staged/; keep the answers file (re-runs use it). diff --git a/skills/super-board/references/run.md b/skills/super-board/references/run.md index e1dfe35f..81eb4bf2 100644 --- a/skills/super-board/references/run.md +++ b/skills/super-board/references/run.md @@ -335,6 +335,9 @@ column), never built unchecked. `qa` cards are not pre-flighted: nothing is buil 2. `target.url` set → health-check it first. Unhealthy → Block. 3. Read issue + PR + Builder's handoff comment. 4. Build issue-scoped test plan: one observable test per AC. +4a. Existing project verifier → follow `../../super-qa/references/project-verifier.md` +for affected feature selection, actual host readiness and new run-owned proof. +Record uncovered/stale optional coverage alongside ordinary issue AC checks. 4b. **Test-gap check** (super-qa → "Test-gap check (after build)"): map every AC to unit / component / e2e tests, hunt edge cases, write the High gaps red-first. A High gap that needs app code changed → Fail (step 7) with the gap list. Medium/Low go in the handoff under `Test gaps (not written)`. 5. Run the tests. Capture evidence to `docs/super-board/runs/issue--qa-v/`. For UI/visual ACs, capture screenshots at the standard viewports (1920×1080 desktop, 1024×768 tablet, 375×667 mobile). Commit the screenshots to the issue branch BEFORE writing the comment (the markdown image URLs depend on the files being present on the branch). 6. **Pass** → commit test files + screenshots to same branch + push → tick the `ac` block with proof lines, write `visual` for UI, append a `history` row, rewrite `status` → `[qa] [report] ✅` PR comment with results + evidence path **+ inline screenshot embeds** (see "Screenshot embed format" below) → issue comment with the SAME inline screenshot embeds → move card QA → Review. Clean up worktree. diff --git a/skills/super-qa/SKILL.md b/skills/super-qa/SKILL.md index bb540ad6..3f6801a7 100644 --- a/skills/super-qa/SKILL.md +++ b/skills/super-qa/SKILL.md @@ -636,12 +636,21 @@ See `.claude/skills/super-board/references/run.md` → Tester (first pass — re 1. Pull latest of base; checkout `issue--` into worktree (a `qa` card: create it from the base). 2. Read issue + PR + Builder's handoff. 3. Build issue-scoped test plan: ONE observable test per AC. +3a. Existing project verifier → follow [project-verifier.md](references/project-verifier.md) +for affected selection, host readiness, real assertions and fresh proof. 3b. Run the **Test-gap check** (above) against the Builder's diff; its High gaps join the plan. 4. Run tests. Capture evidence to `docs/super-board/runs/issue--qa-v/`. **For any UI-affecting issue, capture at least one screenshot per AC** (Playwright `page.screenshot` or `browse --screenshot`). Save with descriptive names: `ac1-.png`, not `screenshot1.png`. 5. **Commit the evidence directory** to the issue branch alongside test files (`git add docs/super-board/runs/issue--qa-v/ && git commit && git push`). This is non-optional — without it, the inline image markdown in the issue comment won't render on GitHub. 6. **Pass** → tick the PR's `ac` block with proof lines, write `visual` for UI, append a `history` row, rewrite `status` (`super-board-pr-body.sh`) → `[qa] [report] ✅` PR comment with results + evidence path → **issue comment with screenshot evidence** (see "Issue-comment evidence format" below) → move QA → Review. Clean up worktree. 7. **Fail** → leave failing ACs unchecked with the reason, append a `history` row, rewrite `status` → `[qa] [report] ❌` PR comment with per-AC expected/actual + repro file:line + evidence path + "what fixed should look like" → issue comment with the failure screenshots → increment rebuild counter → move QA → Ready (label `loop:rebuild-N`). Clean up worktree. +### Optional project feature verifier + +For an existing `.agents/skills/verify-/` map, verifier creation during Onboard, +or source/selector upkeep, read [project-verifier.md](references/project-verifier.md). +Run relevant mapped features alongside issue AC checks; state missing/stale coverage +in the existing handoff. A map inspection or starter template proves no test pass. + ### Optional runtime performance and memory checks For runtime evidence requested by the issue, or a configured diff --git a/skills/super-qa/references/iteration-preamble.md b/skills/super-qa/references/iteration-preamble.md index 57d19cf7..a8efc2f0 100644 --- a/skills/super-qa/references/iteration-preamble.md +++ b/skills/super-qa/references/iteration-preamble.md @@ -9,6 +9,11 @@ Resolve this lane for the actual host, load the report's files/fallbacks, halt on missing requirements and recheck each conditional helper before use. +For existing project verifier coverage relevant to this iteration, follow +[project-verifier.md](project-verifier.md): select mapped features, resolve the +local skill for the actual host, run real checks and preserve proof. State +uncovered/stale coverage in the report; continue ordinary issue checks. + Run **one iteration** of the loop: 1. **Regression** — re-run every spec in `e2e/paths/`. Fix any reds via TDD. 2. **Explore** — pop the top `[ ]` item from `docs/super-qa/queue.md`, diff --git a/skills/super-qa/references/project-verifier.md b/skills/super-qa/references/project-verifier.md new file mode 100644 index 00000000..c9b30f48 --- /dev/null +++ b/skills/super-qa/references/project-verifier.md @@ -0,0 +1,126 @@ +# Project verifier: create, maintain, run + +Use this contract when Onboard can seed a repo-backed app verifier, QA finds an +existing `.agents/skills/verify-/` map, or its instructions need upkeep. +Keep the project's runner, issue AC checks, independent Review and merge policy. +This optional helper adds repeatable feature checks inside existing lanes. + +## Create during Onboard + +1. Read the owning app's instructions, package commands, current source, tests + and decisions. Detect existing verifier skills and maps before staging files. +2. Pick three to five important features with real source owners and accepted + behavior witnesses. Preserve human-owned `critical-paths.md` and `flows.md`. + Insufficient source or setup means leave the optional verifier absent and + state the missing evidence. A starter template proves no app behavior. +3. Adapt `../templates/project-verifier/SKILL.md` and `feature-map.json` into + `.agents/skills/verify-/`. Write real project-owned scenarios and a runner + beside it. Use the app's installed runner; browser/dependency installs and DB + initialization remain separate project decisions. Where supported, Claude + may use a relative `.claude/skills/verify-` compatibility link. +4. Stage the files for Onboard's existing step-8 Save. List them in its review; + write only with that save authorization. Keep `verify_commands` independent: + a map does not turn browser work into every PR's required merge check. +5. Inspect the saved map with the public helper below. Resolve the actual local + skill with `super-board-skills-ready.py --require verify-` for the active + host before loading it. A folder or valid map alone proves no host readiness. + +Templates are unfinished scaffolds: the empty feature array must be replaced. +Onboard reports `starter, not run` until actual scenarios run with proof. + +## Map inspection and selection + +```bash +python3 .claude/bin/super-board-project-verifier.py inspect \ + --root . --map .agents/skills/verify-app/references/feature-map.json +python3 .claude/bin/super-board-project-verifier.py select \ + --root . --map .agents/skills/verify-app/references/feature-map.json \ + --changed - --feature cart.checkout +``` + +Pipe a JSON array of repo-relative changed paths to `--changed -`, or supply a +repo-relative JSON file. `[]` supports explicit feature selection alone. The +helper validates structure and selects IDs; it executes no scenario. Exit 0 +means valid inputs, 9 means missing/stale/unreadable setup, 64 means malformed +input. Unknown IDs fail. Empty selection proves no feature. + +Version 1 fields: + +| Field | Contract | +|---|---| +| `version`, `app`, `skill` | integer `1`, nonempty app name, repo-relative canonical `SKILL.md` | +| `features` | nonempty array; unique stable `id` and nonempty `description` | +| `route`, `witness` | route or non-browser entry point; existing test/decision reference | +| `owners` | nonempty repo-relative path/glob array; at least one existing file per feature | +| `scenario` | nonempty readable project-owned scenario file | +| `expected`, `side_effects` | nonempty text or string array; observable assertions and explicit mocks/absence | + +Extra project fields are retained in the map but not interpreted. Globs are +root-anchored, case-sensitive; `*` stays in one path component and `**` spans +zero or more. Shared owners select several features. Deleted changed paths +still match. Output contains ordered IDs, uncovered paths, canonical relative +map/skill/scenario paths and hashes of their actual bytes, never file contents. +Absolute/traversal/NUL paths and input symlinks escaping the repo are rejected. +Owner globs must not traverse symlinks outside the repo. + +## Run affected features + +QA reads ACs, diff, map and latest source to choose relevant features. With no +map, use ordinary QA. Missing or stale optional coverage goes in the handoff; +it never counts as a green test and creates no new board-wide halt. If a relevant +verifier cannot run, keep its ACs unverified until another valid check proves +them. Resolve its skill for the current host before loading it, for example: + +```bash +python3 .claude/bin/super-board-skills-ready.py --root . --host codex \ + --cards cards.json --lane qa --require verify-app +``` + +`cards.json` is the current hydrated card selection accepted by the resolver. +Keep the active host/catalog options from the lane's readiness check. + +The app owns its runner interface. Document `doctor`, affected selection and an +explicit full sweep, such as `run --feature --output ` and +`run --all`. Complete these operations in order: + +1. **Select:** record IDs, uncovered changed paths and additional issue AC checks. +2. **Doctor:** verify owning checkout/revision, installed runner/browser, allowed + local target, fixture/reset mode, new output directory and process ownership. + Check setup without reading credentials. +3. **Launch:** use a private port and the documented command; refuse occupied + ports, wait for real readiness, record actual checkout and command. +4. **Drive:** reset each scenario with a fresh context or equivalent; use semantic + or test-ID controls, deterministic inputs and bounded waits. +5. **Inspect:** assert visible behavior and relevant persisted/network effects. + Separate mocked effects from observed ones. Capture console/page/request + errors; allowances name only deliberately injected failures. +6. **Proof:** record each feature's assertions and passed/failed/not-run result, + actual HEAD and dirty state, input/scenario/manifest hashes, target/environment, + exact rerun command, screenshots/logs and limits. Put rebuilt output in the + project's dated scratch location; keep canonical inputs in source history. +7. **Cleanup:** close contexts and stop only owned processes. Preserve failing + proof. Cleanup failure makes the run fail. + +Keep the existing `Local tests:` command and per-AC proof in QA's handoff; add +selected verifier results and uncovered paths there. Follow the project's +privacy and existing screenshot publishing rules. Reviewer reruns relevant +checks on the reviewed head into fresh output and compares revisions/hashes. +Old-head proof cannot approve changed code. + +## Maintain on demand + +Triggers: explicit upkeep request, changed mapped source, setup/selector failure, +unmapped important feature, or an authorized expectation change. Read current +source and the live local app against issue ACs, witnesses and newest decisions. + +| Finding | Response | +|---|---| +| Stale instructions | Accepted behavior still holds; fix obsolete route/control/launch guidance or selector, preserve assertions, rerun affected IDs. | +| Harness gap | Repair wrong fixtures/environment/selector, escaped requests, reused server or pre-behavior timeout; otherwise report setup blocked. It proves no product bug. | +| Product regression | Preserve failing evidence for the unchanged accepted behavior; return through Builder/QA and localize the regression test using the project's testing ladder. | +| Accepted product change | Cite explicit issue/decision authorization before changing expected values; record before/after rationale and rerun proof. | + +Report old/new guidance, reason/source, selected rerun and evidence. Preserve +failing mapped features and behavior assertions. Never copy observed output into +expectations or weaken checks to obtain green. No whole-app sweep on every PR, +private-history scan, new worker lane or new lifecycle is required. diff --git a/skills/super-qa/templates/project-verifier/SKILL.md b/skills/super-qa/templates/project-verifier/SKILL.md new file mode 100644 index 00000000..4b486473 --- /dev/null +++ b/skills/super-qa/templates/project-verifier/SKILL.md @@ -0,0 +1,39 @@ +--- +name: verify-app +description: Run affected app feature checks, inspect evidence, or maintain this project's verifier when source or accepted behavior changes. +--- + +# Project verifier starter + +**Starter, not executable coverage.** Before saving, replace `verify-app`, adapt +the map to three to five real features and add project-owned scenarios/runner. +Read the installed super-qa `references/project-verifier.md` create/maintain/run +contract. Keep the owning project's instructions and accepted decisions. + +## Commands and ownership + +Document exact project runner paths and commands for `doctor`, selected features, +explicit `--all` and new run-owned output directories. Name allowed targets, +installed dependencies, reset/fixture policy and owned-process cleanup. Keep +credentials out of commands and proof. Mark unavailable setup explicitly. + +## Feature checks + +Read `references/feature-map.json`, issue ACs, diff and current source. Select +mapped IDs, state uncovered paths and additional AC checks. Resolve this skill +for the actual host before loading it. A valid map is structure, not a test pass. + +## Run and proof + +Write executable steps in this order: Select, Doctor, Launch, Drive, Inspect, +Proof, Cleanup. Define real assertions for each feature and relevant side +effects, including which are mocked. Record HEAD/dirty state, input hashes, +selected IDs, outcomes, screenshots/logs, exact rerun command and limits. +Preserve failure evidence and fail if cleanup fails. + +## Upkeep + +Read current source and live local behavior before editing. Distinguish stale +instructions, harness gaps, product regressions and authorized product changes. +Preserve accepted assertions; cite an issue/decision before changing expectations. +Rerun affected features and report old/new guidance, reason and proof. diff --git a/skills/super-qa/templates/project-verifier/feature-map.json b/skills/super-qa/templates/project-verifier/feature-map.json new file mode 100644 index 00000000..5444da69 --- /dev/null +++ b/skills/super-qa/templates/project-verifier/feature-map.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "app": "Replace with the owning app name", + "skill": ".agents/skills/verify-app/SKILL.md", + "features": [] +} diff --git a/tests/test-setup.sh b/tests/test-setup.sh index 80678a31..fd90b397 100644 --- a/tests/test-setup.sh +++ b/tests/test-setup.sh @@ -31,7 +31,7 @@ TXT=$(python3 "$SETUP" check --root "$T" --text || true) echo "$TXT" | grep -q "✓ super-board skills, scripts and board engine present" || fail "text report should list what is present: $TXT" # 1b — missing safety helpers must trigger a repair even on a current installation. -for helper in super-board-approval.py super-board-github-read.py; do +for helper in super-board-approval.py super-board-github-read.py super-board-project-verifier.py; do rm "$T/.claude/bin/$helper" OUT=$(python3 "$SETUP" check --root "$T" || true) echo "$OUT" | q --arg helper "script $helper" '.missing | index($helper)' \ diff --git a/tests/test_project_verifier.py b/tests/test_project_verifier.py new file mode 100644 index 00000000..e6464a73 --- /dev/null +++ b/tests/test_project_verifier.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Exercise project-map inspection and selection through the shipped CLI.""" +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +KIT = Path(__file__).resolve().parents[1] +CLI = KIT / 'scripts/super-board-project-verifier.py' + + +class ProjectVerifier(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / 'app' + self.root.mkdir() + self.skill = '.agents/skills/verify-example/SKILL.md' + self.map = '.agents/skills/verify-example/references/feature-map.json' + for name, content in [(self.skill, 'Real app instructions'), + ('checks/cart.py', 'assert result == 3'), + ('src/cart.py', 'product code')]: + self.put(name, content) + self.data = dict(version=1, app='Example', skill=self.skill, features=[dict( + id='cart.total', description='Cart totals', route='/cart', owners=['src/*.py'], + scenario='checks/cart.py', witness='checks/cart.test.py:12', + expected='Three items are shown', side_effects=['API responses are mocked'])]) + self.save() + + def put(self, name, content): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + def save(self): + self.put(self.map, json.dumps(self.data)) + + def run_cli(self, command='inspect', changed=None, extra=(), expected=0): + process = subprocess.run([sys.executable, str(CLI), command, '--root', str(self.root), + '--map', self.map, *(['--changed', '-'] if changed is not None else []), *extra], + input=json.dumps(changed) if changed is not None else None, + capture_output=True, text=True) + self.assertEqual(process.returncode, expected, process.stderr or process.stdout) + return json.loads(process.stdout) + + def test_inspect_reports_actual_inputs_without_claiming_a_test_pass(self): + result = self.run_cli() + self.assertTrue(result['valid']) + self.assertEqual(result['features'], ['cart.total']) + self.assertEqual(result['inputs']['map']['path'], self.map) + self.assertEqual(result['inputs']['skill']['sha256'], hashlib.sha256(b'Real app instructions').hexdigest()) + self.assertEqual(result['inputs']['scenarios'][0]['path'], 'checks/cart.py') + self.assertNotIn('passed', result) + self.assertNotIn('Real app instructions', json.dumps(result)) + + def test_selection_unions_shared_owners_and_explicit_ids_in_map_order(self): + self.put('docs/notes.md', 'instructions') + shared = dict(self.data['features'][0], id='cart.checkout') + separate = dict(shared, id='docs.guide', owners=['docs/*.md']) + self.data['features'] += [shared, separate] + self.save() + result = self.run_cli('select', ['src/cart.py', 'README.md', 'README.md'], + extra=['--feature', 'docs.guide']) + self.assertEqual(result['selected'], ['cart.total', 'cart.checkout', 'docs.guide']) + self.assertEqual(result['uncovered_paths'], ['README.md']) + self.assertEqual(self.run_cli('select', [])['selected'], []) + + def test_owner_patterns_are_root_anchored_and_match_deleted_recursive_files(self): + self.data['features'][0]['owners'] = ['src/**/*.py'] + self.save() + result = self.run_cli('select', ['src/deleted.py', 'src/nested/deleted.py', 'other/src/cart.py']) + self.assertEqual(result['selected'], ['cart.total']) + self.assertEqual(result['uncovered_paths'], ['other/src/cart.py']) + + def test_bad_schema_and_unknown_ids_are_usage_errors(self): + self.run_cli('select', [], extra=['--feature', 'unknown'], expected=64) + self.data['features'].append(dict(self.data['features'][0])) + self.save(); self.run_cli(expected=64) + self.data['features'].pop() + for key, value in [('version', True), ('app', ''), ('features', [])]: + saved = self.data[key]; self.data[key] = value + self.save(); self.run_cli(expected=64) + self.data[key] = saved + self.put(self.map, '{'); self.run_cli(expected=64) + + def test_missing_empty_unreadable_and_broken_inputs_are_setup_errors(self): + for name in [self.map, self.skill, 'checks/cart.py']: + path = self.root / name + original = path.read_bytes() + path.write_text(''); self.run_cli(expected=9) + path.write_bytes(original); path.chmod(0) + self.run_cli(expected=9) + path.chmod(0o644); path.unlink(); self.run_cli(expected=9) + path.symlink_to(self.root / 'absent'); self.run_cli(expected=9) + path.unlink(); path.write_bytes(original) + self.data['features'][0]['owners'] = ['missing/*.py'] + self.save(); self.run_cli(expected=9) + + def test_absolute_traversal_nul_and_symlink_escape_are_rejected(self): + for path in ['/tmp/outside', '../outside', 'src/../cart.py', 'C:/outside', 'src\\cart.py', 'src/\x00x']: + self.run_cli('select', [path], expected=64) + for key in ['scenario', 'owners']: + saved = self.data['features'][0][key] + self.data['features'][0][key] = ['../outside'] if key == 'owners' else '../outside' + self.save(); self.run_cli(expected=64) + self.data['features'][0][key] = saved + self.save() + outside = Path(self.temp.name) / 'outside'; outside.mkdir() + (outside / 'file.py').write_text('private') + (self.root / 'escape').symlink_to(outside, target_is_directory=True) + self.run_cli('select', ['escape/file.py'], expected=9) + self.data['features'][0]['owners'] = ['escape/*.py'] + self.save(); self.run_cli(expected=9) + self.data['features'][0]['owners'] = ['*/absent/*.py', 'src/*.py'] + self.save(); self.run_cli(expected=9) + self.data['features'][0]['owners'] = ['src/*.py'] + self.data['features'][0]['scenario'] = 'escape/file.py' + self.save(); self.run_cli(expected=9) + + def test_actual_hash_changes_and_every_operation_preserves_project_bytes(self): + def snapshot(): + return {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob('*') if path.is_file()} + before = snapshot() + initial = self.run_cli() + self.run_cli('select', ['src/cart.py']) + self.run_cli('select', [], extra=['--feature', 'unknown'], expected=64) + self.assertEqual(before, snapshot()) + self.put('checks/cart.py', 'assert result == 4') + updated = self.run_cli() + self.assertNotEqual(initial['inputs']['scenarios'], updated['inputs']['scenarios']) + self.put(self.skill, 'Revised instructions') + self.assertNotEqual(updated['inputs']['skill'], self.run_cli()['inputs']['skill']) + self.data['extra_app_specific_field'] = {'allowed': True} + self.save() + self.assertNotEqual(initial['inputs']['map'], self.run_cli()['inputs']['map']) + + def test_installed_helper_inspects_project_without_creating_verifier_coverage(self): + process = subprocess.run(['bash', str(KIT / 'install.sh'), '--no-hooks', str(self.root)], + capture_output=True, text=True) + self.assertEqual(process.returncode, 0, process.stderr) + installed = self.root / '.claude/bin/super-board-project-verifier.py' + process = subprocess.run([sys.executable, str(installed), 'inspect', '--root', str(self.root), + '--map', self.map], capture_output=True, text=True) + self.assertEqual(process.returncode, 0, process.stderr or process.stdout) + self.assertEqual(json.loads(process.stdout)['features'], ['cart.total']) + starter = self.root / '.claude/skills/super-qa/templates/project-verifier/feature-map.json' + self.assertEqual(json.loads(starter.read_text())['features'], []) + self.assertEqual((self.root / self.skill).read_text(), 'Real app instructions') + + def test_root_symlink_loop_returns_setup_error_without_traceback(self): + loop = Path(self.temp.name) / 'loop' + loop.symlink_to(loop, target_is_directory=True) + result = self.run_cli(extra=['--root', str(loop)], expected=9) + self.assertFalse(result['valid']) + + +if __name__ == '__main__': + unittest.main()