From 1fa3e3bfbe075d1fdd09047f3a127ef3f22b8b5f Mon Sep 17 00:00:00 2001 From: Daniel Petty Date: Sat, 7 Nov 2026 22:18:45 +0000 Subject: [PATCH] feat(roadmap): add a set count to every-member plan claims (#3304) Add a rule to the stage prompt and proposal config that requires a count for claims about every member of a set. Closes #3304 --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/roadmap-stage/spec.md | 0 .../tasks.md | 6 +-- openspec/config.yaml | 2 + openspec/specs/roadmap-stage/spec.md | 34 ++++++++++++++++ .../ai/tasks/roadmap_stage_change.md | 1 + .../stage/test_roadmap_stage_prompt_rules.py | 39 +++++++++++++++++++ 9 files changed, 79 insertions(+), 3 deletions(-) rename openspec/changes/{3304-add-a-set-count-to-every => archive/2026-11-07-3304-add-a-set-count-to-every}/.openspec.yaml (100%) rename openspec/changes/{3304-add-a-set-count-to-every => archive/2026-11-07-3304-add-a-set-count-to-every}/design.md (100%) rename openspec/changes/{3304-add-a-set-count-to-every => archive/2026-11-07-3304-add-a-set-count-to-every}/proposal.md (100%) rename openspec/changes/{3304-add-a-set-count-to-every => archive/2026-11-07-3304-add-a-set-count-to-every}/specs/roadmap-stage/spec.md (100%) rename openspec/changes/{3304-add-a-set-count-to-every => archive/2026-11-07-3304-add-a-set-count-to-every}/tasks.md (94%) diff --git a/openspec/changes/3304-add-a-set-count-to-every/.openspec.yaml b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/.openspec.yaml similarity index 100% rename from openspec/changes/3304-add-a-set-count-to-every/.openspec.yaml rename to openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/.openspec.yaml diff --git a/openspec/changes/3304-add-a-set-count-to-every/design.md b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/design.md similarity index 100% rename from openspec/changes/3304-add-a-set-count-to-every/design.md rename to openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/design.md diff --git a/openspec/changes/3304-add-a-set-count-to-every/proposal.md b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/proposal.md similarity index 100% rename from openspec/changes/3304-add-a-set-count-to-every/proposal.md rename to openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/proposal.md diff --git a/openspec/changes/3304-add-a-set-count-to-every/specs/roadmap-stage/spec.md b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/specs/roadmap-stage/spec.md similarity index 100% rename from openspec/changes/3304-add-a-set-count-to-every/specs/roadmap-stage/spec.md rename to openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/specs/roadmap-stage/spec.md diff --git a/openspec/changes/3304-add-a-set-count-to-every/tasks.md b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/tasks.md similarity index 94% rename from openspec/changes/3304-add-a-set-count-to-every/tasks.md rename to openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/tasks.md index 3380e07f1..c2e0bc65a 100644 --- a/openspec/changes/3304-add-a-set-count-to-every/tasks.md +++ b/openspec/changes/archive/2026-11-07-3304-add-a-set-count-to-every/tasks.md @@ -4,16 +4,16 @@ Reads: `src/devops_cli/ai/tasks/roadmap_stage_change.md`, `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`, `openspec/changes/3236-read-only-tools-take-paths-that/design.md`. Changes: `src/devops_cli/ai/tasks/roadmap_stage_change.md`, `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`. -- [ ] 1.1 Add `test_the_prompt_asks_for_a_count_of_a_claimed_set` to `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`. It asserts that the prompt holds `gives a count from a tool of the plan run or from a probe line`, `the count is unverified`, `census command`, `verify` of the first group, `removed, restricted, or allowlisted with a reason`, `a member with no decision`, `narrows the claim`, `follow-up issue`, `the set that the issue names` and `not the set of its examples`. Add `test_the_plan_check_refuses_no_word_of_a_requirement_text`, which builds a plan whose requirement text holds `every` with `make_stage_proposal` and asserts that `check_proposal` returns no problem that names the word. The prompt test fails on the base, and the check test passes +- [x] 1.1 Add `test_the_prompt_asks_for_a_count_of_a_claimed_set` to `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`. It asserts that the prompt holds `gives a count from a tool of the plan run or from a probe line`, `the count is unverified`, `census command`, `verify` of the first group, `removed, restricted, or allowlisted with a reason`, `a member with no decision`, `narrows the claim`, `follow-up issue`, `the set that the issue names` and `not the set of its examples`. Add `test_the_plan_check_refuses_no_word_of_a_requirement_text`, which builds a plan whose requirement text holds `every` with `make_stage_proposal` and asserts that `check_proposal` returns no problem that names the word. The prompt test fails on the base, and the check test passes - Verification: `$ uv run devops ci test tests/roadmap/stage/test_roadmap_stage_prompt_rules.py` fails on the base -- [ ] 1.2 Add one bullet to the rules of `src/devops_cli/ai/tasks/roadmap_stage_change.md`, after the feasibility bullet: a claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, the plan says that the count is unverified, makes the census command the `verify` of the first group, and stops when the result differs from the plan; the plan decides each member as removed, restricted, or allowlisted with a reason, and a member with no decision fails the verification of the group that counts the set; a count above zero for a claim of no members narrows the claim or names the follow-up issue for the rest; the set is the set that the issue names, and not the set of its examples. Then run `uv run devops roadmap eval stage --replay` and state the score of the stage model before and after the change +- [x] 1.2 Add one bullet to the rules of `src/devops_cli/ai/tasks/roadmap_stage_change.md`, after the feasibility bullet: a claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, the plan says that the count is unverified, makes the census command the `verify` of the first group, and stops when the result differs from the plan; the plan decides each member as removed, restricted, or allowlisted with a reason, and a member with no decision fails the verification of the group that counts the set; a count above zero for a claim of no members narrows the claim or names the follow-up issue for the rest; the set is the set that the issue names, and not the set of its examples. Then run `uv run devops roadmap eval stage --replay` and state the score of the stage model before and after the change - Verification: `$ uv run devops ci test tests/roadmap/stage/test_roadmap_stage_prompt_rules.py` passes ## 2. State the same rule in the proposal config and pin the shared phrase Reads: `openspec/config.yaml`, `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`, `tests/commands/test_openspec_scaffold.py`. Changes: `openspec/config.yaml`, `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py`. -- [ ] 2.1 Add one item to `rules.proposal` of `openspec/config.yaml`, written as a folded scalar (`>-`): a claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, say that the count is unverified, make the census command the `verify` of the first group, and decide each member as removed, restricted, or allowlisted with a reason; a count above zero for a claim of no members narrows the claim or names the follow-up issue; the set is the set that the issue names, and not the set of its examples. Add the pair `("openspec/config.yaml", "gives a count from a tool of the plan run or from a probe line")` to `_SHARED` of `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py` +- [x] 2.1 Add one item to `rules.proposal` of `openspec/config.yaml`, written as a folded scalar (`>-`): a claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, say that the count is unverified, make the census command the `verify` of the first group, and decide each member as removed, restricted, or allowlisted with a reason; a count above zero for a claim of no members narrows the claim or names the follow-up issue; the set is the set that the issue names, and not the set of its examples. Add the pair `("openspec/config.yaml", "gives a count from a tool of the plan run or from a probe line")` to `_SHARED` of `tests/roadmap/stage/test_roadmap_stage_prompt_rules.py` - Verification: `$ uv run devops ci test tests/roadmap/stage/test_roadmap_stage_prompt_rules.py` passes ## Workflow follow-up diff --git a/openspec/config.yaml b/openspec/config.yaml index b52100e21..3db3f39be 100644 --- a/openspec/config.yaml +++ b/openspec/config.yaml @@ -14,6 +14,8 @@ rules: State whether the change fits one pull request. - >- A change that alters no behavior a consumer observes (tests, CI workflows, documentation, a refactor that keeps behavior) has no spec delta and sets skip_specs: true in its .openspec.yaml. A devops command, option or message, a manifest and a service's behavior are observable: they carry a delta. Say why in What Changes. + - >- + A claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, say that the count is unverified, make the census command the `verify` of the first group, and decide each member as removed, restricted, or allowlisted with a reason; a count above zero for a claim of no members narrows the claim or names the follow-up issue; the set is the set that the issue names, and not the set of its examples. specs: - >- Run openspec list --specs first and reuse a capability. The delta that creates a capability starts with a "## Purpose" section of at least 50 characters, so archive never writes TBD. A capability that another active change creates is extended with ADDED requirements and no Purpose of its own. diff --git a/openspec/specs/roadmap-stage/spec.md b/openspec/specs/roadmap-stage/spec.md index a37393b2a..378caa7ee 100644 --- a/openspec/specs/roadmap-stage/spec.md +++ b/openspec/specs/roadmap-stage/spec.md @@ -2871,3 +2871,37 @@ The stage prompt MUST name the `security` key, the `## Security` section and the #### Scenario: The prompt and the reply agree - **WHEN** a test reads `roadmap_stage_change.md` and the fields of `StageProposal` - **THEN** the prompt names `security` and the trigger, and the keys of the reply are the fields of the proposal + +### Requirement: The stage prompt asks for a count of a claimed set +The stage prompt SHALL ask the plan to give a count of the set that a claim about every member covers, from the `references` tool or from a probe line, and it SHALL ask the proposal rule of `openspec/config.yaml` to hold the same phrase. + +#### Scenario: A claim about a set +- **WHEN** the test reads `src/devops_cli/ai/tasks/roadmap_stage_change.md` +- **THEN** the prompt holds `gives a count from a tool of the plan run or from a probe line` + +#### Scenario: No tool counts the set +- **WHEN** the test reads the stage prompt +- **THEN** the prompt holds `the count is unverified`, `census command` and `verify` of the first group + +#### Scenario: A member has no decision +- **WHEN** the test reads the stage prompt +- **THEN** the prompt holds `removed, restricted, or allowlisted with a reason` and `a member with no decision` + +#### Scenario: The count of a claim of no members is above zero +- **WHEN** the test reads the stage prompt +- **THEN** the prompt holds `narrows the claim` and `follow-up issue` + +#### Scenario: The issue gives examples of a larger set +- **WHEN** the test reads the stage prompt +- **THEN** the prompt holds `the set that the issue names` and `not the set of its examples` + +#### Scenario: The prompt and the config keep one phrase +- **WHEN** the test reads `openspec/config.yaml` and the stage prompt +- **THEN** both files hold `gives a count from a tool of the plan run or from a probe line` + +### Requirement: The plan check refuses no word of a requirement text +The plan check SHALL NOT refuse a plan for the word `every`, `all` or `no` in a requirement text, so a claim about a set is judged by its count and never by a word. + +#### Scenario: A requirement text holds every +- **WHEN** the test builds a plan whose requirement text holds `every` and whose paths exist at the staged commit +- **THEN** `check_proposal` returns no problem that names the word diff --git a/src/devops_cli/ai/tasks/roadmap_stage_change.md b/src/devops_cli/ai/tasks/roadmap_stage_change.md index d9c23b0ea..0bdd15064 100644 --- a/src/devops_cli/ai/tasks/roadmap_stage_change.md +++ b/src/devops_cli/ai/tasks/roadmap_stage_change.md @@ -8,6 +8,7 @@ The rules of the project: - Unless a section on tools follows these rules, you cannot run a command. Write only what the inputs and your tool answers show, and cite the repository paths that show it. In `feasibility`, say how the premise of the issue was confirmed against the inputs: quote the lines of `probes` that confirm it, word for word. Then name the first checks that the implementation runs against the real system, each with its command. If the inputs show that the issue cannot be built as written, say so there, and plan no code that compensates for it. - Cite the staged value of each constant that the plan relies on, with the constant name and its value, as `feasibility` quotes the probe lines, and give no new name to the bound of a constant that the staged tree defines. +- A claim about every member of a set, or a new guard over a set, gives a count from a tool of the plan run or from a probe line; when no tool can count the set, the plan says that the count is unverified, makes the census command the `verify` of the first group, and stops when the result differs from the plan; the plan decides each member as removed, restricted, or allowlisted with a reason, and a member with no decision fails the verification of the group that counts the set; a count above zero for a claim of no members narrows the claim or names the follow-up issue for the rest; the set is the set that the issue names, and not the set of its examples. - In `feasibility`, give one line for each function that the plan calls whose arguments need a checkout, a client or a network. Name that need in the line as a precondition of the call: a checkout of the repository, a client of a service, or a network. - When `rejected_plan` is in the input, state in `feasibility` how the new plan answers that reason. Do not plan again the premise that the reason shows to be false. If the issue still cannot be built as written, say so in `feasibility` and plan no code that compensates for it. - The reply has no key to report a false premise. When `probes` or the other inputs show that the issue cannot be built as written, for example a cited path or identifier that a commit deleted, renamed or removed, say so in `feasibility` with a quote of the lines that show it, and plan no code that compensates for it. A path or identifier that no commit ever held can be a thing that the issue asks to create: it is no false premise. diff --git a/tests/roadmap/stage/test_roadmap_stage_prompt_rules.py b/tests/roadmap/stage/test_roadmap_stage_prompt_rules.py index 8fa8a2af2..14b7eb734 100644 --- a/tests/roadmap/stage/test_roadmap_stage_prompt_rules.py +++ b/tests/roadmap/stage/test_roadmap_stage_prompt_rules.py @@ -453,6 +453,10 @@ def test_the_stage_prompt_asks_for_the_precondition_of_each_called_function() -> "openspec/config.yaml", "`! git grep -n src/ tests/` (a command that exits 0 when it finds nothing)", ), + ( + "openspec/config.yaml", + "gives a count from a tool of the plan run or from a probe line", + ), ( "docs/TESTING.md", "`! git grep -n src/ tests/` (a command that exits 0 when it finds nothing)", @@ -556,3 +560,38 @@ def test_the_stage_prompt_requires_citation_of_staged_constant_values() -> None: "and its value, as `feasibility` quotes the probe lines, and give no new name to the " "bound of a constant that the staged tree defines." ) in prompt + + +# ── Count of a claimed set (#3304) ──────────────────────────────────────────── + + +def test_the_prompt_asks_for_a_count_of_a_claimed_set() -> None: + prompt = load_task_prompt("roadmap_stage_change.md") + + assert "gives a count from a tool of the plan run or from a probe line" in prompt + assert "the count is unverified" in prompt + assert "census command" in prompt + assert "`verify` of the first group" in prompt + assert "removed, restricted, or allowlisted with a reason" in prompt + assert "a member with no decision" in prompt + assert "narrows the claim" in prompt + assert "follow-up issue" in prompt + assert "the set that the issue names" in prompt + assert "not the set of its examples" in prompt + + +def test_the_plan_check_refuses_no_word_of_a_requirement_text() -> None: + proposal = make_stage_proposal() + req = ( + proposal.specs[0] + .added[0] + .model_copy(update={"text": "Every caller SHALL pass a valid configuration."}) + ) + spec = proposal.specs[0].model_copy(update={"added": [req]}) + proposal_with_every = proposal.model_copy(update={"specs": [spec]}) + problems = check_proposal( + proposal_with_every, + known=[s.capability for s in proposal.specs], + tree=StagedTree(proposal.groups[0].changes), + ) + assert not any("every" in p.lower() for p in problems)