From b57b4e3cabdd7c57b47390607ff7eae84f2198a4 Mon Sep 17 00:00:00 2001 From: evgen Date: Thu, 10 Sep 2026 08:11:57 -0400 Subject: [PATCH 1/3] fix(compliance): drop comply_test_controller from canonical_format_validate_input required_tools (OR-gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-storyboard gate (runner.js) admits a storyboard when any listed tool is present, while the suite selector (comply.js) requires all of them. With comply_test_controller in the list, every agent exposing the universal test controller was selected for this storyboard and then failed all 17 steps on the missing validate_input — on the hosted grader, 17 spurious failures on each of two registered agents (#7404). Same shape as #6774. Refs #7404 (bug 1 only). --- .changeset/canonical-format-validate-input-required-tools.md | 5 +++++ .../source/universal/canonical-format-validate-input.yaml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .changeset/canonical-format-validate-input-required-tools.md diff --git a/.changeset/canonical-format-validate-input-required-tools.md b/.changeset/canonical-format-validate-input-required-tools.md new file mode 100644 index 00000000000..766804d7109 --- /dev/null +++ b/.changeset/canonical-format-validate-input-required-tools.md @@ -0,0 +1,5 @@ +--- +"adcontextprotocol": patch +--- + +`canonical_format_validate_input` no longer lists `comply_test_controller` in `required_tools`. The storyboard runner's per-storyboard gate admits a storyboard when any listed tool is present, so any agent exposing the (universal) test controller was selected and then failed all 17 steps on the missing `validate_input`. Agents without `validate_input` now receive a coverage-gap skip; agents implementing it run unchanged. Same shape as #6774 (fixes #7404, bug 1). diff --git a/static/compliance/source/universal/canonical-format-validate-input.yaml b/static/compliance/source/universal/canonical-format-validate-input.yaml index ab410bf422a..1d993a8c9e7 100644 --- a/static/compliance/source/universal/canonical-format-validate-input.yaml +++ b/static/compliance/source/universal/canonical-format-validate-input.yaml @@ -7,7 +7,6 @@ track: creative introduced_in: "3.1" required_tools: - validate_input - - comply_test_controller requires: - controller From cf5f6f3f733db04497b5bbe0d806ea53e9827a92 Mon Sep 17 00:00:00 2001 From: evgen Date: Thu, 10 Sep 2026 16:02:16 -0400 Subject: [PATCH 2/3] ci: lower the governance and brand passing-step floors by one for the storyboard no longer selected With comply_test_controller off canonical_format_validate_input's required_tools, the /governance and /brand training tenants (which expose the controller but not validate_input) no longer select that storyboard, and the one step it used to contribute to their passing-step count goes away: governance 161 -> 160, brand 116 -> 115. Clean-storyboard floors are untouched (56/47 and 55/45 observed). Same tuples mirrored in run-storyboards-matrix.sh. --- .github/workflows/training-agent-storyboards.yml | 4 ++-- scripts/run-storyboards-matrix.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/training-agent-storyboards.yml b/.github/workflows/training-agent-storyboards.yml index 7e7cbb76c68..d8a5462d7ec 100644 --- a/.github/workflows/training-agent-storyboards.yml +++ b/.github/workflows/training-agent-storyboards.yml @@ -80,7 +80,7 @@ jobs: - surface: current tenant: governance min_clean_storyboards: 47 - min_passing_steps: 161 + min_passing_steps: 160 - surface: current tenant: creative min_clean_storyboards: 49 @@ -92,7 +92,7 @@ jobs: - surface: current tenant: brand min_clean_storyboards: 45 - min_passing_steps: 116 + min_passing_steps: 115 - surface: current tenant: si min_clean_storyboards: 42 diff --git a/scripts/run-storyboards-matrix.sh b/scripts/run-storyboards-matrix.sh index a826a103d91..641fe1f8548 100755 --- a/scripts/run-storyboards-matrix.sh +++ b/scripts/run-storyboards-matrix.sh @@ -269,10 +269,10 @@ else # clean-result-row and passing-step regression floors. "signals:45:80" "sales:133:632" - "governance:47:161" + "governance:47:160" "creative:49:209" "creative-builder:50:184" - "brand:45:116" + "brand:45:115" "si:42:50" ) fi From 7c2b8a1aa1a0374e4e82699bc0963ca30e94272b Mon Sep 17 00:00:00 2001 From: evgen Date: Thu, 10 Sep 2026 16:32:16 -0400 Subject: [PATCH 3/3] test: mirror the lowered governance/brand passing-step floors in the ratchet guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run-storyboards-sharding pins the same (tenant, min_clean, min_passed) tuples a third time, so lowering a floor has to be stated there too — which is the point of that guard. Current surface only: governance 161 -> 160, brand 116 -> 115. The 3.0-compat block is untouched because canonical_format_validate_input is introduced_in 3.1 and is never selected in that bundle. --- tests/run-storyboards-sharding.test.cjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/run-storyboards-sharding.test.cjs b/tests/run-storyboards-sharding.test.cjs index c95a719ddf9..033ffbc6d98 100644 --- a/tests/run-storyboards-sharding.test.cjs +++ b/tests/run-storyboards-sharding.test.cjs @@ -300,13 +300,19 @@ test('creative-builder uses bounded isolated children in local and CI matrices', test('current training-agent floors are ratcheted and mirrored by local and CI runners', () => { const workflow = fs.readFileSync(STORYBOARD_WORKFLOW, 'utf8'); const matrixRunner = fs.readFileSync(MATRIX_RUNNER, 'utf8'); + // governance and brand dropped one passing step each when + // canonical_format_validate_input stopped listing comply_test_controller in + // required_tools: both tenants expose the controller but not validate_input, + // so the storyboard is no longer selected for them and the single step it + // contributed is gone. Deliberate de-ratchet, not a regression — the + // clean-storyboard floors are untouched. const baselines = [ ['signals', 45, 80], ['sales', 133, 632], - ['governance', 47, 161], + ['governance', 47, 160], ['creative', 49, 209], ['creative-builder', 50, 184], - ['brand', 45, 116], + ['brand', 45, 115], ['si', 42, 50], ];