fix: restore extracted benchmark campaign execution and evidence paths

.github/workflows/ci.yml

@@ -4,7 +4,8 @@
 #   python-syntax      — byte-compiles every Python tree (catches syntax/import-
 #                        time breakage cheaply, no third-party deps needed).
 #   root-pytest        — the suite-level tests/ directory (registry, scoring,
-#                        normalization, acceptance gate). Needs only pytest;
+#                        normalization, acceptance gate) plus source-bound
+#                        orchestrator resume/cohort contracts. Needs only pytest;
 #                        dataset-dependent cases skip gracefully when their
 #                        artifacts are absent.
 #   bun-workspace      — root bun install resolving @elizaos/* from the eliza/
@@ -71,7 +72,11 @@ jobs:
       - name: Run suite-level tests
         env:
           PYTHONPATH: ${{ github.workspace }}
-        run: python -m pytest benchmarks/tests -q
+        run: >-
+          python -m pytest benchmarks/tests
+          benchmarks/suites/orchestrator/tests/test_execution_identity.py
+          benchmarks/suites/orchestrator/tests/test_cohort.py
+          benchmarks/suites/orchestrator/tests/test_cohort_provider_forwarder.py -q
 
   bun-workspace:
     name: bun workspace (plugin + TS suites)
@@ -85,6 +90,8 @@ jobs:
           bun run --cwd plugin-benchmarks test
           bun run --cwd plugin-benchmarks typecheck
           bun run --cwd plugin-benchmarks build
+      - name: Manifest exporter CLI smoke
+        run: bun run lifeops-bench:manifest --help
       - name: framework typecheck
         run: bun run --cwd framework/typescript typecheck
       - name: suite tests (keyless vitest/bun-test lanes)

.github/workflows/lifeops-bench-python.yml

@@ -197,14 +197,12 @@ jobs:
         run: |
           set -e
           mkdir -p lifeops_bench_results
-          # The `wrong` reference oracle runs entirely in-process and
-          # exercises the runner / scorer / executor pipeline without any
-          # LLM. PerfectAgent's CLI entry point currently requires a
-          # per-scenario constructor (see __main__._build_agent_fn) and is
-          # covered instead by the conformance test in unit-tests above.
+          # A deterministic negative control exercises five static domains.
+          # Its results are explicitly non-publishable conformance evidence.
           uv run python -m eliza_lifeops_bench \
             --agent wrong \
-            --mode static \
+            --suite smoke \
+            --offline-conformance \
             --seeds 1 \
             --concurrency 4 \
             --max-cost-usd 0.01 \

registry/scores.py

@@ -653,7 +653,9 @@ def _attest_installed_orchestrator_lifecycle_sources() -> None:
     ):
         raise ValueError("orchestrator_lifecycle: loaded TASKS tool contract drifted")
 
-    lifecycle_dir = Path(__file__).resolve().parents[1] / "orchestrator_lifecycle"
+    lifecycle_dir = (
+        Path(__file__).resolve().parents[1] / "suites" / "orchestrator_lifecycle"
+    )
     contract_path = lifecycle_dir / "contract.py"
     tasks_tool_path = lifecycle_dir / "tasks-tool.json"
     try:
@@ -753,7 +755,10 @@ def _score_from_orchestrator_lifecycle_json(data: JSONValue) -> ScoreExtraction:
             )
 
     scenario_dir = (
-        Path(__file__).resolve().parents[1] / "orchestrator_lifecycle" / "scenarios"
+        Path(__file__).resolve().parents[1]
+        / "suites"
+        / "orchestrator_lifecycle"
+        / "scenarios"
     )
     pinned_scenarios = LifecycleDataset(str(scenario_dir)).load()
     if (

suites/lifeops-bench/tests/test_manifest_export.py

@@ -8,25 +8,14 @@
 from pathlib import Path
 
 PACKAGE_ROOT = Path(__file__).resolve().parents[1]
-REPO_ROOT = PACKAGE_ROOT.parents[1]
 MANIFEST_PATH = PACKAGE_ROOT / "manifests" / "actions.manifest.json"
 SUMMARY_PATH = PACKAGE_ROOT / "manifests" / "actions.summary.md"
-ROOT_PACKAGE_JSON = REPO_ROOT / "package.json"
 
 
 def _manifest() -> dict[str, object]:
     return json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
 
 
-def test_root_manifest_regeneration_script_is_registered() -> None:
-    root_package = json.loads(ROOT_PACKAGE_JSON.read_text(encoding="utf-8"))
-    command = root_package["scripts"]["lifeops-bench:manifest"]
-    assert "suites/lifeops-bench/scripts/export-action-manifest.ts" in command
-    assert "--conditions=eliza-source" in command
-    assert "--conditions=development" in command
-    assert "--import tsx" in command
-
-
 def test_manifest_has_in_tree_generator_metadata() -> None:
     manifest = _manifest()
     assert manifest["schemaVersion"] == 1

suites/orchestrator/README.md

@@ -23,6 +23,13 @@ python) for consistent dependency versions across benchmark subprocesses.
 - Viewer dataset: `benchmark_results/viewer_data.json`
 - Static viewer UI: `viewer/index.html`
 
+All producers, cohort resume, and the viewer use the repo-root result store.
+If an older checkout wrote data under `suites/benchmark_results`, new runs fail
+before dispatch. Explicitly archive that legacy directory or migrate the complete
+store to `benchmark_results` before resuming. Never combine two non-empty stores
+by copying individual database or result files; their run identities and artifacts
+must stay together. The runner does not move or delete legacy artifacts.
+
 ## List integrated benchmarks
 
 ```bash
@@ -66,7 +73,7 @@ python3 -m benchmarks.orchestrator run \
 
 Idempotent behavior:
 
-- Existing successful signatures are skipped automatically.
+- Existing successful signatures are skipped only when the request, repository revisions, runtime and benchmark/harness source, corpus inputs, and root lockfiles match. Changed inputs require a new run. Subscription checkpoint namespaces use the same source-sensitive execution contract; old namespaces are not reused. Cross-harness comparison grouping remains separate.
 - `--rerun-failed` reruns only signatures whose latest run failed.
 - `--force` always creates a fresh run.
 
@@ -87,11 +94,11 @@ adapter discovery before spending quota. Each phase runs Eliza, Hermes, and
 OpenClaw together; phases and benchmarks advance serially, and the default
 stops on the first failed cohort.
 
-The current manifest accounts for all 59 discovered adapters exactly once:
+The current manifest accounts for all 58 discovered adapters exactly once:
 25 automatic cohort entries (81 full phases, including all 55 ClawBench
-scenarios and all three Mind2Web test splits), 21 externally provisioned manual
-entries, 2 unsupported three-harness paths, and 11 non-agent/replay entries. It
-also records 15 direct
+scenarios and all three Mind2Web test splits), 17 externally provisioned manual
+entries, 5 unsupported three-harness paths, and 11 non-agent/replay entries. It
+also records 14 direct
 or supporting workloads that are outside adapter discovery. Unsupported,
 non-agent, infrastructure, and unintegrated work is reported, never converted
 into misleading comparison rows.
@@ -100,8 +107,7 @@ Inspect the machine-checked workload and subscription-call ledger without
 launching a benchmark:
 
 ```bash
-PYTHONPATH=packages python3 \
-  -m benchmarks.orchestrator.campaign_ledger
+python3 -m benchmarks.orchestrator.campaign_ledger
 ```
 
 The report separates authored tasks, expanded scenarios, and scored result
@@ -110,10 +116,10 @@ data-dependent rankers and native agent loops are never presented as fixed
 model-call totals. The 25 automatic cohorts currently total 33,981 base tasks,
 213,801 expanded scenarios, and 308,639 result cells per harness (101,943 /
 641,403 / 925,917 respectively across Eliza, Hermes, and OpenClaw).
-Across all 46 intended comparative adapters, the exact-known subtotal is
-67,373 base tasks, 577,403 expanded scenarios, and 672,241 result cells per
-harness (202,119 / 1,732,209 / 2,016,723 across all three harnesses). These are
-known-cardinality subtotals, not campaign totals: four externally selected
+Across all 42 intended comparative adapters, the exact-known subtotal is
+50,083 base tasks, 387,213 expanded scenarios, and 482,051 result cells per
+harness (150,249 / 1,161,639 / 1,446,153 across all three harnesses). These are
+known-cardinality subtotals, not campaign totals: three externally selected
 corpora still have unknown cardinality, and their IDs are emitted in every
 report.
 

suites/orchestrator/artifact_guard.py

@@ -164,7 +164,7 @@ def build_artifact_guard_report(
     ``run_git`` is injectable so the classifier can be tested without a real
     repository; by default it shells out to ``git ls-files`` at the repo root.
     """
-    repo_root = workspace_root.parent  # workspace_root is ``.../packages``
+    repo_root = workspace_root
     runner = run_git or _default_git_runner(repo_root)
     tracked = [line for line in runner(["ls-files"]).splitlines() if line.strip()]
     offending = find_committed_generated_artifacts(tracked)

suites/orchestrator/campaign_ledger.py

@@ -1057,15 +1057,6 @@ def _direct(
         _M.NOT_APPLICABLE,
         basis="Discovered plugin bundles determine deterministic size-gate cells at execution time.",
     ),
-    _direct(
-        "voice_pipeline",
-        _C.NON_AGENT,
-        None,
-        None,
-        None,
-        _M.NOT_APPLICABLE,
-        basis="Native voice backends available to the CI matrix determine cells; no comparison harness is selected.",
-    ),
     _direct(
         "voice_emotion",
         _C.NON_AGENT,

suites/orchestrator/cli.py

@@ -149,7 +149,7 @@ def _cmd_validate_matrix(args: argparse.Namespace) -> int:
     workspace_root = _workspace_root_from_here()
     _apply_model_profile(args, workspace_root)
     report = build_cross_matrix_report(
-        workspace_root.parent,
+        workspace_root,
         provider=args.provider,
         model=args.model,
         extra_config=_parse_json_arg(args.extra),
@@ -163,7 +163,7 @@ def _cmd_validate_matrix(args: argparse.Namespace) -> int:
 
 def _cmd_inventory(args: argparse.Namespace) -> int:
     workspace_root = _workspace_root_from_here()
-    report = build_inventory_report(workspace_root.parent)
+    report = build_inventory_report(workspace_root)
     if args.format == "json":
         print(inventory_report_to_json(report))
     else:
@@ -550,7 +550,7 @@ def _emit(step: str, code: int) -> None:
     # baseline (they predate this wrapper) and are NOT drift to block a review on
     # — so preflight fails only when a *registered* benchmark has no adapter.
     print("[review] step 1/5: preflight (inventory)")
-    inventory = build_inventory_report(workspace_root.parent)
+    inventory = build_inventory_report(workspace_root)
     print(inventory_report_to_markdown(inventory))
     if inventory.registry_entries_without_adapters:
         _emit("preflight (inventory)", 2)

suites/orchestrator/code_agent_matrix.py

@@ -578,7 +578,12 @@ def preflight_matrix(
         )
 
     opencode_bin = _opencode_bin(root, env)
-    if any(cell.adapter == "opencode" for cell in cells) and not opencode_bin:
+    # Explicit mock commands use the suite fixture agent, not the external CLI.
+    # A mixed cohort still requires OpenCode for every live cell.
+    if any(
+        cell.adapter == "opencode" and "--mock" not in cell.command
+        for cell in cells
+    ) and not opencode_bin:
         issues.append(
             {
                 "severity": "error",