diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f03b490..eb39f2cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,8 @@ # python-syntax — byte-compiles every Python tree (catches syntax/import- # time breakage cheaply, no third-party deps needed). # root-pytest — the suite-level tests/ directory (registry, scoring, -# normalization, acceptance gate). Needs only pytest; +# normalization, acceptance gate) plus source-bound +# orchestrator resume/cohort contracts. Needs only pytest; # dataset-dependent cases skip gracefully when their # artifacts are absent. # bun-workspace — root bun install resolving @elizaos/* from the eliza/ @@ -71,7 +72,11 @@ jobs: - name: Run suite-level tests env: PYTHONPATH: ${{ github.workspace }} - run: python -m pytest benchmarks/tests -q + run: >- + python -m pytest benchmarks/tests + benchmarks/suites/orchestrator/tests/test_execution_identity.py + benchmarks/suites/orchestrator/tests/test_cohort.py + benchmarks/suites/orchestrator/tests/test_cohort_provider_forwarder.py -q bun-workspace: name: bun workspace (plugin + TS suites) @@ -85,6 +90,8 @@ jobs: bun run --cwd plugin-benchmarks test bun run --cwd plugin-benchmarks typecheck bun run --cwd plugin-benchmarks build + - name: Manifest exporter CLI smoke + run: bun run lifeops-bench:manifest --help - name: framework typecheck run: bun run --cwd framework/typescript typecheck - name: suite tests (keyless vitest/bun-test lanes) diff --git a/.github/workflows/lifeops-bench-python.yml b/.github/workflows/lifeops-bench-python.yml index cc1681fd..440376b1 100644 --- a/.github/workflows/lifeops-bench-python.yml +++ b/.github/workflows/lifeops-bench-python.yml @@ -197,14 +197,12 @@ jobs: run: | set -e mkdir -p lifeops_bench_results - # The `wrong` reference oracle runs entirely in-process and - # exercises the runner / scorer / executor pipeline without any - # LLM. PerfectAgent's CLI entry point currently requires a - # per-scenario constructor (see __main__._build_agent_fn) and is - # covered instead by the conformance test in unit-tests above. + # A deterministic negative control exercises five static domains. + # Its results are explicitly non-publishable conformance evidence. uv run python -m eliza_lifeops_bench \ --agent wrong \ - --mode static \ + --suite smoke \ + --offline-conformance \ --seeds 1 \ --concurrency 4 \ --max-cost-usd 0.01 \ diff --git a/registry/scores.py b/registry/scores.py index 19351308..696de3bf 100644 --- a/registry/scores.py +++ b/registry/scores.py @@ -653,7 +653,9 @@ def _attest_installed_orchestrator_lifecycle_sources() -> None: ): raise ValueError("orchestrator_lifecycle: loaded TASKS tool contract drifted") - lifecycle_dir = Path(__file__).resolve().parents[1] / "orchestrator_lifecycle" + lifecycle_dir = ( + Path(__file__).resolve().parents[1] / "suites" / "orchestrator_lifecycle" + ) contract_path = lifecycle_dir / "contract.py" tasks_tool_path = lifecycle_dir / "tasks-tool.json" try: @@ -753,7 +755,10 @@ def _score_from_orchestrator_lifecycle_json(data: JSONValue) -> ScoreExtraction: ) scenario_dir = ( - Path(__file__).resolve().parents[1] / "orchestrator_lifecycle" / "scenarios" + Path(__file__).resolve().parents[1] + / "suites" + / "orchestrator_lifecycle" + / "scenarios" ) pinned_scenarios = LifecycleDataset(str(scenario_dir)).load() if ( diff --git a/suites/lifeops-bench/tests/test_manifest_export.py b/suites/lifeops-bench/tests/test_manifest_export.py index eabcf57e..31805152 100644 --- a/suites/lifeops-bench/tests/test_manifest_export.py +++ b/suites/lifeops-bench/tests/test_manifest_export.py @@ -8,25 +8,14 @@ from pathlib import Path PACKAGE_ROOT = Path(__file__).resolve().parents[1] -REPO_ROOT = PACKAGE_ROOT.parents[1] MANIFEST_PATH = PACKAGE_ROOT / "manifests" / "actions.manifest.json" SUMMARY_PATH = PACKAGE_ROOT / "manifests" / "actions.summary.md" -ROOT_PACKAGE_JSON = REPO_ROOT / "package.json" def _manifest() -> dict[str, object]: return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) -def test_root_manifest_regeneration_script_is_registered() -> None: - root_package = json.loads(ROOT_PACKAGE_JSON.read_text(encoding="utf-8")) - command = root_package["scripts"]["lifeops-bench:manifest"] - assert "suites/lifeops-bench/scripts/export-action-manifest.ts" in command - assert "--conditions=eliza-source" in command - assert "--conditions=development" in command - assert "--import tsx" in command - - def test_manifest_has_in_tree_generator_metadata() -> None: manifest = _manifest() assert manifest["schemaVersion"] == 1 diff --git a/suites/orchestrator/README.md b/suites/orchestrator/README.md index fa7177b0..7af94543 100644 --- a/suites/orchestrator/README.md +++ b/suites/orchestrator/README.md @@ -23,6 +23,13 @@ python) for consistent dependency versions across benchmark subprocesses. - Viewer dataset: `benchmark_results/viewer_data.json` - Static viewer UI: `viewer/index.html` +All producers, cohort resume, and the viewer use the repo-root result store. +If an older checkout wrote data under `suites/benchmark_results`, new runs fail +before dispatch. Explicitly archive that legacy directory or migrate the complete +store to `benchmark_results` before resuming. Never combine two non-empty stores +by copying individual database or result files; their run identities and artifacts +must stay together. The runner does not move or delete legacy artifacts. + ## List integrated benchmarks ```bash @@ -66,7 +73,7 @@ python3 -m benchmarks.orchestrator run \ Idempotent behavior: -- Existing successful signatures are skipped automatically. +- Existing successful signatures are skipped only when the request, repository revisions, runtime and benchmark/harness source, corpus inputs, and root lockfiles match. Changed inputs require a new run. Subscription checkpoint namespaces use the same source-sensitive execution contract; old namespaces are not reused. Cross-harness comparison grouping remains separate. - `--rerun-failed` reruns only signatures whose latest run failed. - `--force` always creates a fresh run. @@ -87,11 +94,11 @@ adapter discovery before spending quota. Each phase runs Eliza, Hermes, and OpenClaw together; phases and benchmarks advance serially, and the default stops on the first failed cohort. -The current manifest accounts for all 59 discovered adapters exactly once: +The current manifest accounts for all 58 discovered adapters exactly once: 25 automatic cohort entries (81 full phases, including all 55 ClawBench -scenarios and all three Mind2Web test splits), 21 externally provisioned manual -entries, 2 unsupported three-harness paths, and 11 non-agent/replay entries. It -also records 15 direct +scenarios and all three Mind2Web test splits), 17 externally provisioned manual +entries, 5 unsupported three-harness paths, and 11 non-agent/replay entries. It +also records 14 direct or supporting workloads that are outside adapter discovery. Unsupported, non-agent, infrastructure, and unintegrated work is reported, never converted into misleading comparison rows. @@ -100,8 +107,7 @@ Inspect the machine-checked workload and subscription-call ledger without launching a benchmark: ```bash -PYTHONPATH=packages python3 \ - -m benchmarks.orchestrator.campaign_ledger +python3 -m benchmarks.orchestrator.campaign_ledger ``` The report separates authored tasks, expanded scenarios, and scored result @@ -110,10 +116,10 @@ data-dependent rankers and native agent loops are never presented as fixed model-call totals. The 25 automatic cohorts currently total 33,981 base tasks, 213,801 expanded scenarios, and 308,639 result cells per harness (101,943 / 641,403 / 925,917 respectively across Eliza, Hermes, and OpenClaw). -Across all 46 intended comparative adapters, the exact-known subtotal is -67,373 base tasks, 577,403 expanded scenarios, and 672,241 result cells per -harness (202,119 / 1,732,209 / 2,016,723 across all three harnesses). These are -known-cardinality subtotals, not campaign totals: four externally selected +Across all 42 intended comparative adapters, the exact-known subtotal is +50,083 base tasks, 387,213 expanded scenarios, and 482,051 result cells per +harness (150,249 / 1,161,639 / 1,446,153 across all three harnesses). These are +known-cardinality subtotals, not campaign totals: three externally selected corpora still have unknown cardinality, and their IDs are emitted in every report. diff --git a/suites/orchestrator/artifact_guard.py b/suites/orchestrator/artifact_guard.py index d7f333c3..af1a255e 100644 --- a/suites/orchestrator/artifact_guard.py +++ b/suites/orchestrator/artifact_guard.py @@ -164,7 +164,7 @@ def build_artifact_guard_report( ``run_git`` is injectable so the classifier can be tested without a real repository; by default it shells out to ``git ls-files`` at the repo root. """ - repo_root = workspace_root.parent # workspace_root is ``.../packages`` + repo_root = workspace_root runner = run_git or _default_git_runner(repo_root) tracked = [line for line in runner(["ls-files"]).splitlines() if line.strip()] offending = find_committed_generated_artifacts(tracked) diff --git a/suites/orchestrator/campaign_ledger.py b/suites/orchestrator/campaign_ledger.py index d120e587..07f19fa6 100644 --- a/suites/orchestrator/campaign_ledger.py +++ b/suites/orchestrator/campaign_ledger.py @@ -1057,15 +1057,6 @@ def _direct( _M.NOT_APPLICABLE, basis="Discovered plugin bundles determine deterministic size-gate cells at execution time.", ), - _direct( - "voice_pipeline", - _C.NON_AGENT, - None, - None, - None, - _M.NOT_APPLICABLE, - basis="Native voice backends available to the CI matrix determine cells; no comparison harness is selected.", - ), _direct( "voice_emotion", _C.NON_AGENT, diff --git a/suites/orchestrator/cli.py b/suites/orchestrator/cli.py index 4cc3ced1..22cfd726 100644 --- a/suites/orchestrator/cli.py +++ b/suites/orchestrator/cli.py @@ -149,7 +149,7 @@ def _cmd_validate_matrix(args: argparse.Namespace) -> int: workspace_root = _workspace_root_from_here() _apply_model_profile(args, workspace_root) report = build_cross_matrix_report( - workspace_root.parent, + workspace_root, provider=args.provider, model=args.model, extra_config=_parse_json_arg(args.extra), @@ -163,7 +163,7 @@ def _cmd_validate_matrix(args: argparse.Namespace) -> int: def _cmd_inventory(args: argparse.Namespace) -> int: workspace_root = _workspace_root_from_here() - report = build_inventory_report(workspace_root.parent) + report = build_inventory_report(workspace_root) if args.format == "json": print(inventory_report_to_json(report)) else: @@ -550,7 +550,7 @@ def _emit(step: str, code: int) -> None: # baseline (they predate this wrapper) and are NOT drift to block a review on # — so preflight fails only when a *registered* benchmark has no adapter. print("[review] step 1/5: preflight (inventory)") - inventory = build_inventory_report(workspace_root.parent) + inventory = build_inventory_report(workspace_root) print(inventory_report_to_markdown(inventory)) if inventory.registry_entries_without_adapters: _emit("preflight (inventory)", 2) diff --git a/suites/orchestrator/code_agent_matrix.py b/suites/orchestrator/code_agent_matrix.py index a350cfef..e9740d4b 100644 --- a/suites/orchestrator/code_agent_matrix.py +++ b/suites/orchestrator/code_agent_matrix.py @@ -578,7 +578,12 @@ def preflight_matrix( ) opencode_bin = _opencode_bin(root, env) - if any(cell.adapter == "opencode" for cell in cells) and not opencode_bin: + # Explicit mock commands use the suite fixture agent, not the external CLI. + # A mixed cohort still requires OpenCode for every live cell. + if any( + cell.adapter == "opencode" and "--mock" not in cell.command + for cell in cells + ) and not opencode_bin: issues.append( { "severity": "error", diff --git a/suites/orchestrator/cohort.py b/suites/orchestrator/cohort.py index 7ad4732b..1a347e06 100644 --- a/suites/orchestrator/cohort.py +++ b/suites/orchestrator/cohort.py @@ -9,7 +9,6 @@ from __future__ import annotations -import hashlib import json import shutil import threading @@ -41,6 +40,8 @@ repair_nonzero_returncode_statuses, resume_run_group, ) +from .result_store import result_store_root +from .execution_identity import PhaseExecutionIdentity, build_phase_execution_identity from .locking import campaign_execution_lock from .provider_forwarder import ( ProviderForwarderProcess, @@ -88,49 +89,6 @@ DEFAULT_STORAGE_MIN_FREE_BYTES = 8 * 1024**3 DEFAULT_STORAGE_EXPECTED_HEADROOM_BYTES = 4 * 1024**3 DEFAULT_STORAGE_CHECK_INTERVAL_SECONDS = 30.0 -_STORAGE_CONTROL_KEYS = frozenset( - { - "campaign_storage_min_free_bytes", - "campaign_storage_expected_headroom_bytes", - "campaign_storage_check_interval_s", - } -) -_NAMESPACE_RUNTIME_KEYS = _STORAGE_CONTROL_KEYS | frozenset( - { - "_replace_adapter_defaults", - "campaign_silent_timeout_s", - "claude_subscription_gateway_url", - "eliza_bench_http_timeout_s", - "hermes_timeout_s", - "hl_bench_command_timeout_s", - "openclaw_timeout_s", - "timeout_s", - } -) -_FINGERPRINT_EXCLUDED_PARTS = frozenset( - { - ".cache", - ".git", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".turbo", - ".venv", - "__pycache__", - "artifacts", - "benchmark_results", - "build", - "coverage", - "dist", - "node_modules", - "outputs", - "results", - "runs", - "venv", - } -) - - class BenchmarkCohortStatus(StrEnum): SUCCEEDED = "succeeded" FAILED = "failed" @@ -139,13 +97,6 @@ class BenchmarkCohortStatus(StrEnum): UNSUPPORTED = "unsupported" -@dataclass(frozen=True) -class PhaseExecutionIdentity: - namespace: str - contract: Mapping[str, object] - checkpoint_relpath: str - - @dataclass(frozen=True) class StoragePreflight: checked_at: str @@ -213,124 +164,6 @@ def _canonical_json(value: object) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) -def build_phase_execution_identity( - *, - workspace_root: Path, - adapter: BenchmarkAdapter, - request: RunRequest, - harnesses: tuple[str, ...], - repo_meta: Mapping[str, str | None], -) -> PhaseExecutionIdentity: - """Build the PID/run-group-independent replay identity for one phase.""" - - extra = dict(request.extra_config) - dataset_config = { - key: value - for key, value in extra.items() - if key not in _NAMESPACE_RUNTIME_KEYS - and key - not in { - "campaign_corpus_sha256", - "campaign_phase", - "campaign_profile", - "reasoning_effort", - } - } - source_fingerprint = _relevant_source_fingerprint( - workspace_root=workspace_root, - adapter=adapter, - ) - explicit_corpus = extra.get("campaign_corpus_sha256") - if explicit_corpus is not None and ( - not isinstance(explicit_corpus, str) or not explicit_corpus.strip() - ): - raise ValueError("campaign_corpus_sha256 must be a non-empty string") - corpus_sha256 = ( - explicit_corpus.strip() - if isinstance(explicit_corpus, str) - else hashlib.sha256( - _canonical_json( - { - "adapter_directory": adapter.directory, - "benchmarks_commit": repo_meta.get("benchmarks_commit"), - "dataset_config": dataset_config, - "source_fingerprint_sha256": source_fingerprint, - } - ).encode("utf-8") - ).hexdigest() - ) - contract: dict[str, object] = { - "schema_version": 1, - "campaign_profile": str(extra.get("campaign_profile") or "ad-hoc"), - "benchmark_id": adapter.id, - "benchmark_directory": adapter.directory, - "phase": str(extra.get("campaign_phase") or "single"), - "dataset_config": dataset_config, - "corpus_sha256": corpus_sha256, - "source_fingerprint_sha256": source_fingerprint, - "model": request.model.strip(), - "provider": request.provider.strip().lower(), - "reasoning_effort": str(extra.get("reasoning_effort") or "").strip().lower(), - "harnesses": list(harnesses), - } - digest = hashlib.sha256(_canonical_json(contract).encode("utf-8")).hexdigest() - namespace = f"benchmark-phase-v1-{digest}" - return PhaseExecutionIdentity( - namespace=namespace, - contract=contract, - checkpoint_relpath=( - f".subscription-checkpoints/{namespace}/responses.jsonl" - ), - ) - - -def _relevant_source_fingerprint( - *, - workspace_root: Path, - adapter: BenchmarkAdapter, -) -> str: - """Hash dirty source/corpus inputs that a Git HEAD cannot represent.""" - - benchmarks_root = workspace_root / "suites" - candidate_roots = ( - benchmarks_root / adapter.directory, - benchmarks_root / "orchestrator", - benchmarks_root / "claude-subscription-gateway", - benchmarks_root.parent / "harnesses" / "eliza", - benchmarks_root.parent / "harnesses" / "hermes", - benchmarks_root.parent / "harnesses" / "openclaw", - workspace_root.parent / "package.json", - workspace_root.parent / "bun.lock", - workspace_root.parent / "bun.lockb", - ) - files: set[Path] = set() - for root in candidate_roots: - if root.is_file(): - files.add(root) - continue - if not root.is_dir(): - continue - for path in root.rglob("*"): - try: - relative_parts = path.relative_to(root).parts - except ValueError: - continue - if any(part in _FINGERPRINT_EXCLUDED_PARTS for part in relative_parts): - continue - if path.is_file() and not path.is_symlink(): - files.add(path) - digest = hashlib.sha256() - fingerprint_base = workspace_root.parent - for path in sorted(files, key=lambda value: value.as_posix()): - relative = path.relative_to(fingerprint_base).as_posix().encode("utf-8") - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) - with path.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - def _storage_integer(extra: Mapping[str, object], key: str, default: int) -> int: value = extra.get(key, default) if isinstance(value, bool) or not isinstance(value, int) or value < 0: @@ -422,7 +255,7 @@ def _prepare_cohort( execution_identity: PhaseExecutionIdentity | None, storage_preflight: StoragePreflight | None, ) -> None: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: initialize_database(conn) @@ -500,7 +333,7 @@ def _finalize_cohort( failures: dict[str, Exception], success_cleanup: Callable[[], None] | None = None, ) -> Path: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: if failures: @@ -693,7 +526,7 @@ def _attach_gateway_audit( run_group_id: str, audit_path: Path, ) -> dict[str, str]: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: return attach_subscription_gateway_provenance( @@ -764,7 +597,7 @@ def _find_reusable_subscription_cohort( if request.provider.strip().lower() != "claude-subscription": return None - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) db_path = output_root / "orchestrator.sqlite" if not db_path.is_file(): return None @@ -777,6 +610,8 @@ def _find_reusable_subscription_cohort( harness: _signature_for( adapter, _effective_request(adapter, replace(request, agent=harness)), + workspace_root=workspace_root, + repo_meta=_repo_meta(workspace_root), ) for harness in harnesses } @@ -874,9 +709,7 @@ def _load_resumable_execution( identity: PhaseExecutionIdentity, ) -> dict[str, object] | None: db_path = ( - workspace_root - - / "benchmark_results" + result_store_root(workspace_root) / "orchestrator.sqlite" ) if not db_path.is_file(): @@ -959,7 +792,7 @@ def _resume_cohort( identity: PhaseExecutionIdentity, storage_preflight: StoragePreflight, ) -> None: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: initialize_database(conn) @@ -996,7 +829,7 @@ def _persist_gateway_pause( pause: GatewayPauseState, extra_metadata: Mapping[str, object] | None = None, ) -> None: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: initialize_database(conn) @@ -1034,7 +867,7 @@ def _record_storage_preflight( run_group_id: str, preflight: StoragePreflight, ) -> None: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: record_run_group_storage_preflight( @@ -1077,7 +910,7 @@ def _refresh_publication( workspace_root: Path, adapters: dict[str, BenchmarkAdapter], ) -> Path: - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) conn = connect_database(output_root / "orchestrator.sqlite") try: initialize_database(conn) @@ -1131,7 +964,7 @@ def run_benchmark_cohorts( if request.provider.strip().lower() == "claude-subscription": # Fail without even allocating the results directory or campaign lock. check_campaign_storage(workspace_root=workspace_root, request=request) - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) output_root.mkdir(parents=True, exist_ok=True) campaign_repo_meta = _repo_meta(workspace_root) results: list[BenchmarkCohortResult] = [] diff --git a/suites/orchestrator/execution_identity.py b/suites/orchestrator/execution_identity.py new file mode 100644 index 00000000..804dc32c --- /dev/null +++ b/suites/orchestrator/execution_identity.py @@ -0,0 +1,196 @@ +"""Binds benchmark resume and provider checkpoints to complete execution inputs. + +Comparison grouping is deliberately separate from this source-sensitive cache +identity. Generated output and dependency caches do not change an execution; +runtime source, revisions, corpus, harnesses, and lockfiles do. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +from .types import BenchmarkAdapter, RunRequest + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +_STORAGE_CONTROL_KEYS = frozenset( + { + "campaign_storage_min_free_bytes", + "campaign_storage_expected_headroom_bytes", + "campaign_storage_check_interval_s", + } +) +_NAMESPACE_RUNTIME_KEYS = _STORAGE_CONTROL_KEYS | frozenset( + { + "_replace_adapter_defaults", + "campaign_silent_timeout_s", + "claude_subscription_gateway_url", + "eliza_bench_http_timeout_s", + "hermes_timeout_s", + "hl_bench_command_timeout_s", + "openclaw_timeout_s", + "timeout_s", + } +) +_FINGERPRINT_EXCLUDED_PARTS = frozenset( + { + ".cache", + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".turbo", + ".venv", + "__pycache__", + "artifacts", + "benchmark_results", + "build", + "coverage", + "dist", + "node_modules", + "outputs", + "results", + "runs", + "venv", + } +) + + +@dataclass(frozen=True) +class PhaseExecutionIdentity: + namespace: str + contract: Mapping[str, object] + checkpoint_relpath: str + + +def build_phase_execution_identity( + *, + workspace_root: Path, + adapter: BenchmarkAdapter, + request: RunRequest, + harnesses: tuple[str, ...], + repo_meta: Mapping[str, str | None], +) -> PhaseExecutionIdentity: + """Build the PID/run-group-independent replay identity for one phase.""" + + extra = dict(request.extra_config) + dataset_config = { + key: value + for key, value in extra.items() + if key not in _NAMESPACE_RUNTIME_KEYS + and key + not in { + "campaign_corpus_sha256", + "campaign_phase", + "campaign_profile", + "reasoning_effort", + } + } + source_fingerprint = _relevant_source_fingerprint( + workspace_root=workspace_root, + adapter=adapter, + ) + explicit_corpus = extra.get("campaign_corpus_sha256") + if explicit_corpus is not None and ( + not isinstance(explicit_corpus, str) or not explicit_corpus.strip() + ): + raise ValueError("campaign_corpus_sha256 must be a non-empty string") + corpus_sha256 = ( + explicit_corpus.strip() + if isinstance(explicit_corpus, str) + else hashlib.sha256( + _canonical_json( + { + "adapter_directory": adapter.directory, + "benchmarks_commit": repo_meta.get("benchmarks_commit"), + "dataset_config": dataset_config, + "source_fingerprint_sha256": source_fingerprint, + } + ).encode("utf-8") + ).hexdigest() + ) + contract: dict[str, object] = { + "schema_version": 2, + "repositories": dict(repo_meta), + "campaign_profile": str(extra.get("campaign_profile") or "ad-hoc"), + "benchmark_id": adapter.id, + "benchmark_directory": adapter.directory, + "phase": str(extra.get("campaign_phase") or "single"), + "dataset_config": dataset_config, + "corpus_sha256": corpus_sha256, + "source_fingerprint_sha256": source_fingerprint, + "model": request.model.strip(), + "provider": request.provider.strip().lower(), + "reasoning_effort": str(extra.get("reasoning_effort") or "").strip().lower(), + "harnesses": list(harnesses), + } + digest = hashlib.sha256(_canonical_json(contract).encode("utf-8")).hexdigest() + namespace = f"benchmark-phase-v2-{digest}" + return PhaseExecutionIdentity( + namespace=namespace, + contract=contract, + checkpoint_relpath=(f".subscription-checkpoints/{namespace}/responses.jsonl"), + ) + + +def _relevant_source_fingerprint( + *, + workspace_root: Path, + adapter: BenchmarkAdapter, +) -> str: + """Hash dirty source/corpus inputs that a Git HEAD cannot represent.""" + + benchmarks_root = workspace_root / "suites" + candidate_roots = ( + benchmarks_root / adapter.directory, + benchmarks_root / "orchestrator", + benchmarks_root / "claude-subscription-gateway", + workspace_root / "harnesses", + workspace_root / "eliza", + workspace_root / "registry", + workspace_root / "framework", + workspace_root / "lib", + workspace_root / "package.json", + workspace_root / "bun.lock", + workspace_root / "bun.lockb", + *sorted(workspace_root.glob("*.py")), + ) + files: set[Path] = set() + for root in candidate_roots: + if root.is_file(): + files.add(root) + continue + if not root.is_dir(): + continue + for directory, subdirs, names in os.walk(root, followlinks=False): + subdirs[:] = sorted( + name + for name in subdirs + if name not in _FINGERPRINT_EXCLUDED_PARTS + and not (Path(directory) / name).is_symlink() + ) + for name in names: + path = Path(directory) / name + if name.startswith(".env") or path.is_symlink(): + continue + if path.is_file(): + files.add(path) + digest = hashlib.sha256() + fingerprint_base = workspace_root + for path in sorted(files, key=lambda value: value.as_posix()): + relative = path.relative_to(fingerprint_base).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(path.stat().st_size.to_bytes(8, "big")) + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() diff --git a/suites/orchestrator/full_campaign.py b/suites/orchestrator/full_campaign.py index 238e9ba3..b2b0615b 100644 --- a/suites/orchestrator/full_campaign.py +++ b/suites/orchestrator/full_campaign.py @@ -451,7 +451,7 @@ def _mind2web_phases() -> tuple[FullCampaignPhase, ...]: ), _entry( "framework", - "framework", + "../framework", registered=False, disposition=CampaignDisposition.UNSUPPORTED, phases=_phase( @@ -755,7 +755,7 @@ def _mind2web_phases() -> tuple[FullCampaignPhase, ...]: DIRECT_CAMPAIGN_ENTRIES: tuple[DirectCampaignEntry, ...] = ( DirectCampaignEntry( "agentbench_matrix", - "agentbench_matrix", + "orchestrator", CampaignDisposition.MANUAL, ("python", "-m", "benchmarks.orchestrator.code_agent_matrix"), "Code-agent matrix workbench; imported by its dedicated orchestrator and not a cohort adapter.", diff --git a/suites/orchestrator/result_store.py b/suites/orchestrator/result_store.py new file mode 100644 index 00000000..db13a431 --- /dev/null +++ b/suites/orchestrator/result_store.py @@ -0,0 +1,17 @@ +"""Resolve the standalone result store without silently mixing legacy artifacts.""" +from pathlib import Path + + +class LegacyResultStoreError(RuntimeError): + """A caller must explicitly archive or migrate results from the retired path.""" + + +def result_store_root(workspace_root: Path) -> Path: + legacy = workspace_root / "suites" / "benchmark_results" + if legacy.is_dir() and next(legacy.iterdir(), None) is not None: + raise LegacyResultStoreError( + f"Legacy benchmark results exist at {legacy}. Archive or migrate that " + f"directory explicitly before running or resuming in " + f"{workspace_root / 'benchmark_results'}; no artifacts were moved or combined." + ) + return workspace_root / "benchmark_results" diff --git a/suites/orchestrator/runner.py b/suites/orchestrator/runner.py index 0d821535..7277899a 100644 --- a/suites/orchestrator/runner.py +++ b/suites/orchestrator/runner.py @@ -46,6 +46,8 @@ replace_run_trajectories, update_run_result, ) +from .result_store import result_store_root +from .execution_identity import build_phase_execution_identity from .env_utils import ( git_head, load_env_file, @@ -135,7 +137,13 @@ def _sanitize_name(value: str) -> str: return cleaned or "item" -def _signature_for(adapter: BenchmarkAdapter, request: RunRequest) -> str: +def _signature_for( + adapter: BenchmarkAdapter, + request: RunRequest, + *, + workspace_root: Path, + repo_meta: dict[str, str | None], +) -> str: extra_config = dict(request.extra_config) if request.agent.strip().lower() in CALIBRATION_HARNESSES: extra_config["calibration_spec_version"] = CALIBRATION_SPEC_VERSION @@ -146,6 +154,13 @@ def _signature_for(adapter: BenchmarkAdapter, request: RunRequest) -> str: "provider": request.provider, "model": request.model, "extra_config": extra_config, + "execution_namespace": build_phase_execution_identity( + workspace_root=workspace_root, + adapter=adapter, + request=request, + harnesses=(request.agent,), + repo_meta=repo_meta, + ).namespace, } return hashlib.sha256( json.dumps( @@ -590,14 +605,13 @@ def _default_env(workspace_root: Path, request: RunRequest) -> dict[str, str]: def _repo_meta(workspace_root: Path) -> dict[str, str | None]: - benchmarks_root = workspace_root / "suites" eliza_root = workspace_root / "eliza" return { - "benchmarks_commit": git_head(benchmarks_root), - "eliza_commit": git_head(eliza_root), + "benchmarks_commit": git_head(workspace_root), + "eliza_commit": git_head(eliza_root) if (eliza_root / ".git").exists() else None, "eliza_version": safe_version_from_package_json(eliza_root / "package.json"), "benchmarks_version": safe_version_from_package_json( - benchmarks_root / "package.json" + workspace_root / "package.json" ), } @@ -1057,7 +1071,7 @@ def _ensure_viewer_snapshot( ) -> Path: from .viewer_data import build_viewer_dataset - output_root = workspace_root / "suites" / "benchmark_results" + output_root = result_store_root(workspace_root) out = output_root / "viewer_data.json" out.parent.mkdir(parents=True, exist_ok=True) with latest_publication_lock(output_root): @@ -2991,7 +3005,7 @@ def run_benchmarks( "shared_run_group_id and defer_publication must be supplied together" ) benchmarks_root = workspace_root / "suites" - output_root = benchmarks_root / "benchmark_results" + output_root = result_store_root(workspace_root) output_root.mkdir(parents=True, exist_ok=True) run_group_id = shared_run_group_id or ( f"rg_{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{uuid4().hex[:8]}" @@ -3043,7 +3057,9 @@ def run_benchmarks( for benchmark_id in selected_ids: adapter = discovery.adapters[benchmark_id] effective_request = _effective_request(adapter, request) - signature = _signature_for(adapter, effective_request) + signature = _signature_for( + adapter, effective_request, workspace_root=workspace_root, repo_meta=repo_meta + ) # Harness/agent compatibility — if the harness is not in the adapter's # supported list, record an ``incompatible`` outcome and skip without diff --git a/suites/orchestrator/tests/test_adapter_discovery.py b/suites/orchestrator/tests/test_adapter_discovery.py index cb2d6a61..8c525e3a 100644 --- a/suites/orchestrator/tests/test_adapter_discovery.py +++ b/suites/orchestrator/tests/test_adapter_discovery.py @@ -1109,7 +1109,7 @@ def test_cross_matrix_validation_constructs_all_compatible_cells( lambda: ("eliza", "hermes", "openclaw"), ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1152,7 +1152,7 @@ def test_hyperliquid_matrix_rows_require_live_credentials( ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1174,7 +1174,7 @@ def test_hyperliquid_live_matrix_rows_require_trading_key( ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1189,7 +1189,7 @@ def test_hyperliquid_live_matrix_rows_require_trading_key( def test_cross_matrix_validation_redacts_secret_config_values() -> None: report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", extra_config={ @@ -1237,7 +1237,7 @@ def test_direct_and_native_rows_keep_truthful_matrix_compatibility( ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1355,7 +1355,7 @@ def test_real_matrix_compatible_commands_do_not_default_to_mock_or_stub( lambda: False, ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1841,7 +1841,7 @@ def test_osworld_requires_reachable_docker_backend( assert _is_harness_compatible(adapter, "openclaw") is False report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), harnesses=("eliza", "hermes", "openclaw"), ) for cell in [cell for cell in report.cells if cell.benchmark_id == "osworld"]: @@ -1893,7 +1893,7 @@ def test_hermes_native_env_matrix_reports_sandbox_unavailable( ) report = build_cross_matrix_report( - _workspace_root().parent, + _workspace_root(), provider="cerebras", model="gpt-oss-120b", ) @@ -1966,7 +1966,7 @@ def test_smithers_benchmark_compatibility_has_real_routes( assert missing_from_registry == [] report = build_cross_matrix_report( - workspace_root.parent, + workspace_root, harnesses=("smithers",), provider="cerebras", model="gpt-oss-120b", @@ -1982,9 +1982,7 @@ def test_smithers_benchmark_compatibility_has_real_routes( def test_smithers_adapter_modules_import_for_declared_factories() -> None: - adapter_root = ( - _workspace_root().parent / "harnesses" / "smithers" - ) + adapter_root = _workspace_root() / "harnesses" / "smithers" sys.path.insert(0, str(adapter_root)) try: for module_name in ( diff --git a/suites/orchestrator/tests/test_campaign_ledger.py b/suites/orchestrator/tests/test_campaign_ledger.py index ddaf069d..15857289 100644 --- a/suites/orchestrator/tests/test_campaign_ledger.py +++ b/suites/orchestrator/tests/test_campaign_ledger.py @@ -25,9 +25,9 @@ def test_ledger_exactly_covers_every_manifest_entry_and_disposition() -> None: if entry.kind is ledger.CampaignEntryKind.DIRECT } - assert len(entries) == 73 + assert len(entries) == 72 assert len(adapters) == 58 - assert len(direct) == 15 + assert len(direct) == 14 assert set(adapters) == { entry.benchmark_id for entry in campaign.ADAPTER_CAMPAIGN_ENTRIES } @@ -321,8 +321,8 @@ def test_cli_json_is_a_validated_side_effect_free_report(capsys) -> None: assert payload["campaign_profile"] == campaign.FULL_CAMPAIGN_PROFILE assert payload["canonical_harnesses"] == ["eliza", "hermes", "openclaw"] assert payload["manifest"] == { - "entries": 73, + "entries": 72, "adapter_entries": 58, - "direct_entries": 15, + "direct_entries": 14, } - assert len(payload["entries"]) == 73 + assert len(payload["entries"]) == 72 diff --git a/suites/orchestrator/tests/test_cli_repository_root.py b/suites/orchestrator/tests/test_cli_repository_root.py new file mode 100644 index 00000000..c9d358e6 --- /dev/null +++ b/suites/orchestrator/tests/test_cli_repository_root.py @@ -0,0 +1,74 @@ +"""Exercise standalone matrix and inventory commands outside the checkout cwd.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("command", ["validate-matrix", "inventory"]) +def test_repository_commands_resolve_their_own_checkout(command: str, tmp_path: Path) -> None: + repository = Path(__file__).resolve().parents[3] + env = os.environ.copy() + env["PYTHONPATH"] = str(repository.parent) + result = subprocess.run( + [sys.executable, "-m", "benchmarks.orchestrator", command, "--format", "json"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stderr + result.stdout + report = json.loads(result.stdout) + if command == "validate-matrix": + assert report["cells"], "The command must produce adapter/harness decisions" + assert report["error_count"] == 0 + else: + assert report["adapter_count"] > 0 + + +@pytest.mark.parametrize("smoke", [True, False]) +def test_code_agent_preflight_requires_only_executed_adapter(tmp_path: Path, smoke: bool) -> None: + repository = Path(__file__).resolve().parents[3] + env = os.environ.copy() + env["PYTHONPATH"] = str(repository.parent) + env["OPENCODE_BIN"] = str(tmp_path / "absent-opencode") + env.pop("CEREBRAS_API_KEY", None) + command = [sys.executable, "-m", "benchmarks.orchestrator.code_agent_matrix", + "--benchmarks", "webshop", "--adapters", "opencode", "--preflight", + "--run-root", str(tmp_path / "run")] + if smoke: + command.append("--smoke") + result = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, + text=True, timeout=60, check=False) + report = json.loads(result.stdout) + findings = {issue["kind"] for issue in report["issues"]} + if smoke: + assert result.returncode == 0, result.stderr + result.stdout + assert report["ok"] is True + assert findings == set() + else: + assert result.returncode == 2 + assert "missing_opencode_cli" in findings + assert "missing_provider_key" in findings + + +def test_mixed_mock_and_live_cells_keep_opencode_prerequisite(tmp_path: Path) -> None: + from benchmarks.orchestrator.code_agent_matrix import build_cell, preflight_matrix + + repository = Path(__file__).resolve().parents[3] + cells = [build_cell(root=repository, run_root=tmp_path / str(smoke), + benchmark="webshop", adapter="opencode", provider="cerebras", + model="fixture-model", max_tasks=1, smoke=smoke, no_docker=True) + for smoke in (True, False)] + report = preflight_matrix(root=repository, cells=cells, provider="cerebras", + require_provider_key=False, + env={"OPENCODE_BIN": str(tmp_path / "absent-opencode")}) + assert report["ok"] is False + assert [issue["kind"] for issue in report["issues"]] == ["missing_opencode_cli"] diff --git a/suites/orchestrator/tests/test_code_agent_matrix.py b/suites/orchestrator/tests/test_code_agent_matrix.py index 3a4e2092..6fbfe2be 100644 --- a/suites/orchestrator/tests/test_code_agent_matrix.py +++ b/suites/orchestrator/tests/test_code_agent_matrix.py @@ -77,7 +77,7 @@ def _root() -> Path: def test_builds_swe_bench_elizaos_cell_without_secret_values(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="swe_bench", adapter="elizaos", @@ -113,7 +113,7 @@ def _clear_base_url_env(monkeypatch: pytest.MonkeyPatch) -> None: def _build_base_url_probe_cell(tmp_path: Path, provider: str = "cerebras"): return build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="swe_bench", adapter="elizaos", @@ -232,7 +232,7 @@ def test_swe_bench_repo_cache_dir_can_be_overridden( def test_builds_swe_bench_multilingual_cell_with_variant(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="swe_bench_multilingual", adapter="opencode", @@ -472,7 +472,7 @@ def test_builds_nl2repo_cell_with_optional_agent_command_template( ) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="nl2repo", adapter="opencode", @@ -505,7 +505,7 @@ def test_nl2repo_cell_uses_builtin_agent_command_by_default( monkeypatch.delenv("NL2REPO_DISABLE_BUILTIN_AGENT_COMMAND", raising=False) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="nl2repo", adapter="elizaos", @@ -532,7 +532,7 @@ def test_standard_humaneval_cell_uses_builtin_agent_command_by_default( monkeypatch.delenv("STANDARD_HUMANEVAL_DISABLE_BUILTIN_AGENT_COMMAND", raising=False) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="standard_humaneval", adapter="elizaos", @@ -558,7 +558,7 @@ def test_standard_humaneval_cell_uses_builtin_agent_command_by_default( def test_mint_cell_runs_coding_slice_through_matrix_wrapper(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="mint", adapter="opencode", @@ -589,7 +589,7 @@ def test_browser_and_agentbench_cells_forward_edge_expansion_env( for benchmark in ("mind2web", "visualwebbench", "webshop", "agentbench"): cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path / benchmark, benchmark=benchmark, adapter="opencode", @@ -604,7 +604,7 @@ def test_browser_and_agentbench_cells_forward_edge_expansion_env( def test_agentbench_cell_runs_matrix_wrapper(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="agentbench", adapter="opencode", @@ -630,7 +630,7 @@ def test_agentbench_cell_runs_matrix_wrapper(tmp_path: Path) -> None: def test_builds_real_webshop_cell_with_bridge(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="elizaos", @@ -649,7 +649,7 @@ def test_builds_real_webshop_cell_with_bridge(tmp_path: Path) -> None: def test_builds_five_task_webshop_cell_with_train_split(tmp_path: Path) -> None: cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="elizaos", @@ -2628,7 +2628,7 @@ def test_preflight_reports_missing_provider_key_and_opencode_cli(tmp_path: Path) provider="cerebras", model="gpt-oss-120b", max_tasks=1, - smoke=True, + smoke=False, no_docker=True, ) @@ -2688,7 +2688,7 @@ def test_preflight_can_skip_provider_key_for_smoke_or_dry_runs(tmp_path: Path) - opencode_bin = tmp_path / "opencode" opencode_bin.write_text("#!/bin/sh\n", encoding="utf-8") cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2721,7 +2721,7 @@ def test_preflight_blocks_release_readiness_without_quality_guardrail_summary( opencode_bin = tmp_path / "opencode" opencode_bin.write_text("#!/bin/sh\n", encoding="utf-8") cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2768,7 +2768,7 @@ def test_preflight_blocks_release_readiness_for_missing_quality_guardrail_file( opencode_bin.write_text("#!/bin/sh\n", encoding="utf-8") missing_guardrail = tmp_path / "missing-readiness.json" cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2816,7 +2816,7 @@ def test_preflight_accepts_existing_quality_guardrail_summary( encoding="utf-8", ) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2864,7 +2864,7 @@ def test_preflight_rejects_quality_guardrail_summary_with_missing_latest_dir( encoding="utf-8", ) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2905,7 +2905,7 @@ def test_preflight_rejects_handwritten_quality_guardrail_summary( encoding="utf-8", ) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -2961,7 +2961,7 @@ def test_preflight_blocks_unclean_quality_guardrail_summary( encoding="utf-8", ) cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="webshop", adapter="opencode", @@ -3009,7 +3009,7 @@ def test_nl2repo_preflight_blocks_live_without_agent_command_template( monkeypatch.delenv("NL2REPO_AGENT_COMMAND_TEMPLATE_ELIZAOS", raising=False) monkeypatch.setenv("NL2REPO_DISABLE_BUILTIN_AGENT_COMMAND", "1") cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="nl2repo", adapter="elizaos", @@ -3052,7 +3052,7 @@ def test_standard_humaneval_preflight_blocks_live_without_agent_command_template monkeypatch.delenv("STANDARD_HUMANEVAL_AGENT_COMMAND_TEMPLATE_ELIZAOS", raising=False) monkeypatch.setenv("STANDARD_HUMANEVAL_DISABLE_BUILTIN_AGENT_COMMAND", "1") cell = build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="standard_humaneval", adapter="elizaos", @@ -3094,7 +3094,7 @@ def test_nl2repo_preflight_reports_shared_docker_issue_once( monkeypatch.delenv("NL2REPO_DISABLE_BUILTIN_AGENT_COMMAND", raising=False) cells = [ build_cell( - root=_root().parent, + root=_root(), run_root=tmp_path, benchmark="nl2repo", adapter=adapter, diff --git a/suites/orchestrator/tests/test_cohort.py b/suites/orchestrator/tests/test_cohort.py index 4adcdfd3..4ff1d2b8 100644 --- a/suites/orchestrator/tests/test_cohort.py +++ b/suites/orchestrator/tests/test_cohort.py @@ -89,7 +89,7 @@ def _request(*benchmark_ids: str) -> RunRequest: def _workspace(tmp_path: Path) -> Path: workspace_root = tmp_path / "packages" - (workspace_root / "benchmarks").mkdir(parents=True) + (workspace_root / "suites").mkdir(parents=True) return workspace_root @@ -771,7 +771,6 @@ def fake_run_benchmarks(**kwargs): if request.agent == "hermes": conn = connect_database( workspace_root - / "benchmarks" / "benchmark_results" / "orchestrator.sqlite" ) diff --git a/suites/orchestrator/tests/test_cohort_provider_forwarder.py b/suites/orchestrator/tests/test_cohort_provider_forwarder.py index c32685b9..f650b211 100644 --- a/suites/orchestrator/tests/test_cohort_provider_forwarder.py +++ b/suites/orchestrator/tests/test_cohort_provider_forwarder.py @@ -83,7 +83,7 @@ def _outcome(benchmark_id: str, harness: str) -> BenchmarkRunOutcome: def _workspace(tmp_path: Path) -> Path: workspace_root = tmp_path / "packages" - (workspace_root / "benchmarks").mkdir(parents=True) + (workspace_root / "suites").mkdir(parents=True) return workspace_root @@ -367,6 +367,8 @@ def run_and_capture(remote: bool, subdir: str) -> dict[str, str]: harness: runner_module._signature_for( adapter, runner_module._effective_request(adapter, request), + workspace_root=workspace_root, + repo_meta=runner_module._repo_meta(workspace_root), ) for harness, request in captured.items() } diff --git a/suites/orchestrator/tests/test_execution_identity.py b/suites/orchestrator/tests/test_execution_identity.py new file mode 100644 index 00000000..a154e238 --- /dev/null +++ b/suites/orchestrator/tests/test_execution_identity.py @@ -0,0 +1,194 @@ +"""Checks real filesystem inputs and SQLite reuse after runtime/source changes.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from benchmarks.orchestrator.cohort import build_phase_execution_identity +from benchmarks.orchestrator.db import ( + connect_database, + initialize_database, + insert_run_start, + get_latest_succeeded_run_for_signature, + update_run_result, +) +from benchmarks.orchestrator.runner import ( + _signature_for, + _comparison_signature_for, + _repo_meta, +) +from benchmarks.orchestrator.types import BenchmarkAdapter, RunRequest, ScoreSummary + + +def _inputs(root: Path): + adapter = BenchmarkAdapter( + id="context", + directory="context", + description="Context integration fixture", + cwd=".", + command_builder=lambda *_: [], + result_locator=lambda *_: None, + score_extractor=lambda _: ScoreSummary(1.0, "ratio", True), + agent_compatibility=("eliza",), + ) + request = RunRequest( + benchmarks=("context",), + agent="eliza", + provider="test", + model="test", + extra_config={"campaign_corpus_sha256": "pinned-corpus"}, + ) + return adapter, request + + +def _signature(root, adapter, request, meta): + return _signature_for(adapter, request, workspace_root=root, repo_meta=meta) + + +@pytest.mark.parametrize( + "relative", + [ + "eliza/packages/core/src/runtime.ts", + "suites/context/corpus.jsonl", + "harnesses/eliza/agent.ts", + "harnesses/codex/runner.py", + "run.py", + "registry/commands.py", + "framework/runner.py", + "lib/results.py", + "package.json", + "bun.lock", + ], +) +def test_changed_inputs_cannot_reuse_prior_success_or_checkpoint( + tmp_path: Path, relative: str +): + source = tmp_path / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("first complete input Ω\n", encoding="utf-8") + adapter, request = _inputs(tmp_path) + meta = {"benchmarks_commit": "benchmark-a", "eliza_commit": "runtime-a"} + signature = _signature(tmp_path, adapter, request, meta) + identity = build_phase_execution_identity( + workspace_root=tmp_path, + adapter=adapter, + request=request, + harnesses=("eliza",), + repo_meta=meta, + ) + comparison = _comparison_signature_for(adapter, request) + conn = connect_database(tmp_path / "suites/benchmark_results/orchestrator.sqlite") + try: + initialize_database(conn) + insert_run_start( + conn, + run_id="prior", + run_group_id="group", + benchmark_id=adapter.id, + benchmark_directory=adapter.directory, + signature=signature, + attempt=1, + agent=request.agent, + provider=request.provider, + model=request.model, + extra_config=dict(request.extra_config), + started_at="2026-09-14T00:00:00Z", + command=[], + cwd=str(tmp_path), + stdout_path="", + stderr_path="", + benchmark_version=None, + benchmarks_commit="benchmark-a", + eliza_commit="runtime-a", + eliza_version=None, + ) + update_run_result( + conn, + run_id="prior", + status="succeeded", + ended_at="2026-09-14T00:00:01Z", + duration_seconds=1, + score=None, + unit=None, + higher_is_better=None, + metrics={}, + result_json_path=None, + artifacts=[], + error=None, + high_score_label=None, + high_score_value=None, + delta_to_high_score=None, + ) + unchanged = _signature(tmp_path, adapter, request, meta) + assert get_latest_succeeded_run_for_signature(conn, unchanged).run_id == "prior" + assert ( + build_phase_execution_identity( + workspace_root=tmp_path, + adapter=adapter, + request=request, + harnesses=("eliza",), + repo_meta=meta, + ).checkpoint_relpath + == identity.checkpoint_relpath + ) + source.write_text("second complete input Ω\n", encoding="utf-8") + changed = _signature(tmp_path, adapter, request, meta) + assert get_latest_succeeded_run_for_signature(conn, changed) is None + assert ( + build_phase_execution_identity( + workspace_root=tmp_path, + adapter=adapter, + request=request, + harnesses=("eliza",), + repo_meta=meta, + ).checkpoint_relpath + != identity.checkpoint_relpath + ) + assert _comparison_signature_for(adapter, request) == comparison + finally: + conn.close() + + +def test_revision_changes_invalidate_even_with_explicit_corpus(tmp_path: Path): + adapter, request = _inputs(tmp_path) + initial = {"benchmarks_commit": "a", "eliza_commit": "b"} + previous = _signature(tmp_path, adapter, request, initial) + for key in initial: + assert ( + _signature(tmp_path, adapter, request, {**initial, key: "changed"}) + != previous + ) + + +def test_uninitialized_runtime_is_not_attributed_to_parent(tmp_path: Path): + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + (tmp_path / "package.json").write_text('{"version":"1.2.3"}') + subprocess.run(["git", "-C", str(tmp_path), "add", "package.json"], check=True) + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + (tmp_path / "eliza").mkdir() + meta = _repo_meta(tmp_path) + assert ( + meta["benchmarks_commit"] + == subprocess.check_output( + ["git", "-C", str(tmp_path), "rev-parse", "HEAD"], text=True + ).strip() + ) + assert meta["eliza_commit"] is None + assert meta["benchmarks_version"] == "1.2.3" diff --git a/suites/orchestrator/tests/test_inventory.py b/suites/orchestrator/tests/test_inventory.py index cadddf98..b4567efc 100644 --- a/suites/orchestrator/tests/test_inventory.py +++ b/suites/orchestrator/tests/test_inventory.py @@ -17,7 +17,7 @@ def _workspace_root() -> Path: def test_inventory_report_lists_real_adapters_and_operator_contracts() -> None: - report = build_inventory_report(_workspace_root().parent) + report = build_inventory_report(_workspace_root()) assert report.adapter_count == len(report.rows) assert report.registry_entry_count > 0 diff --git a/suites/orchestrator/tests/test_result_store.py b/suites/orchestrator/tests/test_result_store.py new file mode 100644 index 00000000..2fef9385 --- /dev/null +++ b/suites/orchestrator/tests/test_result_store.py @@ -0,0 +1,33 @@ +"""Check filesystem preservation and explicit legacy-store rejection before dispatch.""" +from pathlib import Path +import pytest +from benchmarks.orchestrator.runner import run_benchmarks +from benchmarks.orchestrator.types import RunRequest +from benchmarks.orchestrator.result_store import LegacyResultStoreError, result_store_root + + +def test_fresh_store_is_repository_local_without_creating_directories(tmp_path: Path) -> None: + target = result_store_root(tmp_path) + assert target == tmp_path / "benchmark_results" + assert not target.exists() + + +@pytest.mark.parametrize("canonical_exists", [False, True]) +def test_legacy_results_are_preserved_and_never_implicitly_combined(tmp_path: Path, canonical_exists: bool) -> None: + legacy = tmp_path / "suites" / "benchmark_results" + legacy.mkdir(parents=True) + saved = legacy / "result.json" + saved.write_text('{"score": 0.75}\n') + canonical = tmp_path / "benchmark_results" + if canonical_exists: + canonical.mkdir() + (canonical / "result.json").write_text('{"score": 0.5}\n') + with pytest.raises(LegacyResultStoreError, match="Archive or migrate"): + run_benchmarks(workspace_root=tmp_path, + request=RunRequest(benchmarks=("unused",), agent="eliza", + provider="mock", model="mock", extra_config={}), + execution_repo_meta={}) + assert saved.read_text() == '{"score": 0.75}\n' + assert canonical.exists() is canonical_exists + if canonical_exists: + assert (canonical / "result.json").read_text() == '{"score": 0.5}\n' diff --git a/suites/orchestrator/tests/test_runner_deadlines.py b/suites/orchestrator/tests/test_runner_deadlines.py index d530e2e4..18e2d8d6 100644 --- a/suites/orchestrator/tests/test_runner_deadlines.py +++ b/suites/orchestrator/tests/test_runner_deadlines.py @@ -13,6 +13,7 @@ from benchmarks.campaign_profile import FULL_CAMPAIGN_PROFILE from benchmarks.orchestrator import runner +from benchmarks.orchestrator.viewer_server import _load_trajectories from benchmarks.orchestrator.db import ( connect_database, create_run_group, @@ -252,6 +253,11 @@ def test_generic_runner_inserts_row_and_reaches_real_process_boundary( "from pathlib import Path\n" "output = Path(os.environ['BENCHMARK_OUTPUT_ROOT'])\n" "(output / 'result.json').write_text(json.dumps({'score': 0.75}))\n" + "row = {'format': 'eliza_native_v1', 'boundary': 'vercel_ai_sdk.generateText', " + "'request': {'messages': [{'role': 'user', 'content': 'complete fixture evidence ' * 5000 + 'END'}]}, " + "'response': {'text': 'observed fixture reply'}, 'agent_id': 'eliza', " + "'benchmark_id': 'generic-process', 'task_id': output.parent.name, 'step_index': 0}\n" + "(output / 'trajectory.canonical.jsonl').write_text(json.dumps(row) + '\\n')\n" "print('generic-process-reached', flush=True)\n" ) adapter = BenchmarkAdapter( @@ -310,3 +316,11 @@ def test_generic_runner_inserts_row_and_reaches_real_process_boundary( assert len(rows) == 1 assert rows[0]["run_group_id"] == "rg-generic-process" assert rows[0]["status"] == "succeeded" + + payload = _load_trajectories(workspace_root, run_group_id=run_group_id, + benchmark_id=adapter.id, task_id=outcomes[0].run_id) + observed = payload["harnesses"]["eliza"] + assert len(observed) == 1 + assert observed[0]["request"]["messages"][0]["content"] == "complete fixture evidence " * 5000 + "END" + assert observed[0]["response"]["text"] == "observed fixture reply" + assert not (workspace_root / "suites" / "benchmark_results").exists() diff --git a/suites/orchestrator/tests/test_subscription_gateway.py b/suites/orchestrator/tests/test_subscription_gateway.py index 6aa3d078..3eadbfe6 100644 --- a/suites/orchestrator/tests/test_subscription_gateway.py +++ b/suites/orchestrator/tests/test_subscription_gateway.py @@ -268,7 +268,6 @@ def test_gateway_log_open_failure_never_stages_content_contract( workspace = _workspace(tmp_path) process_root = ( workspace - / "benchmarks" / "benchmark_results" / "rg_contract_log_failure" / "subscription-gateway" diff --git a/suites/orchestrator/tests/test_viewer_server.py b/suites/orchestrator/tests/test_viewer_server.py index 682fef1c..4d6169da 100644 --- a/suites/orchestrator/tests/test_viewer_server.py +++ b/suites/orchestrator/tests/test_viewer_server.py @@ -61,7 +61,6 @@ def test_load_trajectories_includes_smithers_canonical_file(tmp_path: Path) -> N task_id = "smithers_woobench_20260702T100000Z_1_abc123" canonical_path = ( workspace_root - / "benchmarks" / "benchmark_results" / "rg_live" / "woobench"