From 5e8c53a17f5b73aa79b582df3a8440d765d7ebc3 Mon Sep 17 00:00:00 2001 From: tadamcz Date: Mon, 9 Nov 2026 20:09:52 -0500 Subject: [PATCH] Add swarm_leader: lead a swarm with a leader member, via member names and roles --- apn/solver.py | 32 ++++++++++++++++++-- apn/task.py | 55 ++++++++++++++++++----------------- tests/test_agent.py | 27 +++++++++++++++++ tests/test_sandbox_backend.py | 17 +++++++++++ 4 files changed, 102 insertions(+), 29 deletions(-) diff --git a/apn/solver.py b/apn/solver.py index 1dca982..2950286 100644 --- a/apn/solver.py +++ b/apn/solver.py @@ -151,12 +151,36 @@ def _prover_submit() -> AgentSubmit: CONTINUE_MESSAGE = "Continue working on the problem." -def swarm_member_names(swarm_size: int | None) -> list[str] | None: +LEADER = "leader" +LEADER_ROLE = "lead the swarm" +LED_PROVER_ROLE = ( + "Contribute to the swarm's goal(s). " + f"Follow the direction of the swarm leader, `{LEADER}`" +) + + +def swarm_member_names( + swarm_size: int | None, leader: bool = False +) -> list[str] | None: """The swarm's member names, or ``None`` for no swarm (``swarm_size`` unset, zero or negative).""" if swarm_size is None or swarm_size <= 0: + if leader: + raise ValueError("swarm_leader requires a positive swarm_size.") + return None + if not leader: + return [f"prover-{i}" for i in range(1, swarm_size + 1)] + if swarm_size < 2: + raise ValueError( + f"swarm_leader requires a swarm_size of at least 2, got {swarm_size}." + ) + return [LEADER] + [f"prover-{i}" for i in range(1, swarm_size)] + + +def swarm_member_role(name: str, leader: bool) -> str | None: + if not leader: return None - return [f"prover-{i}" for i in range(1, swarm_size + 1)] + return LEADER_ROLE if name == LEADER else LED_PROVER_ROLE def build_agent( @@ -233,6 +257,7 @@ def lean_prover( literature: bool, util_module: str, swarm_size: int | None = None, + swarm_leader: bool = False, ) -> Solver: """ Args: @@ -252,7 +277,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: _warn_if_ignored_formalizations(state) sketch = state.metadata["sketch"] - names = swarm_member_names(swarm_size) + names = swarm_member_names(swarm_size, swarm_leader) if names is not None: for name in names: await sandbox(name).write_file(ENTRY_PATH, sketch) @@ -261,6 +286,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: member( _prover, name=name, + role=swarm_member_role(name, swarm_leader), args={"agent_type": agent_type, "gated": gated, "swarm": True}, sandbox=name, visible_sandboxes=[comparator_sandbox_name(name)], diff --git a/apn/task.py b/apn/task.py index 2b21168..13713b6 100644 --- a/apn/task.py +++ b/apn/task.py @@ -10,7 +10,7 @@ from inspect_ai import Task, task from apn import __version__ -from apn.solver import AgentType, lean_prover, swarm_member_names +from apn.solver import LEADER, AgentType, lean_prover, swarm_member_names from apn.checker import SandboxComparator, comparator_sandbox_name from apn.dataset import ( ERDOS_DIR, @@ -220,6 +220,13 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]: return yaml.safe_dump({"services": services}, sort_keys=False) +def _sandbox_config_dirname(members: Sequence[str] | None) -> str: + if members is None: + return "single-agent" + name = f"swarm-{len(members)}" + return f"{name}-{LEADER}" if LEADER in members else name + + def get_sandbox_config( fc_commit: str, literature: bool, @@ -243,7 +250,7 @@ def get_sandbox_config( / _docker_tag_component(__version__) / f"fc_{fc_commit[:12]}" / variant - / ("single-agent" if members is None else f"swarm-{len(members)}") + / _sandbox_config_dirname(members) ) directory.mkdir(parents=True, exist_ok=True) if backend == "docker": @@ -274,6 +281,7 @@ def apn_oeis( literature: bool = False, agent_type: AgentType = "react", swarm_size: int | None = None, + swarm_leader: bool = False, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Formal Conjectures autoformalized OEIS conjectures (492 samples). @@ -284,6 +292,7 @@ def apn_oeis( """ name_list = load_subset(OEIS_DIR, subset) if subset is not None else None pin = fc_commit(OEIS_DIR) + members = swarm_member_names(swarm_size, swarm_leader) return Task( dataset=oeis_dataset(names=name_list), @@ -292,14 +301,11 @@ def apn_oeis( literature=literature, agent_type=agent_type, swarm_size=swarm_size, + swarm_leader=swarm_leader, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer( - SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None - ), - sandbox=get_sandbox_config( - pin, literature, sandbox_backend, swarm_member_names(swarm_size) - ), + scorer=proof_scorer(SandboxComparator(), swarm=members is not None), + sandbox=get_sandbox_config(pin, literature, sandbox_backend, members), ) @@ -310,10 +316,12 @@ def apn_fc100open( literature: bool = False, agent_type: AgentType = "react", swarm_size: int | None = None, + swarm_leader: bool = False, sandbox_backend: SandboxBackend | None = None, ) -> Task: name_list = load_subset(FC100_DIR, subset) if subset is not None else None pin = fc_commit(FC100_DIR) + members = swarm_member_names(swarm_size, swarm_leader) return Task( dataset=fc100open_dataset(names=name_list), solver=lean_prover( @@ -321,14 +329,11 @@ def apn_fc100open( literature=literature, agent_type=agent_type, swarm_size=swarm_size, + swarm_leader=swarm_leader, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer( - SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None - ), - sandbox=get_sandbox_config( - pin, literature, sandbox_backend, swarm_member_names(swarm_size) - ), + scorer=proof_scorer(SandboxComparator(), swarm=members is not None), + sandbox=get_sandbox_config(pin, literature, sandbox_backend, members), ) @@ -339,11 +344,13 @@ def apn_erdos( literature: bool = False, agent_type: AgentType = "react", swarm_size: int | None = None, + swarm_leader: bool = False, sandbox_backend: SandboxBackend | None = None, ) -> Task: """The Bloom statement selection of Erdős problems.""" name_list = load_subset(ERDOS_DIR, subset) if subset is not None else None pin = fc_commit(ERDOS_DIR) + members = swarm_member_names(swarm_size, swarm_leader) return Task( dataset=erdos_dataset(names=name_list), solver=lean_prover( @@ -351,14 +358,11 @@ def apn_erdos( literature=literature, agent_type=agent_type, swarm_size=swarm_size, + swarm_leader=swarm_leader, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer( - SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None - ), - sandbox=get_sandbox_config( - pin, literature, sandbox_backend, swarm_member_names(swarm_size) - ), + scorer=proof_scorer(SandboxComparator(), swarm=members is not None), + sandbox=get_sandbox_config(pin, literature, sandbox_backend, members), ) @@ -369,6 +373,7 @@ def apn_personal_corresp( literature: bool = False, agent_type: AgentType = "react", swarm_size: int | None = None, + swarm_leader: bool = False, sandbox_backend: SandboxBackend | None = None, ) -> Task: """Open conjectures sent to us in personal correspondence, formalized by @@ -377,6 +382,7 @@ def apn_personal_corresp( No predefined subsets; the default runs the full manifest.""" name_list = load_subset(PERSONAL_CORRESP_DIR, subset) if subset is not None else None pin = fc_commit(PERSONAL_CORRESP_DIR) + members = swarm_member_names(swarm_size, swarm_leader) return Task( dataset=personal_corresp_dataset(names=name_list), solver=lean_prover( @@ -384,12 +390,9 @@ def apn_personal_corresp( literature=literature, agent_type=agent_type, swarm_size=swarm_size, + swarm_leader=swarm_leader, util_module=fc_profile(pin).util_module, ), - scorer=proof_scorer( - SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None - ), - sandbox=get_sandbox_config( - pin, literature, sandbox_backend, swarm_member_names(swarm_size) - ), + scorer=proof_scorer(SandboxComparator(), swarm=members is not None), + sandbox=get_sandbox_config(pin, literature, sandbox_backend, members), ) diff --git a/tests/test_agent.py b/tests/test_agent.py index 48ed3b0..721eba4 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -22,10 +22,15 @@ from apn.limits import continue_unless_looping from apn.solver import ( INCORRECT_MESSAGE, + LEADER, + LEADER_ROLE, + LED_PROVER_ROLE, RESOURCE_INCORRECT_MESSAGE, AgentType, build_agent, gated_incorrect_message, + swarm_member_names, + swarm_member_role, ) @@ -112,3 +117,25 @@ def test_react_agent_gets_no_subagents_kwarg(monkeypatch: pytest.MonkeyPatch) -> # react() has no subagents parameter; the deep-only kwarg must not leak. seen = _build(monkeypatch, "react") assert "subagents" not in seen + + +def test_swarm_without_leader_has_numbered_provers_and_no_roles() -> None: + names = swarm_member_names(3) + assert names == ["prover-1", "prover-2", "prover-3"] + assert [swarm_member_role(n, leader=False) for n in names] == [None, None, None] + + +def test_swarm_leader_is_first_of_swarm_size_members() -> None: + names = swarm_member_names(3, leader=True) + assert names == [LEADER, "prover-1", "prover-2"] + assert [swarm_member_role(n, leader=True) for n in names] == [ + LEADER_ROLE, + LED_PROVER_ROLE, + LED_PROVER_ROLE, + ] + + +@pytest.mark.parametrize("swarm_size", [None, 0, 1]) +def test_swarm_leader_needs_another_member(swarm_size: int | None) -> None: + with pytest.raises(ValueError, match="swarm_leader"): + swarm_member_names(swarm_size, leader=True) diff --git a/tests/test_sandbox_backend.py b/tests/test_sandbox_backend.py index 70f6787..eabb782 100644 --- a/tests/test_sandbox_backend.py +++ b/tests/test_sandbox_backend.py @@ -8,6 +8,7 @@ import apn.task from apn.dataset import ERDOS_DIR, fc_commit +from apn.solver import swarm_member_names from apn.task import get_sandbox_config, resolve_sandbox_backend @@ -43,3 +44,19 @@ def test_hawk_default_carries_values_only_settings( for name in ("default", "comparator"): assert "ephemeral-storage" in services[name]["resources"]["requests"] assert services["comparator"]["runtimeClassName"] == "CLUSTER_DEFAULT" + + +def test_leader_swarm_gets_its_own_config_beside_a_plain_swarm() -> None: + pin = fc_commit(ERDOS_DIR) + _, plain = get_sandbox_config(pin, False, "docker", swarm_member_names(3)) + _, led = get_sandbox_config(pin, False, "docker", swarm_member_names(3, leader=True)) + assert plain != led + services = yaml.safe_load(Path(led).read_text())["services"] + assert set(services) == { + "leader", + "comparator-leader", + "prover-1", + "comparator-prover-1", + "prover-2", + "comparator-prover-2", + }