Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions apn/solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,36 @@ def _prover_submit() -> AgentSubmit:
CONTINUE_MESSAGE = "Continue working on the problem."


def swarm_member_names(swarm_size: int | None) -> list[str] | None:
LEADER = "leader"
LEADER_ROLE = "lead the swarm"
LED_PROVER_ROLE = (
"Contribute to the swarm's goal(s). "
f"Follow the direction of the swarm leader, `{LEADER}`"
)


def swarm_member_names(
swarm_size: int | None, leader: bool = False
) -> list[str] | None:
"""The swarm's member names, or ``None`` for no swarm (``swarm_size`` unset,
zero or negative)."""
if swarm_size is None or swarm_size <= 0:
if leader:
raise ValueError("swarm_leader requires a positive swarm_size.")
return None
if not leader:
return [f"prover-{i}" for i in range(1, swarm_size + 1)]
if swarm_size < 2:
raise ValueError(
f"swarm_leader requires a swarm_size of at least 2, got {swarm_size}."
)
return [LEADER] + [f"prover-{i}" for i in range(1, swarm_size)]


def swarm_member_role(name: str, leader: bool) -> str | None:
if not leader:
return None
return [f"prover-{i}" for i in range(1, swarm_size + 1)]
return LEADER_ROLE if name == LEADER else LED_PROVER_ROLE


def build_agent(
Expand Down Expand Up @@ -233,6 +257,7 @@ def lean_prover(
literature: bool,
util_module: str,
swarm_size: int | None = None,
swarm_leader: bool = False,
) -> Solver:
"""
Args:
Expand All @@ -252,7 +277,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
_warn_if_ignored_formalizations(state)
sketch = state.metadata["sketch"]

names = swarm_member_names(swarm_size)
names = swarm_member_names(swarm_size, swarm_leader)
if names is not None:
for name in names:
await sandbox(name).write_file(ENTRY_PATH, sketch)
Expand All @@ -261,6 +286,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
member(
_prover,
name=name,
role=swarm_member_role(name, swarm_leader),
args={"agent_type": agent_type, "gated": gated, "swarm": True},
sandbox=name,
visible_sandboxes=[comparator_sandbox_name(name)],
Expand Down
55 changes: 29 additions & 26 deletions apn/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from inspect_ai import Task, task

from apn import __version__
from apn.solver import AgentType, lean_prover, swarm_member_names
from apn.solver import LEADER, AgentType, lean_prover, swarm_member_names
from apn.checker import SandboxComparator, comparator_sandbox_name
from apn.dataset import (
ERDOS_DIR,
Expand Down Expand Up @@ -220,6 +220,13 @@ def resources(memory_gib: int, ephemeral_storage_gib: int) -> dict[str, Any]:
return yaml.safe_dump({"services": services}, sort_keys=False)


def _sandbox_config_dirname(members: Sequence[str] | None) -> str:
if members is None:
return "single-agent"
name = f"swarm-{len(members)}"
return f"{name}-{LEADER}" if LEADER in members else name


def get_sandbox_config(
fc_commit: str,
literature: bool,
Expand All @@ -243,7 +250,7 @@ def get_sandbox_config(
/ _docker_tag_component(__version__)
/ f"fc_{fc_commit[:12]}"
/ variant
/ ("single-agent" if members is None else f"swarm-{len(members)}")
/ _sandbox_config_dirname(members)
)
directory.mkdir(parents=True, exist_ok=True)
if backend == "docker":
Expand Down Expand Up @@ -274,6 +281,7 @@ def apn_oeis(
literature: bool = False,
agent_type: AgentType = "react",
swarm_size: int | None = None,
swarm_leader: bool = False,
sandbox_backend: SandboxBackend | None = None,
) -> Task:
"""The Formal Conjectures autoformalized OEIS conjectures (492 samples).
Expand All @@ -284,6 +292,7 @@ def apn_oeis(
"""
name_list = load_subset(OEIS_DIR, subset) if subset is not None else None
pin = fc_commit(OEIS_DIR)
members = swarm_member_names(swarm_size, swarm_leader)

return Task(
dataset=oeis_dataset(names=name_list),
Expand All @@ -292,14 +301,11 @@ def apn_oeis(
literature=literature,
agent_type=agent_type,
swarm_size=swarm_size,
swarm_leader=swarm_leader,
util_module=fc_profile(pin).util_module,
),
scorer=proof_scorer(
SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None
),
sandbox=get_sandbox_config(
pin, literature, sandbox_backend, swarm_member_names(swarm_size)
),
scorer=proof_scorer(SandboxComparator(), swarm=members is not None),
sandbox=get_sandbox_config(pin, literature, sandbox_backend, members),
)


Expand All @@ -310,25 +316,24 @@ def apn_fc100open(
literature: bool = False,
agent_type: AgentType = "react",
swarm_size: int | None = None,
swarm_leader: bool = False,
sandbox_backend: SandboxBackend | None = None,
) -> Task:
name_list = load_subset(FC100_DIR, subset) if subset is not None else None
pin = fc_commit(FC100_DIR)
members = swarm_member_names(swarm_size, swarm_leader)
return Task(
dataset=fc100open_dataset(names=name_list),
solver=lean_prover(
gated=gated,
literature=literature,
agent_type=agent_type,
swarm_size=swarm_size,
swarm_leader=swarm_leader,
util_module=fc_profile(pin).util_module,
),
scorer=proof_scorer(
SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None
),
sandbox=get_sandbox_config(
pin, literature, sandbox_backend, swarm_member_names(swarm_size)
),
scorer=proof_scorer(SandboxComparator(), swarm=members is not None),
sandbox=get_sandbox_config(pin, literature, sandbox_backend, members),
)


Expand All @@ -339,26 +344,25 @@ def apn_erdos(
literature: bool = False,
agent_type: AgentType = "react",
swarm_size: int | None = None,
swarm_leader: bool = False,
sandbox_backend: SandboxBackend | None = None,
) -> Task:
"""The Bloom statement selection of Erdős problems."""
name_list = load_subset(ERDOS_DIR, subset) if subset is not None else None
pin = fc_commit(ERDOS_DIR)
members = swarm_member_names(swarm_size, swarm_leader)
return Task(
dataset=erdos_dataset(names=name_list),
solver=lean_prover(
gated=gated,
literature=literature,
agent_type=agent_type,
swarm_size=swarm_size,
swarm_leader=swarm_leader,
util_module=fc_profile(pin).util_module,
),
scorer=proof_scorer(
SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None
),
sandbox=get_sandbox_config(
pin, literature, sandbox_backend, swarm_member_names(swarm_size)
),
scorer=proof_scorer(SandboxComparator(), swarm=members is not None),
sandbox=get_sandbox_config(pin, literature, sandbox_backend, members),
)


Expand All @@ -369,6 +373,7 @@ def apn_personal_corresp(
literature: bool = False,
agent_type: AgentType = "react",
swarm_size: int | None = None,
swarm_leader: bool = False,
sandbox_backend: SandboxBackend | None = None,
) -> Task:
"""Open conjectures sent to us in personal correspondence, formalized by
Expand All @@ -377,19 +382,17 @@ def apn_personal_corresp(
No predefined subsets; the default runs the full manifest."""
name_list = load_subset(PERSONAL_CORRESP_DIR, subset) if subset is not None else None
pin = fc_commit(PERSONAL_CORRESP_DIR)
members = swarm_member_names(swarm_size, swarm_leader)
return Task(
dataset=personal_corresp_dataset(names=name_list),
solver=lean_prover(
gated=gated,
literature=literature,
agent_type=agent_type,
swarm_size=swarm_size,
swarm_leader=swarm_leader,
util_module=fc_profile(pin).util_module,
),
scorer=proof_scorer(
SandboxComparator(), swarm=swarm_member_names(swarm_size) is not None
),
sandbox=get_sandbox_config(
pin, literature, sandbox_backend, swarm_member_names(swarm_size)
),
scorer=proof_scorer(SandboxComparator(), swarm=members is not None),
sandbox=get_sandbox_config(pin, literature, sandbox_backend, members),
)
27 changes: 27 additions & 0 deletions tests/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,15 @@
from apn.limits import continue_unless_looping
from apn.solver import (
INCORRECT_MESSAGE,
LEADER,
LEADER_ROLE,
LED_PROVER_ROLE,
RESOURCE_INCORRECT_MESSAGE,
AgentType,
build_agent,
gated_incorrect_message,
swarm_member_names,
swarm_member_role,
)


Expand Down Expand Up @@ -112,3 +117,25 @@ def test_react_agent_gets_no_subagents_kwarg(monkeypatch: pytest.MonkeyPatch) ->
# react() has no subagents parameter; the deep-only kwarg must not leak.
seen = _build(monkeypatch, "react")
assert "subagents" not in seen


def test_swarm_without_leader_has_numbered_provers_and_no_roles() -> None:
names = swarm_member_names(3)
assert names == ["prover-1", "prover-2", "prover-3"]
assert [swarm_member_role(n, leader=False) for n in names] == [None, None, None]


def test_swarm_leader_is_first_of_swarm_size_members() -> None:
names = swarm_member_names(3, leader=True)
assert names == [LEADER, "prover-1", "prover-2"]
assert [swarm_member_role(n, leader=True) for n in names] == [
LEADER_ROLE,
LED_PROVER_ROLE,
LED_PROVER_ROLE,
]


@pytest.mark.parametrize("swarm_size", [None, 0, 1])
def test_swarm_leader_needs_another_member(swarm_size: int | None) -> None:
with pytest.raises(ValueError, match="swarm_leader"):
swarm_member_names(swarm_size, leader=True)
17 changes: 17 additions & 0 deletions tests/test_sandbox_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import apn.task
from apn.dataset import ERDOS_DIR, fc_commit
from apn.solver import swarm_member_names
from apn.task import get_sandbox_config, resolve_sandbox_backend


Expand Down Expand Up @@ -43,3 +44,19 @@ def test_hawk_default_carries_values_only_settings(
for name in ("default", "comparator"):
assert "ephemeral-storage" in services[name]["resources"]["requests"]
assert services["comparator"]["runtimeClassName"] == "CLUSTER_DEFAULT"


def test_leader_swarm_gets_its_own_config_beside_a_plain_swarm() -> None:
pin = fc_commit(ERDOS_DIR)
_, plain = get_sandbox_config(pin, False, "docker", swarm_member_names(3))
_, led = get_sandbox_config(pin, False, "docker", swarm_member_names(3, leader=True))
assert plain != led
services = yaml.safe_load(Path(led).read_text())["services"]
assert set(services) == {
"leader",
"comparator-leader",
"prover-1",
"comparator-prover-1",
"prover-2",
"comparator-prover-2",
}
Loading