Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/aether-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,5 +76,6 @@ libc = { workspace = true }

[dev-dependencies]
aether-agent-cli = { path = ".", features = ["testing"] }
aether-sessions = { workspace = true, features = ["testing"] }
tokio = { workspace = true, features = ["test-util"] }
acp_utils = { package = "aether-acp-utils", path = "../acp-utils", features = ["testing"], version = "0.5.13" }
84 changes: 17 additions & 67 deletions crates/aether-cli/src/acp/testing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,23 @@ use acp_utils::testing::{TestPeer, initialize_request};
use aether_auth::OAuthCredentialStorage;
use aether_core::agent_spec::{AgentSpec, AgentSpecExposure};
use aether_core::core::{AgentBuilder, AgentHandle, Prompt};
use aether_core::events::{AgentEvent, Command, MessageEvent, ToolEvent, TurnEvent, TurnOutcome, UserCommand};
use aether_core::events::{AgentEvent, Command, TurnOutcome, UserCommand};
use aether_core::mcp::{McpBuilder, McpSession, ServerFactory, mcp};
use aether_project::AgentCatalog;
use aether_sessions::SessionStore;
use aether_sessions::{SessionControlEvent, SessionEvent, SessionMeta, UserEvent, last_agent_from_events};
use aether_sessions::testing::{
agent_switched, assistant_text, tool_call, tool_result, turn_ended, user_message, user_message_with,
};
use aether_sessions::{SessionEvent, SessionMeta, last_agent_from_events};
use agent_client_protocol::schema::v2::{
AbsolutePath, InitializeResponse, ReplayFrom, ReplayFromStart, ResumeSessionRequest, SessionId, SessionUpdate,
StateUpdate, StopReason,
};
use agent_client_protocol::{Agent, Channel, Client, ConnectionTo, on_receive_notification};
use futures::FutureExt;
use llm::testing::FakeLlmProvider;
use llm::{ChatMessage, Context, LlmModel, LlmResponse, SessionUsageEvent, StreamingModelProvider};
use llm::{MessageId, ProviderConnectionOverrides, ToolCallRequest, ToolCallResult};
use llm::ProviderConnectionOverrides;
use llm::testing::{FakeLlmProvider, llm_response};
use llm::{ChatMessage, Context, LlmModel, SessionUsageEvent, StreamingModelProvider};
use mcp_utils::client::{InMemoryServerSpec, McpServer, McpTransport, ToolExposure};
use mcp_utils::testing::{FakeMcpServer, FakeTool, FakeToolResponse, completed_task_payload};
use rmcp::RoleServer;
Expand Down Expand Up @@ -387,13 +390,7 @@ impl AcpTestHarness {
}

pub fn append_agent_switch(&self, session_id: &str, from: Option<&str>, to: Option<&str>) {
self.append_stored_event(
session_id,
&SessionEvent::Control(SessionControlEvent::AgentSwitched {
from: from.map(str::to_string),
to: to.map(str::to_string),
}),
);
self.append_stored_event(session_id, &agent_switched(from, to));
}

/// Register a stub session built from a hand-spawned
Expand Down Expand Up @@ -455,72 +452,28 @@ impl AcpTestHarness {
}

pub fn append_stored_prompt(&self, session_id: &str, prompt: &str) {
self.append_stored_event(
session_id,
&SessionEvent::User(UserEvent::Message {
message_id: llm::MessageId::new(),
content: vec![llm::ContentBlock::text(prompt)],
display_content: None,
}),
);
self.append_stored_event(session_id, &user_message(prompt));
}

pub fn append_stored_user_blocks(&self, session_id: &str, blocks: Vec<llm::ContentBlock>) {
self.append_stored_event(
session_id,
&SessionEvent::User(UserEvent::Message {
message_id: llm::MessageId::new(),
content: blocks,
display_content: None,
}),
);
self.append_stored_event(session_id, &user_message_with(blocks));
}

pub fn append_stored_agent_turn(&self, session_id: &str, text: &str) {
self.append_stored_agent_text(session_id, text);
self.append_stored_event(
session_id,
&SessionEvent::Agent(AgentEvent::Turn(TurnEvent::Ended { outcome: TurnOutcome::Completed })),
);
self.append_stored_event(session_id, &turn_ended(TurnOutcome::Completed));
}

pub fn append_stored_agent_text(&self, session_id: &str, text: &str) {
self.append_stored_event(
session_id,
&SessionEvent::Agent(AgentEvent::Message(MessageEvent::Text {
message_id: MessageId::new(),
chunk: text.to_string(),
is_complete: true,
})),
);
self.append_stored_event(session_id, &assistant_text(llm::MessageId::new().as_str(), text));
}

pub fn append_stored_tool_call(&self, session_id: &str, id: &str, name: &str, arguments: &str) {
self.append_stored_event(
session_id,
&SessionEvent::Agent(AgentEvent::Tool(ToolEvent::Call {
request: ToolCallRequest {
id: id.to_string(),
name: name.to_string(),
arguments: arguments.to_string(),
},
})),
);
self.append_stored_event(session_id, &tool_call(id, name, arguments));
}

pub fn append_stored_tool_result(&self, session_id: &str, id: &str, name: &str, result: &str) {
self.append_stored_event(
session_id,
&SessionEvent::Agent(AgentEvent::Tool(ToolEvent::Result {
result: ToolCallResult {
id: id.to_string(),
name: name.to_string(),
arguments: "{}".to_string(),
result: result.to_string(),
},
result_meta: None,
})),
);
self.append_stored_event(session_id, &tool_result(id, name, result));
}

async fn insert_switching_session(
Expand Down Expand Up @@ -905,11 +858,8 @@ fn background_task(status: TaskStatus) -> Task {

fn fake_agent(name: &str, server_name: &str, prompt_name: &str, reply: &str) -> (FakeAgentDef, FakeAcpAgent) {
const TURNS_BEFORE_AND_AFTER_REATTACH: usize = 2;
let provider = FakeLlmProvider::new(vec![
vec![LlmResponse::Start, LlmResponse::text(reply), LlmResponse::done()];
TURNS_BEFORE_AND_AFTER_REATTACH
])
.with_display_name(name);
let provider = FakeLlmProvider::new(vec![llm_response().text(&[reply]).build(); TURNS_BEFORE_AND_AFTER_REATTACH])
.with_display_name(name);
let captured_contexts = provider.captured_contexts();
let def = FakeAgentDef {
spec: fake_agent_spec(name),
Expand Down
28 changes: 23 additions & 5 deletions crates/aether-core/src/events/agent_event.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use llm::{ToolCallError, ToolCallRequest, ToolCallResult};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

Expand Down Expand Up @@ -37,6 +38,25 @@ impl AgentEvent {
Self::Turn(TurnEvent::Ended { outcome })
}

pub fn tool_call(id: &str, name: &str, arguments: &str) -> Self {
Self::Tool(ToolEvent::Call { request: ToolCallRequest::new(id, name, arguments) })
}

pub fn tool_result(id: &str, name: &str, arguments: &str, result: &str) -> Self {
Self::Tool(ToolEvent::Result { result: ToolCallResult::new(id, name, arguments, result), result_meta: None })
}

pub fn tool_error(id: &str, name: &str, arguments: &str, error: &str) -> Self {
Self::Tool(ToolEvent::Error {
error: ToolCallError {
id: id.into(),
name: name.into(),
arguments: Some(arguments.into()),
error: error.into(),
},
})
}

/// Human-readable text content of this event, if any.
pub fn content(&self) -> Option<String> {
match self {
Expand Down Expand Up @@ -111,11 +131,11 @@ mod tests {
AgentEvent::Model(ModelEvent::Switched { previous: "a".into(), new: "b".into() }),
AgentEvent::SessionUsage(llm::testing::session_usage_event(1, llm::TokenUsage::new(1, 2))),
AgentEvent::Tool(ToolEvent::DisplayUpdate {
request: llm::ToolCallRequest { id: "call".into(), name: "read".into(), arguments: "{}".into() },
request: llm::ToolCallRequest::new("call", "read", "{}"),
meta: ToolDisplayMeta::new("Read file", "main.rs").into(),
}),
AgentEvent::Tool(ToolEvent::SubAgentProgress {
request: llm::ToolCallRequest { id: "call".into(), name: "spawn".into(), arguments: "{}".into() },
request: llm::ToolCallRequest::new("call", "spawn", "{}"),
payload: Box::new(crate::events::SubAgentProgressPayload {
task_id: "task_0".into(),
agent_name: "explorer".into(),
Expand Down Expand Up @@ -218,9 +238,7 @@ mod tests {
AgentEvent::text("m", "partial", StreamState::Complete),
AgentEvent::thought("m", "partial", StreamState::Complete),
AgentEvent::Tool(ToolEvent::InputStarted { id: "call".into(), name: "read".into() }),
AgentEvent::Tool(ToolEvent::Call {
request: llm::ToolCallRequest { id: "call".into(), name: "read".into(), arguments: "{}".into() },
}),
AgentEvent::tool_call("call", "read", "{}"),
AgentEvent::turn_ended(TurnOutcome::Completed),
];
assert!(deltas.iter().all(AgentEvent::is_stream_delta));
Expand Down
8 changes: 1 addition & 7 deletions crates/aether-core/tests/agent/agent_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,7 @@ async fn streamed_tool_input_is_followed_by_the_complete_call() -> Result<(), Bo
&ToolEvent::InputStarted { id: "call_1".into(), name: "test__add_numbers".into() },
&ToolEvent::InputDelta { id: "call_1".into(), chunk: arg_chunk_1.into() },
&ToolEvent::InputDelta { id: "call_1".into(), chunk: arg_chunk_2.into() },
&ToolEvent::Call {
request: llm::ToolCallRequest {
id: "call_1".into(),
name: "test__add_numbers".into(),
arguments: request_json.clone(),
},
},
&ToolEvent::Call { request: llm::ToolCallRequest::new("call_1", "test__add_numbers", &request_json) },
]
);

Expand Down
34 changes: 8 additions & 26 deletions crates/aether-evals/src/agents/transcript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ mod tests {
use super::*;
use crate::{Agent, FakeAgent, Task};
use aether_core::events::TurnEvent;
use llm::TokenUsage;
use llm::testing::session_usage_event;
use llm::{TokenUsage, ToolCallRequest, ToolCallResult};

#[tokio::test]
async fn transcript_from_stream() {
Expand All @@ -154,7 +154,11 @@ mod tests {

#[test]
fn tool_call_count_counts_matching_tool_calls() {
let transcript = transcript_with_events(vec![tool_call("bash"), tool_call("read"), tool_result("bash")]);
let transcript = Transcript::new(vec![
AgentEvent::tool_call("call_1", "bash", "{}"),
AgentEvent::tool_call("call_2", "read", "{}"),
AgentEvent::tool_result("call_3", "bash", "{}", "ok"),
]);

assert!(transcript.tool_called("bash"));
assert!(!transcript.tool_called("read"));
Expand All @@ -179,7 +183,7 @@ mod tests {

#[test]
fn usage_returns_zeroed_totals_when_no_usage_was_recorded() {
let transcript = transcript_with_events(vec![tool_call("bash")]);
let transcript = Transcript::new(vec![AgentEvent::tool_call("call_1", "bash", "{}")]);
assert_eq!(transcript.usage(), SessionUsageTotals::default());
}

Expand All @@ -188,7 +192,7 @@ mod tests {
let mut last = session_usage_event(2, TokenUsage::new(2000, 500));
last.totals.tokens = TokenUsage::new(3000, 600);
last.totals.unpriced_calls = 2;
let transcript = transcript_with_events(vec![
let transcript = Transcript::new(vec![
AgentEvent::SessionUsage(session_usage_event(1, TokenUsage::new(1000, 100))),
AgentEvent::SessionUsage(last),
]);
Expand All @@ -200,26 +204,4 @@ mod tests {
assert_eq!(usage.unpriced_calls, 2);
assert!(!usage.is_fully_priced());
}

fn transcript_with_events(events: Vec<AgentEvent>) -> Transcript {
Transcript::new(events)
}

fn tool_call(name: &str) -> AgentEvent {
AgentEvent::Tool(ToolEvent::Call {
request: ToolCallRequest { id: name.to_string(), name: name.to_string(), arguments: "{}".to_string() },
})
}

fn tool_result(name: &str) -> AgentEvent {
AgentEvent::Tool(ToolEvent::Result {
result: ToolCallResult {
id: name.to_string(),
name: name.to_string(),
arguments: "{}".to_string(),
result: "ok".to_string(),
},
result_meta: None,
})
}
}
25 changes: 4 additions & 21 deletions crates/aether-evals/src/assertions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,45 +44,28 @@ pub fn assert_tool_call_with_args(transcript: &Transcript, name: &str, expected:
#[cfg(test)]
mod tests {
use super::*;
use aether_core::events::{AgentEvent, ToolEvent};
use llm::ToolCallResult;
use aether_core::events::AgentEvent;

#[test]
fn assert_tool_call_with_args_accepts_matching_json_arguments() {
let trace = transcript_with_messages(vec![tool_result("bash", r#"{"command":"pwd"}"#)]);
let trace = Transcript::new(vec![AgentEvent::tool_result("call_1", "bash", r#"{"command":"pwd"}"#, "ok")]);

assert_tool_call_with_args(&trace, "bash", &serde_json::json!({ "command": "pwd" }));
}

#[test]
#[should_panic(expected = "Non-JSON arguments seen for this tool")]
fn assert_tool_call_with_args_surfaces_parse_failures() {
let trace = transcript_with_messages(vec![tool_result("bash", "not json")]);
let trace = Transcript::new(vec![AgentEvent::tool_result("call_1", "bash", "not json", "ok")]);

assert_tool_call_with_args(&trace, "bash", &serde_json::json!({ "command": "pwd" }));
}

#[test]
#[should_panic(expected = "expected tool `missing` to be called")]
fn assert_tool_called_panics_when_tool_was_not_called() {
let trace = transcript_with_messages(vec![]);
let trace = Transcript::new(vec![]);

assert_tool_called(&trace, "missing");
}

fn transcript_with_messages(messages: Vec<AgentEvent>) -> Transcript {
Transcript::new(messages)
}

fn tool_result(name: &str, arguments: &str) -> AgentEvent {
AgentEvent::Tool(ToolEvent::Result {
result: ToolCallResult {
id: name.to_string(),
name: name.to_string(),
arguments: arguments.to_string(),
result: "ok".to_string(),
},
result_meta: None,
})
}
}
Loading
Loading