diff --git a/crates/tinyagents-graph/src/agent_loop/driver.rs b/crates/tinyagents-graph/src/agent_loop/driver.rs index 90e47e207..374983571 100644 --- a/crates/tinyagents-graph/src/agent_loop/driver.rs +++ b/crates/tinyagents-graph/src/agent_loop/driver.rs @@ -20,6 +20,7 @@ use tinyagents_harness::ids::HarnessPhase; use tinyagents_harness::middleware::AgentRun; use tinyagents_harness::runtime::AgentHarness; use tinyagents_harness::steering::PauseState; +use tinyagents_harness::terminal::{TerminalOutcome, TerminalReason}; use tinyinference_llm::message::Message; use crate::command::{NodeResult, RouteTarget}; @@ -81,7 +82,10 @@ where messages: input, ..LoopState::default() }; + ctx.reset_turn_tracker(loop_state.messages.len()); let mut current: &str = node::PLAN; + let mut limit_stop = false; + let mut limit_kind = None; let outcome = loop { // Keeps `run.messages` a running snapshot of the transcript as @@ -165,6 +169,9 @@ where )); } }; + limit_stop |= loop_state.limit_stop; + // Latch the first kind: a later command must not erase it. + limit_kind = limit_kind.or(loop_state.limit_kind); let Some(target) = command.goto.first() else { break Err(TinyAgentsError::Validation( "GraphLoopDriver: loop node's command carried no route".to_string(), @@ -197,11 +204,81 @@ where } }; + let terminal = match &outcome { + Ok(None) => { + let reason = if limit_stop { + TerminalOutcome::limit_reached( + limit_kind, + format!( + "stopped with the partial run: {} limit reached", + limit_kind.map_or("run", |kind| kind.as_str()) + ), + ) + } else { + TerminalOutcome::completed() + }; + Some(reason.with_provider_started(ctx.provider_started())) + } + Ok(Some(interrupt)) => { + let reason = interrupt + .payload + .get("reason") + .or_else(|| interrupt.payload.get("message")) + .and_then(|value| value.as_str()) + .map(str::to_string); + let outcome = if let Some(summary) = ctx.take_halted_by_guard() { + TerminalOutcome::halted(summary) + } else { + TerminalOutcome::new( + TerminalReason::Paused, + reason + .clone() + .unwrap_or_else(|| format!("paused at node `{}`", interrupt.node)), + ) + }; + Some(outcome.with_provider_started(ctx.provider_started())) + } + Err(error) => { + use tinyagents_harness::terminal::TimeoutPhase; + let in_model_call = ctx.model_call_failed() || ctx.active_model_call.is_some(); + let site = if in_model_call && ctx.call_provider_started() { + TimeoutPhase::Provider + } else if in_model_call { + TimeoutPhase::BeforeProvider + } else if ctx.provider_started() { + TimeoutPhase::AfterTurn + } else { + TimeoutPhase::BeforeProvider + }; + // A `LimitExceeded` carries only text; the cap that tripped was + // announced by a `LimitReached` event just before. + let kind = matches!(error, TinyAgentsError::LimitExceeded(_)) + .then(|| ctx.peek_last_limit()) + .flatten(); + let mut outcome = TerminalOutcome::from_error(error, site).with_limit_kind(kind); + // A failed summarizer already received a provider response, though + // summarizer calls bypass the context's dispatch marker. + outcome.provider_started = ctx.provider_started() + || matches!(error, TinyAgentsError::SummarizationUsage { .. }); + Some(outcome) + } + }; + run.terminal = terminal.clone(); status.mark_running(HarnessPhase::Middleware); - harness - .middleware() - .run_after_agent(ctx, state, run) - .await?; + let after_agent = harness.middleware().run_after_agent(ctx, state, run).await; + if let Err(hook_error) = after_agent { + if outcome.is_err() { + // The originating node failure stays authoritative, as in the + // direct loop; the hook still ran for its cleanup. + tracing::warn!( + target: "tinyagents::agent_loop", + error = %hook_error, + "[agent_loop] after_agent failed after a node error; keeping the node error" + ); + } else { + return Err(hook_error); + } + } // `status.mark_completed`/`mark_interrupted`/`mark_failed` and (on // error) `AgentEvent::RunFailed` are applied centrally by @@ -214,8 +291,15 @@ where // interrupt. match outcome { Ok(None) => { + // `after_agent` may have replaced the outcome; report the final one. + let outcome = run + .terminal + .clone() + .or(terminal) + .expect("terminal set before after_agent"); let record = ctx.emit(AgentEvent::RunCompleted { run_id: ctx.run_id().clone(), + outcome: Some(outcome), }); status.set_last_event(record.id); Ok(()) @@ -234,10 +318,15 @@ where .unwrap_or_else(|| format!("paused at node `{}`", interrupt.node)), }); status.set_last_event(record.id); - run.paused = Some(PauseState { - reason, - paused_at_checkpoint: 0, - }); + if !matches!( + run.terminal.as_ref().map(|outcome| outcome.reason), + Some(TerminalReason::Halted) + ) { + run.paused = Some(PauseState { + reason, + paused_at_checkpoint: 0, + }); + } Ok(()) } Err(error) => Err(error), diff --git a/crates/tinyagents-graph/src/agent_loop/runtime.rs b/crates/tinyagents-graph/src/agent_loop/runtime.rs index 7b33b32ab..30543ff4a 100644 --- a/crates/tinyagents-graph/src/agent_loop/runtime.rs +++ b/crates/tinyagents-graph/src/agent_loop/runtime.rs @@ -116,11 +116,13 @@ impl ModelBaseCall { fn call<'a>( &'a self, - _ctx: &'a mut RunContext, + ctx: &'a mut RunContext, state: &'a State, request: ModelRequest, ) -> BoxModelFuture<'a> { Box::pin(async move { + // Reached only when the wrap onion elected to call the provider. + ctx.mark_provider_started(); self.model .invoke(state, request) .await @@ -304,6 +306,8 @@ where tinyagents_harness::limits::LimitBehavior::StopWithPartial ) { loop_state.finished = true; + loop_state.limit_stop = true; + loop_state.limit_kind = Some(tinyagents_harness::events::LimitKind::ModelCalls); if loop_state.final_text.is_none() { loop_state.final_text = Some(last_assistant_text(&loop_state.messages)); } @@ -425,15 +429,27 @@ where model: model_name.clone(), }); status.set_last_event(started_record.id); + status.active_model_call = Some(call_id.clone()); + ctx.active_model_call = Some(call_id.clone()); + ctx.begin_model_call(); let base = DirectModelBase { model: binding.model.as_ref(), }; - let (mut response, wrap_control) = harness + let wrapped = match harness .middleware() .run_wrapped_model(ctx, app_state, request, &base) - .await? - .into_response_with_control(); + .await + { + Ok(wrapped) => wrapped, + Err(error) => { + status.active_model_call = None; + ctx.active_model_call = None; + ctx.mark_model_call_failed(); + return Err(error); + } + }; + let (mut response, wrap_control) = wrapped.into_response_with_control(); if let Some(control) = wrap_control { ctx.request_control(control); } @@ -441,6 +457,8 @@ where run.model_calls += 1; run.steps += 1; status.model_calls = run.model_calls; + status.active_model_call = None; + ctx.active_model_call = None; if let Some(usage) = response.usage { run.usage.record(usage); loop_state.usage = run.usage; @@ -530,7 +548,30 @@ where &mut loop_state.messages, calls, ) - .await?; + .await; + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) + if matches!( + harness.policy().limits.behavior, + tinyagents_harness::limits::LimitBehavior::StopWithPartial + ) && matches!(error, TinyAgentsError::LimitExceeded(_)) + && ctx.peek_last_limit() + == Some(tinyagents_harness::events::LimitKind::ToolCalls) => + { + // Only the tool-cap admission announces `LimitReached(ToolCalls)` + // (and `record_tool_call` cleared any earlier kind first), so a + // `LimitExceeded` raised by middleware is not a partial stop. + loop_state.limit_stop = true; + loop_state.limit_kind = Some(tinyagents_harness::events::LimitKind::ToolCalls); + loop_state.finished = true; + if loop_state.final_text.is_none() { + loop_state.final_text = Some(last_assistant_text(&loop_state.messages)); + } + return Ok(goto(loop_state, node::SETTLE)); + } + Err(error) => return Err(error), + }; loop_state.tool_calls = run.tool_calls; loop_state.executed_tools = run.executed_tools.clone(); let _ = outcome; diff --git a/crates/tinyagents-graph/src/agent_loop/types.rs b/crates/tinyagents-graph/src/agent_loop/types.rs index 4d3f944d2..6d6c67f5d 100644 --- a/crates/tinyagents-graph/src/agent_loop/types.rs +++ b/crates/tinyagents-graph/src/agent_loop/types.rs @@ -69,6 +69,14 @@ pub struct LoopState { /// How many output-validation retries have been spent so far (bounds /// [`tinyagents_harness::runtime::RunPolicy::output_retry`]). pub(crate) output_retry_attempts: u8, + /// Set when the run finished because a call cap tripped under + /// `LimitBehavior::StopWithPartial`, so the driver can report + /// `TerminalReason::LimitReached` instead of a plain completion. + #[serde(default)] + pub(crate) limit_stop: bool, + /// Which cap tripped when `limit_stop` is set (`None` when unknown). + #[serde(default)] + pub(crate) limit_kind: Option, } impl LoopState { diff --git a/crates/tinyagents-harness/src/agent_loop/entry.rs b/crates/tinyagents-harness/src/agent_loop/entry.rs index eac8a7dd1..2ca39301e 100644 --- a/crates/tinyagents-harness/src/agent_loop/entry.rs +++ b/crates/tinyagents-harness/src/agent_loop/entry.rs @@ -441,15 +441,32 @@ impl AgentHarness { // `completed` is what made "paused for a human" look identical // to "the model produced an empty final answer". // A deferred run (A2) is resumable for the same reason. - let paused = terminal.run.paused.is_some() || terminal.run.deferred.is_some(); + // The typed outcome is authoritative (middleware may have + // replaced it after the loop set the legacy fields); fall back + // to the legacy fields only when no outcome was recorded. + let paused = match terminal.run.terminal.as_ref() { + Some(outcome) => outcome.class == TerminalClass::Suspended, + None => terminal.run.paused.is_some() || terminal.run.deferred.is_some(), + }; + // Only `Success` is a completion and `Suspended` is handled by + // `paused`; Failure, Timeout and Cancellation outcomes recorded + // by middleware on an `Ok` return must not read as completed. + let failed = terminal.run.terminal.as_ref().is_some_and(|outcome| { + !matches!( + outcome.class, + TerminalClass::Success | TerminalClass::Suspended + ) + }); if paused { status.mark_interrupted(); + } else if failed { + status.mark_failed("run ended with a non-success terminal outcome".to_string()); } else { status.mark_completed(); } PartialRunOutcome { run: terminal.complete( - !paused, + !paused && !failed, paused.then(|| "hosted turn paused before completion".to_string()), ), status, @@ -457,9 +474,45 @@ impl AgentHarness { } } Err(error) => { + // Where the run stood when it failed decides the timeout phase + // and `provider_started`: an unfinished model call leaves + // `active_model_call` set, a completed one has bumped the + // run's call counter. + // A failure inside the model-call layer is `Provider` only if + // the provider was actually dispatched; a wrap middleware that + // rejected the call first never reached it. + let in_model_call = ctx.active_model_call.is_some() || ctx.model_call_failed(); + let site = if in_model_call && ctx.call_provider_started() { + TimeoutPhase::Provider + } else if in_model_call { + TimeoutPhase::BeforeProvider + } else if terminal.run.model_calls > 0 { + TimeoutPhase::AfterTurn + } else { + TimeoutPhase::BeforeProvider + }; + let last_limit = matches!(error, TinyAgentsError::LimitExceeded(_)) + .then(|| ctx.take_last_limit()) + .flatten(); + let mut outcome = + TerminalOutcome::from_error(&error, site).with_limit_kind(last_limit); + if outcome.reason == crate::terminal::TerminalReason::Timeout + && outcome.timeout_phase.is_none() + { + // A wall-clock kind filled in after classification. + outcome = outcome.with_timeout_phase(site); + } + // `site` describes this failure; the run may still have reached + // the provider on an earlier call. + // A failed summarizer already received a provider response, though + // summarizer calls bypass the context's dispatch marker. + outcome.provider_started = ctx.provider_started() + || matches!(&error, TinyAgentsError::SummarizationUsage { .. }); + terminal.run.terminal = Some(outcome.clone()); let record = ctx.emit(AgentEvent::RunFailed { run_id, error: error.to_string(), + outcome: Some(outcome), }); status.set_last_event(record.id); status.mark_failed(error.to_string()); diff --git a/crates/tinyagents-harness/src/agent_loop/lifecycle.rs b/crates/tinyagents-harness/src/agent_loop/lifecycle.rs new file mode 100644 index 000000000..0d1df4db3 --- /dev/null +++ b/crates/tinyagents-harness/src/agent_loop/lifecycle.rs @@ -0,0 +1,209 @@ +//! Turn and message lifecycle events for the superstep loop. +//! +//! The loop pushes to its working transcript from many places (assistant +//! replies, tool results, recovery nudges, steering, queued messages). Rather +//! than instrument each push, [`TurnTracker`] watches the transcript length and +//! announces whatever was appended since the last look, in order, at the points +//! the loop already treats as boundaries. That keeps every present and future +//! push site covered with no per-site code. Mutations that are *not* appends +//! must say so explicitly: [`TurnTracker::retract_to`] for pops and +//! [`TurnTracker::rebase`] for in-place rewrites. + +use crate::context::RunContext; +use crate::events::{AgentEvent, EventSink}; +use crate::ids::CallId; +use crate::runtime::PayloadCapture; +use tinyinference_llm::message::Message; + +/// Tracks which transcript messages have been announced and which turn is open. +/// +/// Lives on the [`RunContext`](crate::context::RunContext) so every site that +/// mutates the transcript can reach it. +#[derive(Debug, Default)] +pub(crate) struct TurnTracker { + /// Number of initial input messages that were never announced. + seed_len: usize, + /// Messages `[0, announced)` have been announced (or are the seed input). + announced: usize, + /// Number of the most recently opened turn. + turn: u32, + /// The open turn and the transcript index it started at. + open: Option<(u32, usize)>, +} + +pub(crate) fn role_of(message: &Message) -> &'static str { + match message { + Message::System(_) => "system", + Message::User(_) => "user", + Message::Assistant(_) => "assistant", + Message::Tool(_) => "tool", + Message::Custom(_) => "custom", + } +} + +impl TurnTracker { + /// A tracker for a transcript that starts with `seed_len` input messages, + /// which are not announced. + pub(crate) fn new(seed_len: usize) -> Self { + Self { + announced: seed_len, + seed_len, + turn: 0, + open: None, + } + } + + /// Announces every message appended since the last call, in order. + pub(crate) fn flush( + &mut self, + events: &EventSink, + capture: PayloadCapture, + messages: &[Message], + ) { + // A shrunk transcript must have been reported through `retract_to` or + // `rebase`; clamp defensively rather than index out of range. + if messages.len() < self.announced { + tracing::warn!( + target: "tinyagents::agent_loop", + announced = self.announced, + len = messages.len(), + "[agent_loop] transcript shrank without retract_to/rebase; re-basing the lifecycle cursor" + ); + self.announced = messages.len(); + } + for (index, message) in messages.iter().enumerate().skip(self.announced) { + let (call_id, captured) = match message { + Message::Tool(tool) => ( + Some(CallId::new(tool.tool_call_id.clone())), + capture.tool_io, + ), + _ => (None, capture.model_io), + }; + events.emit(AgentEvent::MessageAppended { + role: role_of(message).to_string(), + index, + call_id, + message: captured.then(|| to_value_logged(message)), + }); + } + self.announced = messages.len(); + } + + /// Reports that the transcript was truncated to `new_len` messages (a pop). + /// Emits [`AgentEvent::MessageRetracted`] for each *announced* message + /// removed, highest index first; removing a message that was never + /// announced is silent. + pub(crate) fn retract_to(&mut self, events: &EventSink, new_len: usize) { + for index in (new_len.max(self.seed_len)..self.announced).rev() { + events.emit(AgentEvent::MessageRetracted { index }); + } + self.announced = self.announced.min(new_len.max(self.seed_len)); + if let Some((_, start)) = self.open.as_mut() { + *start = (*start).min(new_len); + } + } + + /// Reports that the transcript was rewritten in place and now holds + /// `new_len` messages. Call [`Self::flush`] *before* the mutation so + /// pending appends are announced against the old transcript. + pub(crate) fn rebase(&mut self, events: &EventSink, new_len: usize, reason: &str) { + events.emit(AgentEvent::TranscriptRewritten { + len: new_len, + reason: reason.to_string(), + }); + self.announced = new_len; + if let Some((_, start)) = self.open.as_mut() { + *start = (*start).min(new_len); + } + } + + /// Opens the next turn, first announcing pending messages and closing any + /// turn still open (a recovery retry re-enters the model call without + /// finishing its predecessor). Returns the new turn number. + pub(crate) fn start_turn( + &mut self, + events: &EventSink, + capture: PayloadCapture, + messages: &[Message], + ) -> u32 { + self.close_turn(events, capture, messages); + self.turn += 1; + self.open = Some((self.turn, messages.len())); + events.emit(AgentEvent::TurnStarted { turn: self.turn }); + self.turn + } + + /// Announces pending messages and closes the open turn, if any, reporting + /// the tool results it added to the transcript. + pub(crate) fn close_turn( + &mut self, + events: &EventSink, + capture: PayloadCapture, + messages: &[Message], + ) { + self.flush(events, capture, messages); + let Some((turn, start)) = self.open.take() else { + return; + }; + let tool_call_ids: Vec = messages + .get(start..) + .unwrap_or_default() + .iter() + .filter_map(|message| match message { + Message::Tool(tool) => Some(CallId::new(tool.tool_call_id.clone())), + _ => None, + }) + .collect(); + tracing::debug!( + target: "tinyagents::agent_loop", + turn, + tool_results = tool_call_ids.len(), + "[agent_loop] turn completed" + ); + events.emit(AgentEvent::TurnCompleted { + turn, + tool_result_count: tool_call_ids.len(), + tool_call_ids, + }); + } +} + +/// Serializes a value for an event payload, logging instead of silently +/// substituting `null` when serialization fails. +pub(crate) fn to_value_logged(value: &T) -> serde_json::Value { + serde_json::to_value(value).unwrap_or_else(|error| { + tracing::warn!( + target: "tinyagents::agent_loop", + %error, + "[agent_loop] could not serialize an event payload; emitting null" + ); + serde_json::Value::Null + }) +} + +impl RunContext { + /// Announces transcript appends not yet announced. + pub(crate) fn flush_transcript(&mut self, capture: PayloadCapture, messages: &[Message]) { + self.turns.flush(&self.events, capture, messages); + } + + /// Opens the next turn (see [`TurnTracker::start_turn`]). + pub(crate) fn start_turn(&mut self, capture: PayloadCapture, messages: &[Message]) -> u32 { + self.turns.start_turn(&self.events, capture, messages) + } + + /// Closes the open turn (see [`TurnTracker::close_turn`]). + pub(crate) fn close_turn(&mut self, capture: PayloadCapture, messages: &[Message]) { + self.turns.close_turn(&self.events, capture, messages); + } + + /// Reports a pop: the transcript now holds `new_len` messages. + pub(crate) fn retract_transcript(&mut self, new_len: usize) { + self.turns.retract_to(&self.events, new_len); + } + + /// Reports an in-place rewrite: the transcript now holds `new_len` messages. + pub(crate) fn rebase_transcript(&mut self, new_len: usize, reason: &str) { + self.turns.rebase(&self.events, new_len, reason); + } +} diff --git a/crates/tinyagents-harness/src/agent_loop/lifecycle_tests.rs b/crates/tinyagents-harness/src/agent_loop/lifecycle_tests.rs new file mode 100644 index 000000000..e51f187d0 --- /dev/null +++ b/crates/tinyagents-harness/src/agent_loop/lifecycle_tests.rs @@ -0,0 +1,585 @@ +//! Turn and message lifecycle events (`TurnStarted`, `TurnCompleted`, +//! `MessageAppended`) and the message-carrying `QueuedMessageApplied`. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::json; + +use crate::context::{RunConfig, RunContext}; +use crate::events::AgentEvent; +use crate::ids::CallId; +use crate::run_queue::{QueueLane, RunQueue}; +use crate::runtime::{AgentHarness, PayloadCapture, RunPolicy}; +use crate::testkit::{EventRecorder, ScriptedModel}; +use tinyinference_llm::message::{AssistantMessage, ContentBlock, Message}; +use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse}; +use tinyinference_llm::tool::ToolCall; +use tinyinference_llm::usage::Usage; +use tinytools::{Tool, ToolResult}; + +struct EchoTool; + +#[async_trait] +impl Tool for EchoTool { + fn name(&self) -> &str { + "echo" + } + fn description(&self) -> &str { + "echo" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type": "object"}) + } + async fn execute(&self, _: serde_json::Value) -> anyhow::Result { + Ok(ToolResult::success("echoed")) + } +} + +struct FailingModel; + +#[async_trait] +impl ChatModel<()> for FailingModel { + async fn invoke(&self, _: &(), _: ModelRequest) -> tinyinference_llm::Result { + Err(tinyinference_llm::Error::Model("boom".into())) + } +} + +fn response(tool_calls: Vec, text: &str) -> ModelResponse { + ModelResponse { + message: AssistantMessage { + id: None, + content: if text.is_empty() { + Vec::new() + } else { + vec![ContentBlock::Text(text.to_string())] + }, + tool_calls, + usage: Some(Usage::new(1, 1)), + origin: None, + }, + usage: Some(Usage::new(1, 1)), + finish_reason: Some("stop".to_string()), + raw: None, + resolved_model: None, + continue_turn: None, + served_from_cache: false, + correlation: None, + resolved_route: None, + } +} + +fn tool_turn(ids: &[&str]) -> ModelResponse { + response( + ids.iter() + .map(|id| ToolCall::new(*id, "echo", json!({}))) + .collect(), + "", + ) +} + +fn harness(responses: Vec, capture: PayloadCapture) -> AgentHarness<()> { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model("mock", Arc::new(ScriptedModel::new(responses))); + harness.register_tool(Arc::new(EchoTool)); + harness.with_policy(RunPolicy { + capture, + ..RunPolicy::default() + }); + harness +} + +/// A compact rendering of the lifecycle events, in emission order. +fn lifecycle(events: &[AgentEvent]) -> Vec { + events + .iter() + .filter_map(|event| match event { + AgentEvent::TurnStarted { turn } => Some(format!("turn.started:{turn}")), + AgentEvent::TurnCompleted { + turn, + tool_result_count, + tool_call_ids, + } => Some(format!( + "turn.completed:{turn}:{tool_result_count}:{}", + tool_call_ids + .iter() + .map(CallId::as_str) + .collect::>() + .join(",") + )), + AgentEvent::MessageAppended { + role, + index, + call_id, + .. + } => Some(format!( + "message:{index}:{role}{}", + call_id + .as_ref() + .map(|id| format!(":{}", id.as_str())) + .unwrap_or_default() + )), + AgentEvent::ModelStarted { .. } => Some("model.started".to_string()), + _ => None, + }) + .collect() +} + +#[tokio::test] +async fn turns_and_messages_are_announced_in_transcript_order() { + let harness = harness( + vec![tool_turn(&["a", "b"]), response(vec![], "done")], + PayloadCapture::default(), + ); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("life"), ()).with_events(recorder.sink()); + + let run = harness + .invoke_in_context(&(), ctx, vec![Message::user("go")]) + .await + .unwrap(); + + assert_eq!(run.messages.len(), 5); + assert_eq!( + lifecycle(&recorder.events()), + vec![ + "turn.started:1", + "model.started", + "message:1:assistant", + "message:2:tool:a", + "message:3:tool:b", + "turn.completed:1:2:a,b", + "turn.started:2", + "model.started", + "message:4:assistant", + "turn.completed:2:0:", + ] + ); +} + +#[tokio::test] +async fn message_payloads_follow_the_capture_policy() { + for (capture, expect_message, expect_tool) in [ + (PayloadCapture::default(), false, false), + (PayloadCapture::all(), true, true), + ] { + let harness = harness(vec![tool_turn(&["a"]), response(vec![], "done")], capture); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("cap"), ()).with_events(recorder.sink()); + harness + .invoke_in_context(&(), ctx, vec![Message::user("go")]) + .await + .unwrap(); + let mut seen = 0; + for event in recorder.events() { + if let AgentEvent::MessageAppended { role, message, .. } = event { + seen += 1; + let expected = if role == "tool" { + expect_tool + } else { + expect_message + }; + assert_eq!(message.is_some(), expected, "role {role}"); + } + } + assert_eq!(seen, 3, "assistant, tool, assistant"); + } +} + +#[tokio::test] +async fn a_failed_turn_is_still_closed() { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model("mock", Arc::new(FailingModel)); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("fail"), ()).with_events(recorder.sink()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("go")]) + .await; + assert!(partial.error.is_some()); + assert_eq!( + lifecycle(&recorder.events()), + vec!["turn.started:1", "model.started", "turn.completed:1:0:"] + ); +} + +#[tokio::test] +async fn queued_message_applied_carries_the_applied_messages() { + let harness = harness( + vec![tool_turn(&["a"]), response(vec![], "done")], + PayloadCapture::all(), + ); + let queue = Arc::new(RunQueue::new()); + queue + .push(QueueLane::Steer, Message::user("be brief")) + .await; + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("queue"), ()) + .with_events(recorder.sink()) + .with_run_queue(Arc::clone(&queue)); + harness + .invoke_in_context(&(), ctx, vec![Message::user("go")]) + .await + .unwrap(); + + let events = recorder.events(); + let applied = events + .iter() + .find_map(|event| match event { + AgentEvent::QueuedMessageApplied { + count, + first_index, + messages, + .. + } => Some((*count, *first_index, messages.clone())), + _ => None, + }) + .expect("queue applied"); + assert_eq!(applied.0, 1); + assert_eq!(applied.1, 3, "after user, assistant, tool"); + assert_eq!(applied.2.len(), 1); + assert_eq!( + applied.2[0], + serde_json::to_value(Message::user("be brief")).unwrap() + ); + // The same message is also announced as an ordinary transcript append. + assert!(lifecycle(&events).contains(&"message:3:user".to_string())); +} + +#[tokio::test] +async fn mixed_capture_keeps_one_payload_slot_per_applied_message() { + // `tool_io` only: the queued user message is not captured, the tool one is. + let harness = harness( + vec![tool_turn(&["a"]), response(vec![], "done")], + PayloadCapture { + model_io: false, + tool_io: true, + }, + ); + let queue = Arc::new(RunQueue::new()); + queue + .push(QueueLane::Steer, Message::user("uncaptured")) + .await; + queue + .push(QueueLane::Steer, Message::tool("t1", "captured")) + .await; + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("mixed"), ()) + .with_events(recorder.sink()) + .with_run_queue(Arc::clone(&queue)); + harness + .invoke_in_context(&(), ctx, vec![Message::user("go")]) + .await + .unwrap(); + let messages = recorder + .events() + .iter() + .find_map(|event| match event { + AgentEvent::QueuedMessageApplied { messages, .. } => Some(messages.clone()), + _ => None, + }) + .expect("queue applied"); + assert_eq!(messages.len(), 2, "one slot per applied message"); + assert!(messages[0].is_null()); + assert_eq!( + messages[1], + serde_json::to_value(Message::tool("t1", "captured")).unwrap() + ); +} + +// ── Transcript mirroring ──────────────────────────────────────────────────── + +/// Folds the lifecycle events into a role list the way a consumer mirroring +/// the transcript would. `?` marks messages known only by position (after a +/// rewrite). +pub(super) fn mirror_roles(events: &[AgentEvent], seed: &[Message]) -> Vec { + let mut roles: Vec = seed + .iter() + .map(|m| super::lifecycle::role_of(m).to_string()) + .collect(); + for event in events { + match event { + AgentEvent::MessageAppended { role, index, .. } => { + assert_eq!(*index, roles.len(), "append index follows the mirror"); + roles.push(role.clone()); + } + AgentEvent::MessageRetracted { index } => { + assert_eq!(*index + 1, roles.len(), "retraction pops the tail"); + roles.pop(); + } + AgentEvent::TranscriptRewritten { len, .. } => roles = vec!["?".to_string(); *len], + _ => {} + } + } + roles +} + +pub(super) fn assert_mirrors(events: &[AgentEvent], seed: &[Message], transcript: &[Message]) { + let mirror = mirror_roles(events, seed); + let actual: Vec = transcript + .iter() + .map(|m| super::lifecycle::role_of(m).to_string()) + .collect(); + assert_eq!(mirror.len(), actual.len(), "{mirror:?} vs {actual:?}"); + for (m, a) in mirror.iter().zip(&actual) { + assert!(m == "?" || m == a, "{mirror:?} vs {actual:?}"); + } +} + +fn truncated_empty(cap: u32) -> ModelResponse { + let mut r = response(vec![], ""); + r.finish_reason = Some("length".to_string()); + let _ = cap; + r +} + +#[tokio::test] +async fn a_recovery_pop_is_retracted_and_the_mirror_stays_exact() { + let model_script = vec![ + truncated_empty(2048), + truncated_empty(4096), + tool_turn(&["c1"]), + response(vec![], "done"), + ]; + let harness = harness(model_script, PayloadCapture::default()); + let recorder = EventRecorder::new(); + let ctx = RunContext::new( + RunConfig::new("recover").with_max_turn_output_tokens(2048), + (), + ) + .with_events(recorder.sink()); + let seed = vec![Message::user("go")]; + let run = harness + .invoke_in_context(&(), ctx, seed.clone()) + .await + .unwrap(); + + let events = recorder.events(); + let retractions = events + .iter() + .filter(|e| matches!(e, AgentEvent::MessageRetracted { index: 1 })) + .count(); + assert_eq!(retractions, 2, "both blank replies were dropped"); + // The recovery nudge is announced as an ordinary append. + assert!( + events.iter().any(|e| matches!( + e, + AgentEvent::MessageAppended { role, .. } if role == "user" + )), + "the nudge is announced" + ); + assert_mirrors(&events, &seed, &run.messages); +} + +#[tokio::test] +async fn steer_between_turns_reports_the_right_first_index() { + let harness = harness( + vec![ + tool_turn(&["a"]), + tool_turn(&["b"]), + response(vec![], "done"), + ], + PayloadCapture::default(), + ); + let queue = Arc::new(RunQueue::new()); + queue.push(QueueLane::Steer, Message::user("s1")).await; + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("steer"), ()) + .with_events(recorder.sink()) + .with_run_queue(Arc::clone(&queue)); + let seed = vec![Message::user("go")]; + let run = harness + .invoke_in_context(&(), ctx, seed.clone()) + .await + .unwrap(); + let events = recorder.events(); + let first = events.iter().find_map(|e| match e { + AgentEvent::QueuedMessageApplied { first_index, .. } => Some(*first_index), + _ => None, + }); + assert_eq!(first, Some(3)); + assert_eq!(run.messages[3].text(), "s1"); + assert_mirrors(&events, &seed, &run.messages); +} + +#[test] +fn tracker_retract_and_rebase_are_explicit() { + use super::lifecycle::TurnTracker; + let recorder = EventRecorder::new(); + let sink = recorder.sink(); + let capture = PayloadCapture::default(); + let mut tracker = TurnTracker::new(0); + let mut messages = vec![Message::user("a"), Message::user("b"), Message::user("c")]; + tracker.flush(&sink, capture, &messages); + messages.pop(); + tracker.retract_to(&sink, messages.len()); + // An unannounced message that is popped again emits nothing. + messages.push(Message::user("tmp")); + messages.pop(); + tracker.retract_to(&sink, messages.len()); + messages.insert(0, Message::system("sys")); + tracker.rebase(&sink, messages.len(), "tool_change"); + messages.push(Message::user("d")); + tracker.flush(&sink, capture, &messages); + let kinds: Vec = recorder + .events() + .iter() + .map(|e| match e { + AgentEvent::MessageAppended { index, .. } => format!("append:{index}"), + AgentEvent::MessageRetracted { index } => format!("retract:{index}"), + AgentEvent::TranscriptRewritten { len, reason } => format!("rewrite:{len}:{reason}"), + other => other.kind().to_string(), + }) + .collect(); + assert_eq!( + kinds, + vec![ + "append:0", + "append:1", + "append:2", + "retract:2", + "rewrite:3:tool_change", + "append:3" + ] + ); +} + +/// Replays scripted responses for a model without native structured output, so +/// the `answer` tool call is the structured-output channel. +struct ToolStructuredScript { + profile: tinyinference_llm::model::ModelProfile, + responses: std::sync::Mutex>, +} + +#[async_trait] +impl ChatModel<()> for ToolStructuredScript { + fn profile(&self) -> Option<&tinyinference_llm::model::ModelProfile> { + Some(&self.profile) + } + async fn invoke(&self, _: &(), _: ModelRequest) -> tinyinference_llm::Result { + Ok(self + .responses + .lock() + .unwrap() + .pop_front() + .expect("the model was called more often than scripted")) + } +} + +/// A mixed structured turn is closed before its queued steering is drained, so +/// the queued message is announced after `TurnCompleted` and is not counted as +/// one of that turn's tool results. +#[tokio::test] +async fn a_mixed_structured_turn_closes_before_queued_messages_are_drained() { + let mixed = response( + vec![ + ToolCall::new("s1", "answer", json!({"value": "first"})), + ToolCall::new("c1", "echo", json!({})), + ], + "", + ); + let last = response( + vec![ToolCall::new("s2", "answer", json!({"value": "final"}))], + "", + ); + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(ToolStructuredScript { + profile: tinyinference_llm::model::ModelProfile { + tool_calling: true, + native_structured_output: false, + json_schema: false, + ..Default::default() + }, + responses: std::sync::Mutex::new(vec![mixed, last].into()), + }), + ); + harness.register_tool(Arc::new(EchoTool)); + harness.with_policy(RunPolicy { + end_strategy: crate::runtime::EndStrategy::Exhaustive, + default_response_format: Some(tinyinference_llm::model::ResponseFormat::auto( + "answer", + json!({"type": "object"}), + )), + ..RunPolicy::default() + }); + let queue = Arc::new(RunQueue::new()); + queue + .push( + QueueLane::Steer, + Message::tool("queued-call", "late result"), + ) + .await; + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("mixed"), ()) + .with_events(recorder.sink()) + .with_run_queue(Arc::clone(&queue)); + harness + .invoke_in_context(&(), ctx, vec![Message::user("go")]) + .await + .unwrap(); + + let events = recorder.events(); + let lines = lifecycle(&events); + let completed = lines + .iter() + .position(|line| line.starts_with("turn.completed:1:")) + .expect("turn 1 completed"); + assert_eq!( + lines[completed], "turn.completed:1:2:s1,c1", + "only the turn's own tool results are counted: {lines:?}" + ); + let queued = lines + .iter() + .position(|line| line.ends_with(":queued-call")) + .expect("queued message announced"); + assert!( + completed < queued, + "queued message after TurnCompleted: {lines:?}" + ); +} + +#[tokio::test] +async fn messages_appended_by_after_agent_middleware_are_announced() { + use crate::middleware::{AgentRun, Middleware}; + struct Appender; + #[async_trait::async_trait] + impl Middleware<(), ()> for Appender { + fn name(&self) -> &str { + "appender" + } + async fn after_agent( + &self, + _: &mut RunContext<()>, + _: &(), + run: &mut AgentRun, + ) -> crate::error::Result<()> { + run.messages.push(Message::user("added by middleware")); + Ok(()) + } + } + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(crate::testkit::ScriptedModel::new(vec![ + crate::testkit::text_response("done"), + ])), + ); + harness.push_middleware(Arc::new(Appender)); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("aa-append"), ()).with_events(recorder.sink()); + let run = harness + .invoke_in_context(&(), ctx, vec![Message::user("hi")]) + .await + .unwrap(); + let last = run.messages.len() - 1; + let announced: Vec = recorder + .events() + .iter() + .filter_map(|event| match event { + AgentEvent::MessageAppended { index, .. } => Some(*index), + _ => None, + }) + .collect(); + assert_eq!(announced.last(), Some(&last), "{announced:?}"); +} diff --git a/crates/tinyagents-harness/src/agent_loop/mixed_turn.rs b/crates/tinyagents-harness/src/agent_loop/mixed_turn.rs index 37c1215d2..ef72f5872 100644 --- a/crates/tinyagents-harness/src/agent_loop/mixed_turn.rs +++ b/crates/tinyagents-harness/src/agent_loop/mixed_turn.rs @@ -84,6 +84,7 @@ impl AgentHarness { )); } run.final_response = Some(response); + ctx.close_turn(self.policy.capture, messages); if self .continue_from_queue_at_finish(ctx, status, messages) .await @@ -144,6 +145,7 @@ impl AgentHarness { { return Ok(TurnFlow::Exit(exit)); } + ctx.close_turn(self.policy.capture, messages); if let ControlEffect::Exit(exit) = self.apply_pending_control(ctx, run, status, messages)? { @@ -208,6 +210,10 @@ impl AgentHarness { return Ok(TurnFlow::Exit(exit)); } + // Close the mixed turn before queued messages are drained, so they are + // announced after its `TurnCompleted` like on the plain tool path. + ctx.close_turn(self.policy.capture, messages); + // Turn boundary (A4): same steer drain as the plain tool path. self.apply_queued_lane(ctx, status, messages, crate::run_queue::QueueLane::Steer) .await; diff --git a/crates/tinyagents-harness/src/agent_loop/mod.rs b/crates/tinyagents-harness/src/agent_loop/mod.rs index 2e6b09727..ec0d1ec34 100644 --- a/crates/tinyagents-harness/src/agent_loop/mod.rs +++ b/crates/tinyagents-harness/src/agent_loop/mod.rs @@ -109,6 +109,7 @@ use crate::middleware::{ use crate::model_registry::{ResolvedModelBinding, model_eligible}; use crate::runtime::{AgentHarness, EndStrategy, InvalidArgsPolicy, UnknownToolPolicy}; use crate::structured::{StructuredExtractor, StructuredStrategy}; +use crate::terminal::{TerminalClass, TerminalOutcome, TimeoutPhase}; use futures::StreamExt; use serde_json::Value; use tinyinference_llm::message::{Message, MessageDelta}; @@ -122,6 +123,7 @@ mod dialect; mod entry; mod handoff_transform; mod host_budget; +mod lifecycle; mod mixed_turn; mod model_call; mod model_switch; @@ -140,6 +142,7 @@ mod turn_control; mod turn_recovery; mod unknown_tool; +pub(crate) use lifecycle::TurnTracker; pub use stream::AgentStreamItem; pub(crate) use stream::{StreamRunner, invoke_stream_with_runner}; @@ -147,6 +150,9 @@ pub(crate) use stream::{StreamRunner, invoke_stream_with_runner}; #[path = "deferred_tests.rs"] mod deferred_test; #[cfg(test)] +#[path = "lifecycle_tests.rs"] +mod lifecycle_test; +#[cfg(test)] #[path = "model_profile_preview_tests.rs"] mod model_profile_preview_test; @@ -166,6 +172,9 @@ mod run_queue_test; #[path = "stream_idle_timeout_tests.rs"] mod stream_idle_timeout_test; #[cfg(test)] +#[path = "terminal_outcome_tests.rs"] +mod terminal_outcome_test; +#[cfg(test)] #[path = "mod_tests.rs"] mod test; #[cfg(test)] diff --git a/crates/tinyagents-harness/src/agent_loop/mod_tests.rs b/crates/tinyagents-harness/src/agent_loop/mod_tests.rs index 8b5ea8b23..fc9be4bbe 100644 --- a/crates/tinyagents-harness/src/agent_loop/mod_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/mod_tests.rs @@ -8929,3 +8929,45 @@ async fn graceful_mixed_turn_without_truncation_still_finishes_in_one_call() { ); assert_eq!(*tool.calls.lock().unwrap(), 1); } + +/// A fold of a tool-set change into the leading system message rewrites the +/// transcript in place; the lifecycle events report it so a mirror stays exact. +#[tokio::test] +async fn dynamic_toolset_fold_is_reported_as_a_transcript_rewrite() { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses(vec![ + tool_call_response("call-1", "search", json!({"q": "x"})), + text_response("done", 4, 2), + ])), + ); + let search: Arc = Arc::new(FakeTool::new("search", "search-output")); + let browse: Arc = Arc::new(FakeTool::new("browse", "browse-output")); + let toolset = Arc::new(DynamicToolSet { + calls: std::sync::atomic::AtomicUsize::new(0), + search: search.clone(), + browse, + }); + harness.with_toolset(toolset.clone()); + harness.register_tool_dispatch(Arc::new(crate::tool::toolset::ToolSetDispatchBridge::new( + toolset, search, + ))); + let recorder = crate::testkit::EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("fold-rewrite"), ()).with_events(recorder.sink()); + let seed = vec![Message::system("baseline persona"), Message::user("go")]; + + let run = harness + .invoke_in_context(&(), ctx, seed.clone()) + .await + .expect("run succeeds"); + + let events = recorder.events(); + assert!( + events + .iter() + .any(|e| matches!(e, AgentEvent::TranscriptRewritten { reason, .. } if reason == "tool_change")), + "the in-place fold is announced" + ); + super::lifecycle_test::assert_mirrors(&events, &seed, &run.messages); +} diff --git a/crates/tinyagents-harness/src/agent_loop/model_call.rs b/crates/tinyagents-harness/src/agent_loop/model_call.rs index 1604878e5..377535e11 100644 --- a/crates/tinyagents-harness/src/agent_loop/model_call.rs +++ b/crates/tinyagents-harness/src/agent_loop/model_call.rs @@ -318,6 +318,10 @@ impl AgentHarness { request }; + // The provider is reached only past the cache lookup and the wrap + // onion, so a cache hit or a short-circuiting middleware never claims + // `provider_started`. + ctx.mark_provider_started(); let response = self .invoke_model_resolving(state, ctx, effective_request, call_id, binding, shape) .await?; diff --git a/crates/tinyagents-harness/src/agent_loop/nested_tests.rs b/crates/tinyagents-harness/src/agent_loop/nested_tests.rs index 99ab45986..405900fac 100644 --- a/crates/tinyagents-harness/src/agent_loop/nested_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/nested_tests.rs @@ -575,6 +575,32 @@ async fn nested_calls_are_summarised_in_parent_metadata_not_the_transcript() { ); } +#[tokio::test] +async fn nested_calls_emit_no_message_appended_events() { + let caller = Caller::new("caller", vec![("leaf", json!({"n": 1}))]); + let mut harness = harness_with(vec![parent_call("p1", "caller")], enabled()); + harness.register_tool(Leaf::new("leaf")); + harness.register_tool(Arc::new(caller)); + + let recorder = EventRecorder::new(); + run(&harness, &recorder).await.expect("run succeeds"); + assert_eq!(nested_started(&recorder), 1); + + // Only the model-issued call owns a transcript row; a nested call must + // never be announced as an appended message. + let tool_rows: Vec> = recorder + .events() + .iter() + .filter_map(|event| match event { + AgentEvent::MessageAppended { role, call_id, .. } if role == "tool" => { + Some(call_id.as_ref().map(ToString::to_string)) + } + _ => None, + }) + .collect(); + assert_eq!(tool_rows, [Some("p1".to_string())]); +} + #[tokio::test] async fn the_nested_summary_is_capped_and_truncates_long_arguments() { let script: Vec<(&'static str, Value)> = (0..40) diff --git a/crates/tinyagents-harness/src/agent_loop/response_recovery.rs b/crates/tinyagents-harness/src/agent_loop/response_recovery.rs index 6bb2188b1..73b1ff8b1 100644 --- a/crates/tinyagents-harness/src/agent_loop/response_recovery.rs +++ b/crates/tinyagents-harness/src/agent_loop/response_recovery.rs @@ -65,6 +65,7 @@ impl AgentHarness { "[agent_loop] length-truncated tool calls keep recurring; truncated-tool-call retry budget exhausted" ); messages.pop(); + ctx.retract_transcript(messages.len()); return Err(TinyAgentsError::LimitExceeded(format!( "run `{}` stopped: {} consecutive \ retries of a tool call truncated by the output token limit did not \ @@ -189,6 +190,7 @@ impl AgentHarness { { turn_recovery.withheld_call_nudges_used += 1; messages.pop(); + ctx.retract_transcript(messages.len()); tracing::info!( target: "tinyagents::agent_loop", run_id = %ctx.run_id(), @@ -232,6 +234,7 @@ impl AgentHarness { // Drop the useless empty assistant row appended above so the // retry re-sends the identical transcript. messages.pop(); + ctx.retract_transcript(messages.len()); turn_recovery.truncated_empty_retries_used += 1; // Grow the token budget when the request set one: double it, // clamped at 4x the original cap. An unset budget stays unset @@ -258,6 +261,7 @@ impl AgentHarness { && ctx.limits.remaining_model_calls() > 0 { messages.pop(); + ctx.retract_transcript(messages.len()); turn_recovery.truncated_empty_nudges_used += 1; let nudge = if tools_available_this_turn { TRUNCATED_EMPTY_TOOL_NUDGE @@ -309,6 +313,7 @@ impl AgentHarness { && ctx.limits.remaining_model_calls() > 0 { messages.pop(); + ctx.retract_transcript(messages.len()); turn_recovery.empty_response_retries_used += 1; tracing::info!( target: "tinyagents::agent_loop", diff --git a/crates/tinyagents-harness/src/agent_loop/run_loop.rs b/crates/tinyagents-harness/src/agent_loop/run_loop.rs index 240883cfb..b1bbd6dca 100644 --- a/crates/tinyagents-harness/src/agent_loop/run_loop.rs +++ b/crates/tinyagents-harness/src/agent_loop/run_loop.rs @@ -35,9 +35,13 @@ impl AgentHarness { // A mid-turn tool failure used to drop everything accumulated so far, // leaving the caller unable to inspect, repair, or resume from the // partial conversation. + ctx.reset_turn_tracker(messages.len()); let outcome = self .run_loop_body(state, ctx, run, status, &mut messages, streaming) .await; + // Announce whatever the final turn appended and close it, on every + // exit path, before the transcript moves onto the run. + ctx.close_turn(self.policy.capture, &messages); run.messages = std::mem::take(&mut messages); // A4: the `Collect` lane is delivered on the run, never on the // transcript, and on every exit path — a host that pushed @@ -60,8 +64,27 @@ impl AgentHarness { } }; + // One typed answer to "how did the loop end", derived once here so the + // event, `run.terminal` and the legacy fields cannot disagree. + let mut terminal = TerminalOutcome::from_loop_exit(&exit, ctx.provider_started()); + // A repeat / no-progress guard halts by pausing; its marker says so. + if matches!(exit, LoopExit::Paused(_)) + && let Some(summary) = ctx.halted_by_guard.take() + { + terminal = + TerminalOutcome::halted(summary).with_provider_started(ctx.provider_started()); + } + run.terminal = Some(terminal.clone()); + status.mark_running(HarnessPhase::Middleware); - self.middleware.run_after_agent(ctx, state, run).await?; + let after_agent = self.middleware.run_after_agent(ctx, state, run).await; + // `after_agent` may post-process `run.messages`; announce anything it + // appended (even if it then failed) so a mirror built from lifecycle + // events matches the returned transcript. + ctx.flush_transcript(self.policy.capture, &run.messages); + after_agent?; + // The hook may have replaced the outcome; the event reports the final one. + let terminal = run.terminal.clone().unwrap_or(terminal); match exit { LoopExit::Finished | LoopExit::LimitStop(_) => { @@ -76,6 +99,7 @@ impl AgentHarness { } let record = ctx.emit(AgentEvent::RunCompleted { run_id: ctx.run_id().clone(), + outcome: Some(terminal), }); status.set_last_event(record.id); } @@ -291,6 +315,7 @@ impl AgentHarness { } // Fail-closed limit and deadline checks before each model call. + ctx.clear_last_limit(); if ctx.check_deadline().is_err() { ctx.emit(AgentEvent::LimitReached { kind: LimitKind::WallClock, @@ -380,10 +405,16 @@ impl AgentHarness { // transcript patch, so it is part of *this* turn's request; then // promote tools a successful `tool_search` returned and assemble // the turn's wire list. - surface + // Announce pending appends first so a rewrite is reported against + // the transcript it actually rewrote. + ctx.flush_transcript(self.policy.capture, messages); + let rewrote = surface .declare_toolset_changes(self, ctx, messages, &host_allows, patch_profile.as_ref()) .await?; - surface.promote_discovered(messages, patch_profile.as_ref()); + let rewrote = surface.promote_discovered(messages, patch_profile.as_ref()) || rewrote; + if rewrote { + ctx.rebase_transcript(messages.len(), "tool_change"); + } surface.assemble_turn_schemas(); // Build the request from the working transcript, tool schemas, and @@ -693,9 +724,11 @@ impl AgentHarness { // call id the loop uses instead of deriving an uncorrelated one // (I-7). Cleared right after the wrap onion returns, below. ctx.active_model_call = Some(call_id.clone()); + ctx.begin_model_call(); // Captured here (where the call actually starts) so the completed // event carries a real start time for duration-aware exporters. let model_started_at_ms = crate::ids::now_ms(); + ctx.start_turn(self.policy.capture, messages); let record = ctx.emit(AgentEvent::ModelStarted { call_id: call_id.clone(), model: model_name.clone(), @@ -759,6 +792,9 @@ impl AgentHarness { { Ok(outcome) => outcome.into_response_with_control(), Err(error) => { + status.active_model_call = None; + ctx.active_model_call = None; + ctx.mark_model_call_failed(); // A response discarded before a retry was billed even if // the replacement call fails. Do not leave that usage in // the context when the `?` below would skip normal @@ -812,6 +848,10 @@ impl AgentHarness { super::dialect::withhold_text_calls(&mut response, &call_id, &recovery.dropped); } + // The provider call has returned: a failure from here on (response + // accounting, `after_model`) is after-turn, not an in-flight call. + ctx.active_model_call = None; + // Account for the completed provider response before fallible // response middleware (see `model_turn.rs`). A middleware rejection // must not erase usage already incurred, and the host admission @@ -837,7 +877,6 @@ impl AgentHarness { .middleware .run_after_model(ctx, state, &mut response) .await; - ctx.active_model_call = None; after_model?; let captured_output = self .policy @@ -854,6 +893,7 @@ impl AgentHarness { status.set_last_event(record.id); messages.push(Message::Assistant(response.message.clone())); + ctx.flush_transcript(self.policy.capture, messages); // Safe checkpoint: honor any control outcome a middleware requested // during this turn (for example an early-exit tool or a budget stop @@ -1061,9 +1101,11 @@ impl AgentHarness { && response.text().trim().is_empty() { messages.pop(); + ctx.retract_transcript(messages.len()); return Err(TinyAgentsError::EmptyResponse); } run.final_response = Some(response); + ctx.close_turn(self.policy.capture, messages); // Natural finish (A4): queued steering or a follow-up turns // "done" into "one more turn" instead of returning. if self @@ -1110,6 +1152,8 @@ impl AgentHarness { return Ok(exit); } + ctx.close_turn(self.policy.capture, messages); + // Turn boundary (A4): every tool result of this batch is on the // transcript, so queued steering can be applied now — never // mid-batch — before the next model call sees it. diff --git a/crates/tinyagents-harness/src/agent_loop/run_queue_tests.rs b/crates/tinyagents-harness/src/agent_loop/run_queue_tests.rs index e3cb36d0b..b0103a949 100644 --- a/crates/tinyagents-harness/src/agent_loop/run_queue_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/run_queue_tests.rs @@ -141,7 +141,7 @@ fn queued_applied(events: &[AgentEvent]) -> Vec<(QueueLane, usize)> { events .iter() .filter_map(|event| match event { - AgentEvent::QueuedMessageApplied { lane, count } => Some((*lane, *count)), + AgentEvent::QueuedMessageApplied { lane, count, .. } => Some((*lane, *count)), _ => None, }) .collect() diff --git a/crates/tinyagents-harness/src/agent_loop/terminal_outcome_tests.rs b/crates/tinyagents-harness/src/agent_loop/terminal_outcome_tests.rs new file mode 100644 index 000000000..98bc9559c --- /dev/null +++ b/crates/tinyagents-harness/src/agent_loop/terminal_outcome_tests.rs @@ -0,0 +1,530 @@ +//! The loop reports a typed [`TerminalOutcome`] on `RunCompleted`/`RunFailed` +//! and on `AgentRun::terminal`, on every exit path. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::cancel::CancellationToken; +use crate::context::{RunConfig, RunContext}; +use crate::error::TinyAgentsError; +use crate::events::{AgentEvent, LimitKind}; +use crate::limits::{LimitBehavior, RunLimits}; +use crate::retry::FailoverReason; +use crate::runtime::{AgentHarness, RunPolicy}; +use crate::steering::{SteeringCommand, SteeringHandle, SteeringPolicy}; +use crate::terminal::{TerminalClass, TerminalOutcome, TerminalReason, TimeoutPhase}; +use crate::testkit::{EventRecorder, ScriptedModel}; +use tinyinference_llm::message::{AssistantMessage, ContentBlock, Message}; +use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse}; +use tinyinference_llm::tool::ToolCall; +use tinyinference_llm::usage::Usage; + +fn response(tool_calls: Vec, text: &str) -> ModelResponse { + ModelResponse { + message: AssistantMessage { + id: None, + content: if text.is_empty() { + Vec::new() + } else { + vec![ContentBlock::Text(text.to_string())] + }, + tool_calls, + usage: Some(Usage::new(1, 1)), + origin: None, + }, + usage: Some(Usage::new(1, 1)), + finish_reason: Some("stop".to_string()), + raw: None, + resolved_model: None, + continue_turn: None, + served_from_cache: false, + correlation: None, + resolved_route: None, + } +} + +struct FailingModel(&'static str); + +#[async_trait] +impl ChatModel<()> for FailingModel { + async fn invoke(&self, _: &(), _: ModelRequest) -> tinyinference_llm::Result { + Err(tinyinference_llm::Error::Model(self.0.to_string())) + } +} + +struct PendingModel; + +#[async_trait] +impl ChatModel<()> for PendingModel { + async fn invoke(&self, _: &(), _: ModelRequest) -> tinyinference_llm::Result { + std::future::pending().await + } +} + +fn harness_with(model: Arc>) -> AgentHarness<()> { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model("mock", model); + harness +} + +fn terminal_events(events: &[AgentEvent]) -> Vec<&AgentEvent> { + events + .iter() + .filter(|e| { + matches!( + e, + AgentEvent::RunCompleted { .. } | AgentEvent::RunFailed { .. } + ) + }) + .collect() +} + +#[tokio::test] +async fn a_finished_run_reports_a_completed_outcome() { + let harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("ok"), ()).with_events(recorder.sink()); + + let run = harness + .invoke_in_context(&(), ctx, vec![Message::user("hi")]) + .await + .unwrap(); + + let outcome = run.terminal.clone().expect("terminal outcome on the run"); + assert_eq!(outcome.reason, TerminalReason::Completed); + assert_eq!(outcome.class, TerminalClass::Success); + assert!(outcome.provider_started); + let events = recorder.events(); + let terminal = terminal_events(&events); + assert_eq!(terminal.len(), 1); + assert!(matches!( + terminal[0], + AgentEvent::RunCompleted { outcome: Some(o), .. } if *o == outcome + )); +} + +#[tokio::test] +async fn a_stop_with_partial_cap_completes_with_a_limit_outcome() { + let harness = { + let mut h = harness_with(Arc::new(ScriptedModel::new(vec![response( + vec![ToolCall::new("c1", "spin", serde_json::json!({}))], + "", + )]))); + h.with_policy(RunPolicy { + limits: RunLimits::default() + .with_max_model_calls(1) + .with_behavior(LimitBehavior::StopWithPartial), + ..RunPolicy::default() + }); + h + }; + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("cap"), ()).with_events(recorder.sink()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("go")]) + .await; + // The first call returns a tool call for an unregistered tool, then the + // second iteration trips the cap. + assert!(partial.error.is_none(), "{:?}", partial.error); + let outcome = partial.run.terminal.expect("outcome"); + assert_eq!( + outcome.reason, + TerminalReason::LimitReached(Some(LimitKind::ModelCalls)) + ); + assert_eq!(outcome.class, TerminalClass::Failure); + let events = recorder.events(); + assert!(matches!( + terminal_events(&events)[0], + AgentEvent::RunCompleted { outcome: Some(o), .. } + if o.reason == TerminalReason::LimitReached(Some(LimitKind::ModelCalls)) + )); +} + +#[tokio::test] +async fn a_provider_failure_is_classified_on_run_failed_and_the_partial_run() { + let harness = harness_with(Arc::new(FailingModel("HTTP 429 too many requests"))); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("fail"), ()).with_events(recorder.sink()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + + let error = partial.error.expect("run fails"); + let outcome = partial.run.terminal.expect("outcome on the partial run"); + assert_eq!( + outcome.reason, + TerminalReason::ProviderFailed(Some(FailoverReason::RateLimit)) + ); + assert!(outcome.provider_started, "the call had started"); + assert_eq!(outcome.message, error.to_string()); + let events = recorder.events(); + match terminal_events(&events)[0] { + AgentEvent::RunFailed { + error: e, + outcome: Some(o), + .. + } => { + assert_eq!(*e, error.to_string(), "legacy string preserved"); + assert_eq!(*o, outcome); + } + other => panic!("unexpected {other:?}"), + } +} + +#[tokio::test] +async fn a_pre_cancelled_run_reports_cancellation_before_the_provider() { + let harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "x")]))); + let token = CancellationToken::new(); + token.cancel(); + let ctx = RunContext::new(RunConfig::new("cancel"), ()).with_cancellation(token); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + let outcome = partial.run.terminal.expect("outcome"); + assert_eq!(outcome.reason, TerminalReason::Cancelled); + assert_eq!(outcome.class, TerminalClass::Cancellation); + assert!(!outcome.provider_started); +} + +#[tokio::test] +async fn a_run_deadline_during_a_provider_call_reports_a_provider_phase_timeout() { + let harness = harness_with(Arc::new(PendingModel)); + let ctx = RunContext::new(RunConfig::new("deadline").with_timeout_ms(30), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert!(matches!(partial.error, Some(TinyAgentsError::Timeout(_)))); + let outcome = partial.run.terminal.expect("outcome"); + assert_eq!(outcome.reason, TerminalReason::Timeout); + assert_eq!(outcome.class, TerminalClass::Timeout); + assert_eq!(outcome.timeout_phase, Some(TimeoutPhase::Provider)); + assert!(outcome.provider_started); +} + +#[tokio::test] +async fn a_paused_run_reports_a_suspended_outcome_without_run_completed() { + let harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "x")]))); + let handle = SteeringHandle::new(SteeringPolicy::allow_all()); + handle.send(SteeringCommand::Pause); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("pause"), ()) + .with_steering(handle) + .with_events(recorder.sink()); + let run = harness + .invoke_in_context(&(), ctx, vec![Message::user("hi")]) + .await + .unwrap(); + assert!(run.paused.is_some()); + let outcome = run.terminal.expect("outcome"); + assert_eq!(outcome.reason, TerminalReason::Paused); + assert_eq!(outcome.class, TerminalClass::Suspended); + assert!(terminal_events(&recorder.events()).is_empty()); +} + +#[test] +fn outcome_helper_is_usable_from_hosts() { + // Public constructors compose with merge for hosts that race signals. + let merged = TerminalOutcome::completed().merge(TerminalOutcome::halted("loop")); + assert_eq!(merged.reason, TerminalReason::Halted); +} + +#[tokio::test] +async fn a_limit_exceeded_failure_carries_the_limit_kind() { + let harness = harness_with(Arc::new(ScriptedModel::new(vec![response( + vec![ToolCall::new("c1", "spin", serde_json::json!({}))], + "", + )]))); + let ctx = RunContext::new(RunConfig::new("cap-error").with_max_model_calls(1), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("go")]) + .await; + assert!(matches!( + partial.error, + Some(TinyAgentsError::LimitExceeded(_)) + )); + assert_eq!( + partial.run.terminal.expect("outcome").reason, + TerminalReason::LimitReached(Some(LimitKind::ModelCalls)) + ); +} + +#[tokio::test] +async fn after_agent_middleware_can_read_the_terminal_outcome() { + use crate::middleware::{AgentRun, Middleware}; + use std::sync::Mutex; + struct Reader(Arc>>); + #[async_trait] + impl Middleware<(), ()> for Reader { + fn name(&self) -> &str { + "reader" + } + async fn after_agent( + &self, + _: &mut RunContext<()>, + _: &(), + run: &mut AgentRun, + ) -> crate::error::Result<()> { + *self.0.lock().unwrap() = run.terminal.clone(); + Ok(()) + } + } + let seen = Arc::new(Mutex::new(None)); + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_middleware(Arc::new(Reader(seen.clone()))); + harness + .invoke_default(&(), vec![Message::user("hi")]) + .await + .unwrap(); + assert_eq!( + seen.lock().unwrap().as_ref().map(|o| o.reason), + Some(TerminalReason::Completed) + ); +} + +#[tokio::test] +async fn a_timeout_outcome_set_by_middleware_on_an_ok_return_is_not_completed() { + use crate::ids::ExecutionStatus; + use crate::middleware::{AgentRun, Middleware}; + struct ForceTimeout; + #[async_trait] + impl Middleware<(), ()> for ForceTimeout { + fn name(&self) -> &str { + "force_timeout" + } + async fn after_agent( + &self, + _: &mut RunContext<()>, + _: &(), + run: &mut AgentRun, + ) -> crate::error::Result<()> { + run.terminal = Some(TerminalOutcome::new(TerminalReason::Timeout, "deadline")); + Ok(()) + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_middleware(Arc::new(ForceTimeout)); + let ctx = RunContext::new(RunConfig::new("to"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert!(partial.error.is_none()); + assert_eq!(partial.status.status, ExecutionStatus::Failed); +} + +#[tokio::test] +async fn a_cache_served_run_does_not_claim_the_provider_started() { + use crate::cache::InMemoryResponseCache; + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.with_response_cache(Arc::new(InMemoryResponseCache::new())); + let input = vec![Message::user("same request")]; + let first = harness.invoke_default(&(), input.clone()).await.unwrap(); + assert!(first.terminal.unwrap().provider_started); + let second = harness.invoke_default(&(), input).await.unwrap(); + assert_eq!(second.text(), Some("done".to_string())); + assert!(!second.terminal.unwrap().provider_started); +} + +#[tokio::test] +async fn an_after_model_timeout_is_classified_after_the_provider_call() { + use crate::middleware::Middleware; + struct SlowAfterModel; + #[async_trait] + impl Middleware<(), ()> for SlowAfterModel { + fn name(&self) -> &str { + "slow_after_model" + } + async fn after_model( + &self, + _: &mut RunContext<()>, + _: &(), + _: &mut ModelResponse, + ) -> crate::error::Result<()> { + Err(TinyAgentsError::Timeout("hook deadline".into())) + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_middleware(Arc::new(SlowAfterModel)); + let ctx = RunContext::new(RunConfig::new("am"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert!(partial.error.is_some()); + let outcome = partial.run.terminal.expect("outcome"); + assert_eq!(outcome.reason, TerminalReason::Timeout); + assert_eq!(outcome.timeout_phase, Some(TimeoutPhase::AfterTurn)); + assert!(outcome.provider_started); +} + +#[tokio::test] +async fn a_wrap_model_rejection_before_dispatch_does_not_claim_the_provider_started() { + use crate::middleware::{MiddlewareModelOutcome, ModelHandler, ModelMiddleware}; + struct Reject; + #[async_trait] + impl ModelMiddleware<(), ()> for Reject { + fn name(&self) -> &str { + "reject" + } + async fn wrap_model( + &self, + _: &mut RunContext<()>, + _: &(), + _: ModelRequest, + _: ModelHandler<'_, (), ()>, + ) -> crate::error::Result { + Err(TinyAgentsError::Timeout("rejected before dispatch".into())) + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_model_middleware(Arc::new(Reject)); + let ctx = RunContext::new(RunConfig::new("rej"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert!(partial.error.is_some()); + let outcome = partial.run.terminal.expect("outcome"); + assert!(!outcome.provider_started); + assert_eq!(outcome.timeout_phase, Some(TimeoutPhase::BeforeProvider)); +} + +#[tokio::test] +async fn run_completed_reports_the_outcome_after_after_agent_middleware_replaced_it() { + use crate::middleware::{AgentRun, Middleware}; + struct Replace; + #[async_trait] + impl Middleware<(), ()> for Replace { + fn name(&self) -> &str { + "replace" + } + async fn after_agent( + &self, + _: &mut RunContext<()>, + _: &(), + run: &mut AgentRun, + ) -> crate::error::Result<()> { + run.terminal = Some(TerminalOutcome::new(TerminalReason::Timeout, "forced")); + Ok(()) + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_middleware(Arc::new(Replace)); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("rep"), ()).with_events(recorder.sink()); + harness + .invoke_in_context(&(), ctx, vec![Message::user("hi")]) + .await + .unwrap(); + let events = recorder.events(); + assert!(matches!( + terminal_events(&events)[0], + AgentEvent::RunCompleted { outcome: Some(o), .. } if o.reason == TerminalReason::Timeout + )); +} + +#[tokio::test] +async fn a_later_call_rejected_before_dispatch_is_not_a_provider_phase_failure() { + use crate::middleware::{MiddlewareModelOutcome, ModelHandler, ModelMiddleware}; + use std::sync::atomic::{AtomicUsize, Ordering}; + struct RejectSecond(AtomicUsize); + #[async_trait] + impl ModelMiddleware<(), ()> for RejectSecond { + fn name(&self) -> &str { + "reject_second" + } + async fn wrap_model( + &self, + ctx: &mut RunContext<()>, + state: &(), + request: ModelRequest, + next: ModelHandler<'_, (), ()>, + ) -> crate::error::Result { + if self.0.fetch_add(1, Ordering::SeqCst) == 0 { + next.run(ctx, state, request).await + } else { + Err(TinyAgentsError::Timeout("rejected before dispatch".into())) + } + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![ + response(vec![ToolCall::new("c1", "t", serde_json::json!({}))], ""), + response(vec![], "done"), + ]))); + harness.push_model_middleware(Arc::new(RejectSecond(AtomicUsize::new(0)))); + let ctx = RunContext::new(RunConfig::new("second"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert!(partial.error.is_some()); + let outcome = partial.run.terminal.expect("outcome"); + assert_eq!(outcome.timeout_phase, Some(TimeoutPhase::BeforeProvider)); + assert!( + outcome.provider_started, + "an earlier call reached the provider" + ); +} + +#[tokio::test] +async fn a_suspended_outcome_set_by_middleware_is_interrupted_not_completed() { + use crate::ids::ExecutionStatus; + use crate::middleware::{AgentRun, Middleware}; + struct Suspend; + #[async_trait] + impl Middleware<(), ()> for Suspend { + fn name(&self) -> &str { + "suspend" + } + async fn after_agent( + &self, + _: &mut RunContext<()>, + _: &(), + run: &mut AgentRun, + ) -> crate::error::Result<()> { + run.terminal = Some(TerminalOutcome::new(TerminalReason::Paused, "hold")); + Ok(()) + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.push_middleware(Arc::new(Suspend)); + let ctx = RunContext::new(RunConfig::new("susp"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("hi")]) + .await; + assert_eq!(partial.status.status, ExecutionStatus::Interrupted); +} + +#[tokio::test] +async fn a_cache_hit_followed_by_an_after_model_error_does_not_claim_the_provider() { + use crate::cache::InMemoryResponseCache; + use crate::middleware::Middleware; + use std::sync::atomic::{AtomicUsize, Ordering}; + struct FailSecond(AtomicUsize); + #[async_trait] + impl Middleware<(), ()> for FailSecond { + fn name(&self) -> &str { + "fail_second" + } + async fn after_model( + &self, + _: &mut RunContext<()>, + _: &(), + _: &mut ModelResponse, + ) -> crate::error::Result<()> { + if self.0.fetch_add(1, Ordering::SeqCst) == 0 { + Ok(()) + } else { + Err(TinyAgentsError::Timeout("late".into())) + } + } + } + let mut harness = harness_with(Arc::new(ScriptedModel::new(vec![response(vec![], "done")]))); + harness.with_response_cache(Arc::new(InMemoryResponseCache::new())); + harness.push_middleware(Arc::new(FailSecond(AtomicUsize::new(0)))); + let input = vec![Message::user("same request")]; + harness.invoke_default(&(), input.clone()).await.unwrap(); + let ctx = RunContext::new(RunConfig::new("cachehit"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, input) + .await; + assert!(partial.error.is_some()); + assert!(!partial.run.terminal.expect("outcome").provider_started); +} diff --git a/crates/tinyagents-harness/src/agent_loop/tool_surface.rs b/crates/tinyagents-harness/src/agent_loop/tool_surface.rs index 069f47a86..482fb6e89 100644 --- a/crates/tinyagents-harness/src/agent_loop/tool_surface.rs +++ b/crates/tinyagents-harness/src/agent_loop/tool_surface.rs @@ -183,19 +183,23 @@ impl ToolSurface { messages: &mut Vec, host_allows: &(dyn Fn(&str) -> bool + Sync), patch_profile: Option<&tinyinference_llm::model::ModelProfile>, - ) -> Result<()> { + ) -> Result { if harness.toolset.is_none() { - return Ok(()); + return Ok(false); } let live_schemas = harness.direct_tool_schemas(ctx, host_allows).await?; + let mut rewrote = false; if let Some(patch) = tool_changes::diff_tool_set(&self.declared_tool_schemas, &live_schemas) { let in_place = tool_changes::patch_inserts_in_place(patch_profile); tool_changes::apply_tool_change_patch(messages, patch, in_place); + // A mid-conversation patch is an ordinary append; the folded or + // front-inserted form rewrites the transcript in place. + rewrote = !in_place; self.declared_tool_schemas = live_schemas.clone(); self.direct_tool_schemas = live_schemas; } - Ok(()) + Ok(rewrote) } /// Promotes only names returned by a successful intrinsic search. The patch @@ -205,18 +209,20 @@ impl ToolSurface { &mut self, messages: &mut Vec, patch_profile: Option<&tinyinference_llm::model::ModelProfile>, - ) { + ) -> bool { let newly_promoted: Vec = self .promoted_names .difference(&self.recorded_promotions) .filter_map(|name| self.deferred_catalog.get(name).cloned()) .collect(); if newly_promoted.is_empty() { - return; + return false; } + let mut rewrote = false; if let Some(patch) = tool_changes::diff_tool_set(&[], &newly_promoted) { let in_place = tool_changes::patch_inserts_in_place(patch_profile); tool_changes::apply_tool_change_patch(messages, patch, in_place); + rewrote = !in_place; } self.recorded_promotions .extend(newly_promoted.iter().map(|schema| schema.name.clone())); @@ -225,6 +231,7 @@ impl ToolSurface { .into_iter() .map(|schema| (schema.name.clone(), schema)), ); + rewrote } /// Rebuilds the turn's wire list: the direct set, then promoted tools not diff --git a/crates/tinyagents-harness/src/agent_loop/turn_control.rs b/crates/tinyagents-harness/src/agent_loop/turn_control.rs index 2c984ef4e..a4c15ae08 100644 --- a/crates/tinyagents-harness/src/agent_loop/turn_control.rs +++ b/crates/tinyagents-harness/src/agent_loop/turn_control.rs @@ -29,8 +29,36 @@ impl AgentHarness { return false; } let count = items.len(); + let first_index = messages.len(); + // Payloads follow the capture policy, like every other event. + // One slot per applied message so payloads stay aligned with + // `first_index..first_index + count`; an uncaptured message is `null`. + // Nothing is emitted at all when no message in the batch is captured. + let is_captured = |message: &Message| match message { + Message::Tool(_) => self.policy.capture.tool_io, + _ => self.policy.capture.model_io, + }; + let captured: Vec = if items.iter().any(is_captured) { + items + .iter() + .map(|message| { + if is_captured(message) { + super::lifecycle::to_value_logged(message) + } else { + serde_json::Value::Null + } + }) + .collect() + } else { + Vec::new() + }; messages.extend(items); - let record = ctx.emit(AgentEvent::QueuedMessageApplied { lane, count }); + let record = ctx.emit(AgentEvent::QueuedMessageApplied { + lane, + count, + first_index, + messages: captured, + }); status.set_last_event(record.id); tracing::debug!( target: "tinyagents::agent_loop", diff --git a/crates/tinyagents-harness/src/context/mod.rs b/crates/tinyagents-harness/src/context/mod.rs index b5d66ecad..17e608930 100644 --- a/crates/tinyagents-harness/src/context/mod.rs +++ b/crates/tinyagents-harness/src/context/mod.rs @@ -324,7 +324,13 @@ impl RunContext { host_agent_id: None, host_authority: None, terminal_observer: None, + turns: crate::agent_loop::TurnTracker::default(), + halted_by_guard: None, + last_limit: std::sync::Mutex::new(None), active_model_call: None, + provider_started: false, + call_provider_started: false, + model_call_failed: false, call_streamed: false, prefix_epoch: 0, discarded_usage: Vec::new(), @@ -801,9 +807,85 @@ impl RunContext { /// Emits `event` on this run's event sink, returning the recorded entry. pub fn emit(&self, event: AgentEvent) -> EventRecord { + // Remember which cap tripped last, so a `LimitExceeded` failure can be + // classified with its kind (the error itself carries only text). + if let AgentEvent::LimitReached { kind } = &event { + *self + .last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(*kind); + } self.events.emit(event) } + /// Forgets the cached limit kind. The loop calls this at each turn's limit + /// checks so a kind emitted earlier cannot be attributed to a later, + /// unrelated `LimitExceeded` that emitted no event of its own. + pub(crate) fn clear_last_limit(&self) { + self.last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + } + + /// The most recent `LimitReached` kind, without consuming it. + pub fn peek_last_limit(&self) -> Option { + *self + .last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub(crate) fn take_last_limit(&self) -> Option { + self.last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + } + + /// Records that a provider call was dispatched. Called by the model base + /// call of each loop driver immediately before it reaches the provider. + pub fn mark_provider_started(&mut self) { + self.provider_started = true; + self.call_provider_started = true; + } + + #[doc(hidden)] + pub fn mark_model_call_failed(&mut self) { + self.model_call_failed = true; + } + + /// Whether a model call surfaced an error (see `model_call_failed`). + pub fn model_call_failed(&self) -> bool { + self.model_call_failed + } + + /// Marks the start of a model call: clears the per-call provider flag. + pub fn begin_model_call(&mut self) { + self.call_provider_started = false; + self.model_call_failed = false; + } + + /// Whether the current (or most recent) model call reached the provider. + pub fn call_provider_started(&self) -> bool { + self.call_provider_started + } + + /// Whether any provider call has been dispatched during this run. + pub fn provider_started(&self) -> bool { + self.provider_started + } + + /// Takes the repeat-progress guard's halt summary, if one was latched. + pub fn take_halted_by_guard(&mut self) -> Option { + self.halted_by_guard.take() + } + + /// Resets lifecycle tracking for a run whose input already contains seed messages. + pub fn reset_turn_tracker(&mut self, seed_len: usize) { + self.turns = crate::agent_loop::TurnTracker::new(seed_len); + } + /// Returns this run's identifier. pub fn run_id(&self) -> &RunId { &self.config.run_id @@ -871,6 +953,10 @@ impl RunContext { /// /// Returns an error if the configured model-call cap is exceeded. pub fn record_model_call(&mut self) -> Result<()> { + self.last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); self.limits.record_model_call() } @@ -878,6 +964,10 @@ impl RunContext { /// /// Returns an error if the configured tool-call cap is exceeded. pub fn record_tool_call(&mut self) -> Result<()> { + self.last_limit + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); self.limits.record_tool_call() } diff --git a/crates/tinyagents-harness/src/context/mod_tests.rs b/crates/tinyagents-harness/src/context/mod_tests.rs index 6361daa68..d90ee281d 100644 --- a/crates/tinyagents-harness/src/context/mod_tests.rs +++ b/crates/tinyagents-harness/src/context/mod_tests.rs @@ -614,3 +614,18 @@ async fn bounded_returns_cancelled_when_the_run_is_cancelled_before_the_future_r Err(crate::error::TinyAgentsError::Cancelled) )); } + +#[test] +fn a_cleared_limit_kind_is_not_attributed_to_a_later_error() { + use crate::events::{AgentEvent, LimitKind}; + let ctx: RunContext<()> = RunContext::new(RunConfig::new("run-limit-cache"), ()); + ctx.emit(AgentEvent::LimitReached { + kind: LimitKind::ToolCalls, + }); + ctx.clear_last_limit(); + assert_eq!(ctx.take_last_limit(), None); + ctx.emit(AgentEvent::LimitReached { + kind: LimitKind::ModelCalls, + }); + assert_eq!(ctx.take_last_limit(), Some(LimitKind::ModelCalls)); +} diff --git a/crates/tinyagents-harness/src/context/types.rs b/crates/tinyagents-harness/src/context/types.rs index 4ca9bb00f..47aa77901 100644 --- a/crates/tinyagents-harness/src/context/types.rs +++ b/crates/tinyagents-harness/src/context/types.rs @@ -488,6 +488,14 @@ pub struct RunContext { /// Runtime-owned terminal lifecycle callback, consumed exactly once by the /// agent-loop guard even when the driving future is cancelled or dropped. pub(crate) terminal_observer: Option, + /// Lifecycle cursor over the working transcript: which messages were + /// announced and which turn is open (see `agent_loop::lifecycle`). + pub(crate) turns: crate::agent_loop::TurnTracker, + /// Set by a no-progress / repeat guard that paused the run, holding its + /// root-cause summary, so the loop can report `TerminalReason::Halted`. + pub(crate) halted_by_guard: Option, + /// The kind of the most recent `LimitReached` event this run emitted. + pub(crate) last_limit: std::sync::Mutex>, /// The [`CallId`] the agent loop minted for the model call currently in /// flight through the model-wrap middleware onion, mirroring /// [`crate::events::HarnessRunStatus::active_model_call`]. @@ -501,6 +509,14 @@ pub struct RunContext { /// `None` outside that window, and always `None` for a caller that never /// goes through the agent loop. pub active_model_call: Option, + pub(crate) provider_started: bool, + /// Whether the *current* model call reached the provider; reset when a + /// call begins (see `begin_model_call`). + pub(crate) call_provider_started: bool, + /// Set when a model call returned an error, so the terminal classifier + /// still knows the failure surfaced inside the provider call after + /// `active_model_call` was cleared. + pub(crate) model_call_failed: bool, /// Whether the model call most recently dispatched by the agent loop used /// the streaming path. Set by the loop's innermost model call before the /// wrap middleware sees the result, so a middleware can tell that diff --git a/crates/tinyagents-harness/src/events/README.md b/crates/tinyagents-harness/src/events/README.md index b01d81df5..82232d831 100644 --- a/crates/tinyagents-harness/src/events/README.md +++ b/crates/tinyagents-harness/src/events/README.md @@ -25,6 +25,10 @@ snapshot rather than a stream. sub-agent recursion. Most `*Started`/`*Completed` pairs also have a `*Failed` terminal partner so an exporter pairing calls by id never sees an open span for a call that actually errored. + Run boundaries carry a typed [`TerminalOutcome`](crate::terminal::TerminalOutcome) on + `RunCompleted`/`RunFailed`, and `TurnStarted`/`TurnCompleted`/ + `MessageAppended` let a consumer mirror the transcript turn by turn; see + `docs/modules/harness/terminal-outcome.md`. - [`EventRecord`] — an [`AgentEvent`] paired with a stable [`EventId`] and a monotonic stream `offset`. - [`EventListener`] — the `Send + Sync` trait a pluggable observer diff --git a/crates/tinyagents-harness/src/events/mod_tests.rs b/crates/tinyagents-harness/src/events/mod_tests.rs index fbf864310..17adf2d8f 100644 --- a/crates/tinyagents-harness/src/events/mod_tests.rs +++ b/crates/tinyagents-harness/src/events/mod_tests.rs @@ -57,6 +57,7 @@ fn smoke_event_sink_records_events() { let _ = sink.emit(AgentEvent::RunCompleted { run_id: run_id.clone(), + outcome: None, }); assert_eq!(recorder.len(), 2); @@ -102,6 +103,7 @@ fn smoke_event_journal_replay() { }); journal.append(AgentEvent::RunCompleted { run_id: run_id.clone(), + outcome: None, }); assert_eq!(journal.len(), 2); @@ -589,6 +591,64 @@ fn emit_delivers_normally_once_a_listener_subscribes_after_a_quiet_run() { assert_eq!(recorder.events()[0].offset, 1); } +#[test] +fn terminal_and_lifecycle_events_have_stable_kinds() { + assert_eq!(AgentEvent::TurnStarted { turn: 1 }.kind(), "turn.started"); + assert_eq!( + AgentEvent::TurnCompleted { + turn: 1, + tool_result_count: 0, + tool_call_ids: vec![] + } + .kind(), + "turn.completed" + ); + assert_eq!( + AgentEvent::MessageAppended { + role: "user".into(), + index: 0, + call_id: None, + message: None + } + .kind(), + "message.appended" + ); +} + +#[test] +fn run_events_with_outcomes_round_trip_and_old_payloads_still_parse() { + use crate::terminal::{TerminalOutcome, TerminalReason}; + let failed = AgentEvent::RunFailed { + run_id: RunId::new("r"), + error: "boom".into(), + outcome: Some(TerminalOutcome::new(TerminalReason::ToolFailed, "boom")), + }; + let json = serde_json::to_value(&failed).unwrap(); + assert_eq!(json["outcome"]["reason"], "tool_failed"); + assert_eq!(serde_json::from_value::(json).unwrap(), failed); + + // Journals written before the field existed deserialize with `None`. + let old: AgentEvent = + serde_json::from_value(serde_json::json!({"kind": "run_completed", "run_id": "r"})) + .unwrap(); + assert!(matches!( + old, + AgentEvent::RunCompleted { outcome: None, .. } + )); + let old: AgentEvent = serde_json::from_value( + serde_json::json!({"kind": "queued_message_applied", "lane": "steer", "count": 2}), + ) + .unwrap(); + assert!(matches!( + old, + AgentEvent::QueuedMessageApplied { + count: 2, + first_index: 0, + .. + } + )); +} + #[test] fn tool_events_serialise_parent_call_id_only_when_nested() { use crate::ids::CallId; diff --git a/crates/tinyagents-harness/src/events/types.rs b/crates/tinyagents-harness/src/events/types.rs index e9162b154..8549c84b8 100644 --- a/crates/tinyagents-harness/src/events/types.rs +++ b/crates/tinyagents-harness/src/events/types.rs @@ -710,13 +710,94 @@ pub enum AgentEvent { /// finish, `Followup` at a natural finish. Emitted once per boundary /// with the number of messages applied; `Collect` items never produce /// this event because they are not applied to the transcript. Payload - /// text is deliberately not carried (events are payload-free by default). + /// text is carried only under the capture policy (see `messages`). QueuedMessageApplied { /// Which lane the messages came from. lane: crate::run_queue::QueueLane, /// How many messages were appended at this boundary (`1` under /// [`QueueMode::OneAtATime`][crate::run_queue::QueueMode::OneAtATime]). count: usize, + /// Transcript index of the first applied message; the applied messages + /// occupy `first_index..first_index + count`. + #[serde(default)] + first_index: usize, + /// The applied messages, serialized, captured per message: `tool` + /// messages when + /// [`PayloadCapture::tool_io`][crate::runtime::PayloadCapture::tool_io] + /// is enabled, all others when + /// [`PayloadCapture::model_io`][crate::runtime::PayloadCapture::model_io] + /// is. One slot per applied message (`null` for an uncaptured one) so + /// slots line up with `first_index..first_index + count`; empty when + /// nothing in the batch is captured, including the default payload-free + /// mode. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + messages: Vec, + }, + + /// A model turn began: the loop is about to dispatch the model call + /// numbered `turn`. A turn is one model call plus the tool batch it + /// requested; `turn` is 1-based and counts model-call attempts, so a + /// recovery retry of an unusable reply opens a new turn. Paired with + /// [`AgentEvent::TurnCompleted`]. + TurnStarted { + /// 1-based turn number within the run. + turn: u32, + }, + + /// A model turn ended: its tool batch (if any) has been folded into the + /// transcript, or the turn produced the final answer, or the run ended + /// mid-turn. Always follows a [`AgentEvent::TurnStarted`] with the same + /// `turn`. + TurnCompleted { + /// 1-based turn number within the run. + turn: u32, + /// How many tool-result messages the turn added to the transcript. + tool_result_count: usize, + /// The call ids those tool results answer, in transcript order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_call_ids: Vec, + }, + + /// A message was appended to the run's working transcript (assistant + /// reply, tool result, nudge, steering injection, queued message, ...). + /// Emitted in transcript order at turn boundaries and run exit, so a + /// consumer can mirror the transcript from events alone. The seed input + /// messages are not announced. + MessageAppended { + /// Message role: `system`, `user`, `assistant`, `tool` or `custom`. + role: String, + /// Position of the message in the working transcript. + index: usize, + /// For `tool` messages, the call id the result answers. + #[serde(default, skip_serializing_if = "Option::is_none")] + call_id: Option, + /// The serialized message, captured only when the capture policy + /// allows it (`model_io`, or `tool_io` for `tool` messages). `None` in + /// the default payload-free mode. + #[serde(default, skip_serializing_if = "Option::is_none")] + message: Option, + }, + + /// The message at `index` (and any announced after it) was removed from the + /// working transcript — for example an unusable assistant reply dropped + /// before a retry or a recovery nudge. Emitted highest index first, so + /// applying them in order to a mirror is a sequence of pops. Always + /// precedes the [`AgentEvent::MessageAppended`] of whatever replaces it. + MessageRetracted { + /// Position the removed message held in the working transcript. + index: usize, + }, + + /// The working transcript was rewritten in place (not by appending or + /// popping): a tool-set change folded into, or inserted before, the + /// leading system message. The transcript now holds `len` messages; a + /// mirror should treat its copy as stale and resynchronise. Later + /// [`AgentEvent::MessageAppended`] indices count from this new length. + TranscriptRewritten { + /// Message count after the rewrite. + len: usize, + /// Why it was rewritten (a stable snake_case label). + reason: String, }, /// A graph routing decision produced a named route. @@ -908,10 +989,16 @@ pub enum AgentEvent { /// without correlating against [`AgentEvent::ModelCompleted`]. StreamClosed, - /// A harness run finished successfully. + /// A harness run finished: it returned a result to the caller. A run + /// that stopped on a `StopWithPartial` cap also completes; its `outcome` + /// says so (`LimitReached`). RunCompleted { /// Identifier for the run that completed. run_id: RunId, + /// How the run ended, structured. `None` for events produced before + /// this field existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + outcome: Option, }, /// A harness run ended with an unrecoverable error. @@ -920,6 +1007,10 @@ pub enum AgentEvent { run_id: RunId, /// Human-readable error description. error: String, + /// Why the run failed, structured. `outcome.message` mirrors `error`. + /// `None` for events produced before this field existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + outcome: Option, }, } @@ -1000,6 +1091,11 @@ impl AgentEvent { AgentEvent::Compacted { .. } => "context.compacted", AgentEvent::OutputRetry { .. } => "output.retry", AgentEvent::QueuedMessageApplied { .. } => "queue.applied", + AgentEvent::TurnStarted { .. } => "turn.started", + AgentEvent::TurnCompleted { .. } => "turn.completed", + AgentEvent::MessageAppended { .. } => "message.appended", + AgentEvent::MessageRetracted { .. } => "message.retracted", + AgentEvent::TranscriptRewritten { .. } => "transcript.rewritten", AgentEvent::RouteSelected { .. } => "route.selected", AgentEvent::UsageRecorded { .. } => "usage.recorded", AgentEvent::CostRecorded { .. } => "cost.recorded", diff --git a/crates/tinyagents-harness/src/lib.rs b/crates/tinyagents-harness/src/lib.rs index 28654878f..818130db0 100644 --- a/crates/tinyagents-harness/src/lib.rs +++ b/crates/tinyagents-harness/src/lib.rs @@ -93,6 +93,7 @@ pub mod store; pub mod stream; pub mod structured; pub mod summarization; +pub mod terminal; pub mod testkit; pub mod title; pub mod token_estimation; @@ -154,4 +155,5 @@ pub use steering::{ RecentRequestIds, RequestIdError, SteeringCommand, SteeringCommandKind, SteeringHandle, SteeringOutcome, SteeringPolicy, }; +pub use terminal::{TerminalClass, TerminalOutcome, TerminalReason, TimeoutPhase}; pub use tool::ToolRegistry; diff --git a/crates/tinyagents-harness/src/middleware/library/repeat_escalation_loop_tests.rs b/crates/tinyagents-harness/src/middleware/library/repeat_escalation_loop_tests.rs index 111680b0b..2595e8062 100644 --- a/crates/tinyagents-harness/src/middleware/library/repeat_escalation_loop_tests.rs +++ b/crates/tinyagents-harness/src/middleware/library/repeat_escalation_loop_tests.rs @@ -164,3 +164,37 @@ async fn a_later_registered_after_tool_sees_the_marker_on_refused_calls() { "executed results are unmarked; refused ones are marked for every hook" ); } + +#[tokio::test] +async fn a_guard_halt_is_reported_as_a_halted_terminal_outcome() { + use crate::terminal::{TerminalClass, TerminalReason}; + let steering = SteeringHandle::allow_all(); + let summary: HaltSummarySlot = Arc::new(Mutex::new(None)); + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses( + (0..12).map(repeat_call).collect::>(), + )), + ); + harness.register_tool(Arc::new(CountingTool { + runs: Mutex::new(0), + })); + harness.push_middleware(Arc::new(RepeatProgressMiddleware::new( + steering.clone(), + summary.clone(), + Arc::new(|_| false), + ))); + let ctx = RunContext::new(RunConfig::new("repeat-halted"), ()).with_steering(steering); + let run = harness + .invoke_in_context_with_status(&(), ctx, vec![Message::user("go")]) + .await + .unwrap() + .run; + + assert!(run.paused.is_some(), "a halt still pauses the run"); + let outcome = run.terminal.expect("outcome"); + assert_eq!(outcome.reason, TerminalReason::Halted); + assert_eq!(outcome.class, TerminalClass::Failure); + assert_eq!(Some(outcome.message), summary.lock().unwrap().clone()); +} diff --git a/crates/tinyagents-harness/src/middleware/library/repeat_progress.rs b/crates/tinyagents-harness/src/middleware/library/repeat_progress.rs index 9174072fc..df01af13c 100644 --- a/crates/tinyagents-harness/src/middleware/library/repeat_progress.rs +++ b/crates/tinyagents-harness/src/middleware/library/repeat_progress.rs @@ -175,7 +175,10 @@ impl RepeatProgressMiddleware { /// Latch a root-cause halt: record the summary the turn surfaces instead of an /// empty/last-model reply, and pause at the top of the next iteration (before /// the next model call), matching the repeated-failure breaker's halt path. - fn halt(&self, summary: String) { + fn halt(&self, ctx: &mut RunContext, summary: String) { + // Mark the run so the loop reports `TerminalReason::Halted` for the + // pause this causes, not a plain steering pause. + ctx.halted_by_guard = Some(summary.clone()); *lock(&self.halt_summary) = Some(summary); self.handle.send(SteeringCommand::Pause); } @@ -339,7 +342,7 @@ impl Middleware<(), C> for RepeatProgressMiddleware { .get_mut(&run_id) .is_none_or(|batch| !std::mem::replace(&mut batch.halted, true)); if first { - self.halt(summary.clone()); + self.halt(ctx, summary.clone()); } Err(TinyAgentsError::ToolFailed(summary)) } @@ -447,7 +450,7 @@ impl Middleware<(), C> for RepeatProgressMiddleware { tool = tool_name, "[tinyagents::mw] crate successful-repeat tracker halted the run" ); - self.halt(summary); + self.halt(ctx, summary); Ok(()) } } diff --git a/crates/tinyagents-harness/src/middleware/types.rs b/crates/tinyagents-harness/src/middleware/types.rs index d7d4a1bdd..f4ffc2d4f 100644 --- a/crates/tinyagents-harness/src/middleware/types.rs +++ b/crates/tinyagents-harness/src/middleware/types.rs @@ -157,6 +157,14 @@ pub struct AgentRun { /// (and re-summarizing) everything the compaction already folded. Set by /// [`ContextCompressionMiddleware`]'s `after_agent` hook. pub compacted_history: Option>, + /// How the run ended, structured (see [`crate::terminal`]). Set by the + /// agent loop on every exit path it controls — completion, a + /// `StopWithPartial` cap, a pause, a deferral — and by the driver on + /// failure, so a host reading a partial run (for example from + /// [`PartialRunOutcome`][crate::agent_loop::PartialRunOutcome]) needs no + /// string parsing. `None` only while the run is still in flight, or when a + /// wrapping middleware replaced the loop. + pub terminal: Option, } /// Host-only metadata one tool call returned, as recorded on diff --git a/crates/tinyagents-harness/src/observability/langfuse/mod_tests.rs b/crates/tinyagents-harness/src/observability/langfuse/mod_tests.rs index 874ccdf2b..3478ed36e 100644 --- a/crates/tinyagents-harness/src/observability/langfuse/mod_tests.rs +++ b/crates/tinyagents-harness/src/observability/langfuse/mod_tests.rs @@ -509,6 +509,7 @@ fn run_span_carries_run_error_and_window() { AgentEvent::RunFailed { run_id: RunId::new("run-1"), error: "boom".to_string(), + outcome: None, }, ), ], diff --git a/crates/tinyagents-harness/src/observability/mod_tests.rs b/crates/tinyagents-harness/src/observability/mod_tests.rs index 6ad8bd3fc..a185a700d 100644 --- a/crates/tinyagents-harness/src/observability/mod_tests.rs +++ b/crates/tinyagents-harness/src/observability/mod_tests.rs @@ -46,6 +46,7 @@ async fn in_memory_journal_append_read_round_trip() { 1, AgentEvent::RunCompleted { run_id: RunId::new("run-1"), + outcome: None, }, ); @@ -157,6 +158,7 @@ fn agent_latency_metrics_include_model_tool_and_run_elapsed() { 90, AgentEvent::RunCompleted { run_id: run_id.clone(), + outcome: None, }, ), ]; @@ -241,6 +243,7 @@ async fn journal_window_and_filter_reads() { 2, AgentEvent::RunCompleted { run_id: RunId::new("run-1"), + outcome: None, }, )) .await @@ -569,6 +572,7 @@ fn redacting_sink_masks_secret_substrings() { event: AgentEvent::RunFailed { run_id: RunId::new("run-r"), error: "auth failed with key sk-SUPERSECRET and pw hunter2".to_string(), + outcome: None, }, }); @@ -625,6 +629,7 @@ fn redacting_sink_empty_secrets_forwards_unchanged() { event: AgentEvent::RunFailed { run_id: RunId::new("run-r"), error: "nothing to redact here".to_string(), + outcome: None, }, }); diff --git a/crates/tinyagents-harness/src/retry/types.rs b/crates/tinyagents-harness/src/retry/types.rs index 1eb9cbfcd..ea3c48caf 100644 --- a/crates/tinyagents-harness/src/retry/types.rs +++ b/crates/tinyagents-harness/src/retry/types.rs @@ -222,7 +222,8 @@ pub(crate) struct RateLimiterState { } /// Why a model call failed, as far as failover policy is concerned. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum FailoverReason { /// Credentials were rejected (`401`/`403`, invalid key). May be fixed by a /// credential refresh, so it is not remembered across calls. diff --git a/crates/tinyagents-harness/src/runtime/agent.rs b/crates/tinyagents-harness/src/runtime/agent.rs index 691f37465..2b296ac93 100644 --- a/crates/tinyagents-harness/src/runtime/agent.rs +++ b/crates/tinyagents-harness/src/runtime/agent.rs @@ -186,8 +186,13 @@ fn hosted_error_message(kind: HostedErrorKind) -> &'static str { /// Builds a [`HostedError`] from the raw loop error and whatever partial /// [`AgentRun`] the loop accumulated before failing. -fn hosted_error(error: &TinyAgentsError, run: AgentRun) -> HostedError { +fn hosted_error(error: &TinyAgentsError, mut run: AgentRun) -> HostedError { let kind = classify_hosted_error(error); + // The typed outcome mirrors the raw error text; keep its classification + // and replace the detail with the fixed, sanitized message. + if let Some(outcome) = run.terminal.as_mut() { + outcome.message = hosted_error_message(kind).to_string(); + } HostedError { kind, message: hosted_error_message(kind).to_string(), @@ -424,8 +429,13 @@ impl Stream for AgentStream<'_, /// typed details for host-owned diagnostics and policy decisions. fn sanitize_hosted_stream_item(mut item: AgentStreamItem) -> AgentStreamItem { match &mut item { - AgentStreamItem::Failed { error, .. } => { + AgentStreamItem::Failed { error, run } => { *error = "hosted agent invocation failed".to_string(); + // The partial run carries its typed outcome; keep the + // classification, drop the raw failure text. + if let Some(outcome) = run.terminal.as_mut() { + outcome.message = error.clone(); + } } AgentStreamItem::Event(record) => sanitize_hosted_event(record), AgentStreamItem::Completed(_) => {} @@ -462,8 +472,13 @@ fn sanitize_hosted_event(record: &mut EventRecord) { AgentEvent::MiddlewareFailed { error, .. } => { *error = "hosted middleware failed".to_string(); } - AgentEvent::RunFailed { error, .. } => { + AgentEvent::RunFailed { error, outcome, .. } => { *error = "hosted agent invocation failed".to_string(); + // The typed outcome mirrors the raw error text; keep its + // classification (reason, class, phase) and drop the detail. + if let Some(outcome) = outcome { + outcome.message = error.clone(); + } } _ => {} } diff --git a/crates/tinyagents-harness/src/runtime/mod_tests.rs b/crates/tinyagents-harness/src/runtime/mod_tests.rs index 3ab6d0ab5..f609f0758 100644 --- a/crates/tinyagents-harness/src/runtime/mod_tests.rs +++ b/crates/tinyagents-harness/src/runtime/mod_tests.rs @@ -3580,3 +3580,48 @@ fn take_one(counter: &AtomicUsize) -> bool { } false } + +#[tokio::test] +async fn hosted_error_run_terminal_outcome_message_is_sanitized() { + struct LeakyModel; + #[async_trait] + impl tinyinference_llm::model::ChatModel<()> for LeakyModel { + async fn invoke( + &self, + _: &(), + _: tinyinference_llm::model::ModelRequest, + ) -> tinyinference_llm::Result { + Err(tinyinference_llm::Error::Model( + "secret-provider-detail".into(), + )) + } + } + let host = crate::host::HostCapabilities::new( + Arc::new(StaticContextComposer::empty()), + Arc::new(InMemoryDefinitionRegistry::new(vec![AgentDefinition::new( + "helper", + "Helper", + "test helper", + )])), + Arc::new(AllowAllSecurityGate), + Arc::new(FixedModelResolver::new(Arc::new(LeakyModel))), + ); + let harness: AgentHarness<()> = AgentHarness::new(); + let error = harness + .invoke_agent( + AgentInvocation::new( + host, + AgentTurnRequest::new( + "helper", + vec![tinyinference_llm::message::Message::user("hi")], + ), + RunContext::new(RunConfig::new("leaky"), ()), + ), + &(), + ) + .await + .expect_err("the provider fails"); + let run = error.run.expect("partial run"); + let outcome = run.terminal.expect("typed outcome"); + assert!(!outcome.message.contains("secret"), "{}", outcome.message); +} diff --git a/crates/tinyagents-harness/src/stream/mod_tests.rs b/crates/tinyagents-harness/src/stream/mod_tests.rs index c6e1120a9..3a3b34c0c 100644 --- a/crates/tinyagents-harness/src/stream/mod_tests.rs +++ b/crates/tinyagents-harness/src/stream/mod_tests.rs @@ -321,6 +321,7 @@ mod project { AgentEvent::MemorySaved, AgentEvent::RunCompleted { run_id: RunId::new("r1"), + outcome: None, }, ] { let mode = projected_mode(&event); diff --git a/crates/tinyagents-harness/src/terminal.rs b/crates/tinyagents-harness/src/terminal.rs new file mode 100644 index 000000000..1045a7913 --- /dev/null +++ b/crates/tinyagents-harness/src/terminal.rs @@ -0,0 +1,338 @@ +//! Typed terminal outcome of a run. +//! +//! A run used to end as one of three loosely related shapes: a +//! [`LoopExit`](crate::agent_loop) inside the loop, a +//! [`TinyAgentsError`] on the failure channel, and a free-form `String` on +//! [`AgentEvent::RunFailed`](crate::events::AgentEvent::RunFailed). A host that +//! wanted to know *why* a run ended — a deadline, a cancel, a provider outage, +//! a tripped guard — had to parse the string. [`TerminalOutcome`] is the one +//! structured answer, and this module is the one place that maps the loop's +//! exits and errors onto it. +//! +//! The legacy string is kept everywhere the outcome is attached +//! ([`TerminalOutcome::message`] mirrors it), so existing hosts are unaffected. +//! +//! # Mapping +//! +//! | Source | [`TerminalReason`] | [`TerminalClass`] | +//! | --- | --- | --- | +//! | model finished / `StopWithFinal` | `Completed` | `Success` | +//! | `StopWithPartial` call cap | `LimitReached(kind)` | `Failure` | +//! | `TinyAgentsError::Cancelled` | `Cancelled` | `Cancellation` | +//! | `TinyAgentsError::Timeout` (run deadline) | `Timeout` | `Timeout` | +//! | `TinyAgentsError::CallTimeout` (one call wedged) | `ProviderFailed(Some(Timeout))` | `Timeout` | +//! | provider / model / overflow / empty-response errors | `ProviderFailed(reason)` | `Failure` | +//! | tool errors | `ToolFailed` | `Failure` | +//! | run caps, depth caps | `LimitReached(..)` | `Failure` | +//! | steering pause | `Paused` | `Suspended` | +//! | deferred tool calls | `Deferred` | `Suspended` | +//! | repeat / no-progress guard | `Halted` ([`TerminalOutcome::halted`]) | `Failure` | +//! | anything else | `Internal` | `Failure` | +//! +//! # Precedence +//! +//! When two outcomes compete for one run — a cancel races a deadline, a tool +//! failure surfaces while a limit trips — [`TerminalOutcome::merge`] keeps the +//! more authoritative one. Highest first: +//! +//! 1. external cancel (`Cancelled`) +//! 2. run deadline (`Timeout`) +//! 3. idle / provider-call timeout (`ProviderFailed(Some(Timeout))`) +//! 4. limits (`LimitReached`) +//! 5. no-progress / repeat guard (`Halted`) +//! 6. other failures (`ProviderFailed`, `ToolFailed`, `Internal`) +//! 7. suspension (`Paused`, `Deferred`) +//! 8. success (`Completed`) +//! +//! Ties keep the outcome already held (the earlier one). The merged outcome's +//! `provider_started` is the OR of both, so "did the provider ever run" is not +//! lost when a higher-precedence outcome wins. + +use serde::{Deserialize, Serialize}; + +use crate::agent_loop::LoopExit; +use crate::error::TinyAgentsError; +use crate::events::LimitKind; +use crate::retry::FailoverReason; + +/// Broad family of a [`TerminalReason`], for hosts that only branch on +/// success / timeout / cancel / failure / suspended. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +pub enum TerminalClass { + /// The run reached a final answer. + Success, + /// A deadline or per-call timeout ended the run. + Timeout, + /// The run was cancelled from outside. + Cancellation, + /// The run ended without a usable result for any other reason. + Failure, + /// The run is resumable, not finished (paused or deferred). + Suspended, +} + +/// Where in the run a timeout landed. +/// +/// Also used as the *site* a failure surfaced at ([`TerminalOutcome::from_error`]): +/// [`TerminalOutcome::from_error`] derives `provider_started` as `false` exactly +/// for [`TimeoutPhase::BeforeProvider`]; the loop drivers then overwrite it with +/// the run-wide dispatch history (a later call rejected before dispatch still +/// reports that an earlier call reached the provider). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +pub enum TimeoutPhase { + /// No model call had started yet (preflight, host preparation, resolution). + BeforeProvider, + /// A provider call was in flight. + Provider, + /// At least one provider call had started and none was in flight: a turn + /// boundary, a tool batch, or post-turn middleware. + AfterTurn, +} + +/// Why a run ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +pub enum TerminalReason { + /// The model produced a final answer. + Completed, + /// A run cap tripped; `None` when the cap's kind is not known. + LimitReached(Option), + /// The run's own wall-clock deadline elapsed. + Timeout, + /// The run was cancelled from outside. + Cancelled, + /// Steering latched a pause; the run is resumable. + Paused, + /// Tool calls are waiting on a human decision or host execution. + Deferred, + /// A no-progress / repeat guard stopped the run. + Halted, + /// The model provider failed, classified when possible. + ProviderFailed(Option), + /// A tool failed in a way that ended the run. + ToolFailed, + /// Anything the other reasons do not name. + Internal, +} + +impl TerminalReason { + /// The [`TerminalClass`] this reason belongs to. + pub fn class(self) -> TerminalClass { + match self { + Self::Completed => TerminalClass::Success, + Self::Timeout | Self::ProviderFailed(Some(FailoverReason::Timeout)) => { + TerminalClass::Timeout + } + Self::Cancelled => TerminalClass::Cancellation, + Self::Paused | Self::Deferred => TerminalClass::Suspended, + Self::LimitReached(_) + | Self::Halted + | Self::ProviderFailed(_) + | Self::ToolFailed + | Self::Internal => TerminalClass::Failure, + } + } + + /// Precedence rank used by [`TerminalOutcome::merge`]; lower wins. + fn rank(&self) -> u8 { + match self { + Self::Cancelled => 0, + Self::Timeout => 1, + Self::ProviderFailed(Some(FailoverReason::Timeout)) => 2, + Self::LimitReached(_) => 3, + Self::Halted => 4, + Self::ProviderFailed(_) | Self::ToolFailed | Self::Internal => 5, + Self::Paused | Self::Deferred => 6, + Self::Completed => 7, + } + } +} + +/// The structured answer to "how and why did this run end". +/// +/// Attached to [`AgentEvent::RunCompleted`](crate::events::AgentEvent::RunCompleted), +/// [`AgentEvent::RunFailed`](crate::events::AgentEvent::RunFailed) and +/// [`AgentRun::terminal`](crate::middleware::AgentRun::terminal). See the +/// [module docs](self) for the mapping and the merge precedence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct TerminalOutcome { + /// Why the run ended. + pub reason: TerminalReason, + /// The broad family of [`Self::reason`]; always `reason.class()`. + pub class: TerminalClass, + /// Where the timeout landed; `Some` only for timeout-class outcomes built + /// from an error. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_phase: Option, + /// Whether any provider call had started before the run ended. + #[serde(default)] + pub provider_started: bool, + /// Human-readable description; mirrors the legacy `RunFailed::error` + /// string for failures. + pub message: String, +} + +impl TerminalOutcome { + /// Builds an outcome for `reason`, deriving the class from it. + pub fn new(reason: TerminalReason, message: impl Into) -> Self { + Self { + reason, + class: reason.class(), + timeout_phase: None, + provider_started: false, + message: message.into(), + } + } + + /// A successful completion. + pub fn completed() -> Self { + Self::new(TerminalReason::Completed, "") + } + + /// A run stopped by a no-progress / repeat guard, with the guard's summary. + pub fn halted(message: impl Into) -> Self { + Self::new(TerminalReason::Halted, message) + } + + /// A tripped run cap. [`LimitKind::WallClock`] is the run deadline, so it + /// maps to [`TerminalReason::Timeout`] rather than a limit. + pub fn limit_reached(kind: Option, message: impl Into) -> Self { + match kind { + Some(LimitKind::WallClock) => Self::new(TerminalReason::Timeout, message), + kind => Self::new(TerminalReason::LimitReached(kind), message), + } + } + + /// Fills in the kind of a [`TerminalReason::LimitReached`] whose kind was + /// not known when it was classified (a `LimitExceeded` error carries only + /// text). Other outcomes are returned unchanged. + pub fn with_limit_kind(mut self, kind: Option) -> Self { + if self.reason == TerminalReason::LimitReached(None) { + self.reason = match kind { + Some(LimitKind::WallClock) => TerminalReason::Timeout, + kind => TerminalReason::LimitReached(kind), + }; + self.class = self.reason.class(); + } + self + } + + /// Records whether a provider call had started. + pub fn with_provider_started(mut self, started: bool) -> Self { + self.provider_started = started; + self + } + + /// Records where a timeout landed. + pub fn with_timeout_phase(mut self, phase: TimeoutPhase) -> Self { + self.timeout_phase = Some(phase); + self + } + + /// Classifies `error`. `site` is where the run stood when the error + /// surfaced; it sets `provider_started` and, for timeouts, the phase. + pub fn from_error(error: &TinyAgentsError, site: TimeoutPhase) -> Self { + use TinyAgentsError as E; + let message = error.to_string(); + let provider_started = site != TimeoutPhase::BeforeProvider; + let outcome = match error { + E::Cancelled => Self::new(TerminalReason::Cancelled, message), + E::Timeout(_) => Self::new(TerminalReason::Timeout, message).with_timeout_phase(site), + E::CallTimeout(_) => Self::new( + TerminalReason::ProviderFailed(Some(FailoverReason::Timeout)), + message, + ) + .with_timeout_phase(site), + E::Provider(_) + | E::Model(_) + | E::ContextOverflow { .. } + | E::ModelNotFound(_) + | E::EmptyResponse + | E::GenerationStalled + | E::SummarizationUsage { .. } => { + // A summarizer failure wraps the real error; classify that. + let reason = match error { + E::SummarizationUsage { error: inner, .. } => FailoverReason::classify(inner), + other => FailoverReason::classify(other), + }; + let outcome = Self::new(TerminalReason::ProviderFailed(Some(reason)), message); + if reason == FailoverReason::Timeout { + outcome.with_timeout_phase(site) + } else { + outcome + } + } + E::Tool(_) | E::ToolFailed(_) | E::ToolNotFound(_) | E::ModelRetry(_) => { + Self::new(TerminalReason::ToolFailed, message) + } + E::LimitExceeded(_) + | E::RecursionLimit(_) + | E::SubAgentDepth(_) + | E::NodeVisitLimit { .. } => Self::new(TerminalReason::LimitReached(None), message), + E::ApprovalRequired { .. } | E::CallDeferred { .. } => { + Self::new(TerminalReason::Deferred, message) + } + E::Interrupted { node, .. } if node == "steering-pause" => { + Self::new(TerminalReason::Paused, message) + } + E::Interrupted { .. } => Self::new(TerminalReason::Internal, message), + _ => Self::new(TerminalReason::Internal, message), + }; + // A provider-call timeout implies a provider call started. + let started = provider_started || outcome.timeout_phase == Some(TimeoutPhase::Provider); + outcome.with_provider_started(started) + } + + /// Maps the loop's own deliberate exits. `provider_started` is whether any + /// model call had been dispatched. + pub(crate) fn from_loop_exit(exit: &LoopExit, provider_started: bool) -> Self { + let outcome = match exit { + LoopExit::Finished => Self::completed(), + LoopExit::LimitStop(kind) => Self::limit_reached( + Some(*kind), + format!( + "stopped with the partial run: {} limit reached", + kind.as_str() + ), + ), + LoopExit::Paused(pause) => Self::new( + TerminalReason::Paused, + pause.reason.clone().unwrap_or_else(|| { + format!("paused at checkpoint {}", pause.paused_at_checkpoint) + }), + ), + LoopExit::Deferred(requests) => Self::new( + TerminalReason::Deferred, + format!( + "{} approval(s), {} external call(s) pending", + requests.approvals.len(), + requests.calls.len() + ), + ), + }; + outcome.with_provider_started(provider_started) + } + + /// Merges two competing outcomes, keeping the one with higher precedence + /// (see the [module docs](self)). `self` wins ties. `provider_started` is + /// the OR of both. + pub fn merge(self, other: Self) -> Self { + let started = self.provider_started || other.provider_started; + let winner = if other.reason.rank() < self.reason.rank() { + other + } else { + self + }; + winner.with_provider_started(started) + } +} + +#[cfg(test)] +#[path = "terminal_tests.rs"] +mod tests; diff --git a/crates/tinyagents-harness/src/terminal_tests.rs b/crates/tinyagents-harness/src/terminal_tests.rs new file mode 100644 index 000000000..d1f54a3fb --- /dev/null +++ b/crates/tinyagents-harness/src/terminal_tests.rs @@ -0,0 +1,283 @@ +use super::*; +use crate::steering::PauseState; +use crate::tool::DeferredToolRequests; + +fn out(reason: TerminalReason) -> TerminalOutcome { + TerminalOutcome::new(reason, "m") +} + +#[test] +fn class_follows_reason() { + use TerminalReason::*; + assert_eq!(Completed.class(), TerminalClass::Success); + assert_eq!(Cancelled.class(), TerminalClass::Cancellation); + assert_eq!(Timeout.class(), TerminalClass::Timeout); + assert_eq!( + ProviderFailed(Some(FailoverReason::Timeout)).class(), + TerminalClass::Timeout + ); + assert_eq!( + ProviderFailed(Some(FailoverReason::RateLimit)).class(), + TerminalClass::Failure + ); + assert_eq!(ProviderFailed(None).class(), TerminalClass::Failure); + assert_eq!(Paused.class(), TerminalClass::Suspended); + assert_eq!(Deferred.class(), TerminalClass::Suspended); + for failure in [LimitReached(None), Halted, ToolFailed, Internal] { + assert_eq!(failure.class(), TerminalClass::Failure, "{failure:?}"); + } + assert_eq!(out(Halted).class, TerminalClass::Failure); +} + +#[test] +fn cancelled_error_maps_to_cancellation() { + let o = TerminalOutcome::from_error(&TinyAgentsError::Cancelled, TimeoutPhase::AfterTurn); + assert_eq!(o.reason, TerminalReason::Cancelled); + assert_eq!(o.class, TerminalClass::Cancellation); + assert_eq!(o.timeout_phase, None); + assert!(o.provider_started); + assert_eq!(o.message, "run cancelled"); +} + +#[test] +fn run_deadline_carries_phase_and_provider_started() { + let error = TinyAgentsError::Timeout("deadline".into()); + let before = TerminalOutcome::from_error(&error, TimeoutPhase::BeforeProvider); + assert_eq!(before.reason, TerminalReason::Timeout); + assert_eq!(before.class, TerminalClass::Timeout); + assert_eq!(before.timeout_phase, Some(TimeoutPhase::BeforeProvider)); + assert!(!before.provider_started); + + let during = TerminalOutcome::from_error(&error, TimeoutPhase::Provider); + assert_eq!(during.timeout_phase, Some(TimeoutPhase::Provider)); + assert!(during.provider_started); + + let after = TerminalOutcome::from_error(&error, TimeoutPhase::AfterTurn); + assert_eq!(after.timeout_phase, Some(TimeoutPhase::AfterTurn)); + assert!(after.provider_started); +} + +#[test] +fn call_timeout_is_a_provider_timeout() { + let o = TerminalOutcome::from_error( + &TinyAgentsError::CallTimeout("wedged".into()), + TimeoutPhase::Provider, + ); + assert_eq!( + o.reason, + TerminalReason::ProviderFailed(Some(FailoverReason::Timeout)) + ); + assert_eq!(o.class, TerminalClass::Timeout); + assert_eq!(o.timeout_phase, Some(TimeoutPhase::Provider)); + assert!( + o.provider_started, + "a call timeout inside a call means it started" + ); +} + +#[test] +fn provider_errors_carry_the_failover_reason() { + let o = TerminalOutcome::from_error( + &TinyAgentsError::Model("HTTP 429 too many requests".into()), + TimeoutPhase::Provider, + ); + assert_eq!( + o.reason, + TerminalReason::ProviderFailed(Some(FailoverReason::RateLimit)) + ); + assert_eq!(o.class, TerminalClass::Failure); + assert_eq!(o.timeout_phase, None); + + let empty = + TerminalOutcome::from_error(&TinyAgentsError::EmptyResponse, TimeoutPhase::Provider); + assert_eq!( + empty.reason, + TerminalReason::ProviderFailed(Some(FailoverReason::EmptyResponse)) + ); +} + +#[test] +fn tool_limit_and_internal_errors_map() { + let site = TimeoutPhase::AfterTurn; + let tool = TerminalOutcome::from_error(&TinyAgentsError::ToolFailed("x".into()), site); + assert_eq!(tool.reason, TerminalReason::ToolFailed); + let limit = TerminalOutcome::from_error(&TinyAgentsError::LimitExceeded("x".into()), site); + assert_eq!(limit.reason, TerminalReason::LimitReached(None)); + let depth = TerminalOutcome::from_error(&TinyAgentsError::SubAgentDepth(3), site); + assert_eq!(depth.reason, TerminalReason::LimitReached(None)); + let internal = TerminalOutcome::from_error(&TinyAgentsError::Middleware("x".into()), site); + assert_eq!(internal.reason, TerminalReason::Internal); + assert_eq!(internal.message, "middleware error: x"); +} + +#[test] +fn deferral_and_interrupt_errors_are_suspended() { + let site = TimeoutPhase::AfterTurn; + let deferred = TerminalOutcome::from_error( + &TinyAgentsError::ApprovalRequired { + metadata: serde_json::Value::Null, + }, + site, + ); + assert_eq!(deferred.class, TerminalClass::Suspended); + let paused = TerminalOutcome::from_error( + &TinyAgentsError::Interrupted { + node: "steering-pause".into(), + message: "m".into(), + }, + site, + ); + assert_eq!(paused.reason, TerminalReason::Paused); + let other = TerminalOutcome::from_error( + &TinyAgentsError::Interrupted { + node: "n".into(), + message: "m".into(), + }, + site, + ); + assert_eq!(other.reason, TerminalReason::Internal); +} + +#[test] +fn loop_exits_map() { + let finished = TerminalOutcome::from_loop_exit(&LoopExit::Finished, true); + assert_eq!(finished.reason, TerminalReason::Completed); + assert_eq!(finished.class, TerminalClass::Success); + assert!(finished.provider_started); + + let limit = TerminalOutcome::from_loop_exit(&LoopExit::LimitStop(LimitKind::ToolCalls), true); + assert_eq!( + limit.reason, + TerminalReason::LimitReached(Some(LimitKind::ToolCalls)) + ); + assert!(limit.message.contains("tool_calls")); + + let wall = TerminalOutcome::from_loop_exit(&LoopExit::LimitStop(LimitKind::WallClock), true); + assert_eq!( + wall.reason, + TerminalReason::Timeout, + "wall clock is the run deadline" + ); + assert_eq!(wall.class, TerminalClass::Timeout); + + let paused = TerminalOutcome::from_loop_exit( + &LoopExit::Paused(PauseState { + reason: Some("operator".into()), + paused_at_checkpoint: 2, + }), + false, + ); + assert_eq!(paused.reason, TerminalReason::Paused); + assert_eq!(paused.message, "operator"); + assert!(!paused.provider_started); + + let deferred = + TerminalOutcome::from_loop_exit(&LoopExit::Deferred(DeferredToolRequests::default()), true); + assert_eq!(deferred.reason, TerminalReason::Deferred); + assert_eq!(deferred.class, TerminalClass::Suspended); +} + +#[test] +fn merge_follows_the_documented_precedence() { + use TerminalReason::*; + // Highest first. + let order = [ + Cancelled, + Timeout, + ProviderFailed(Some(FailoverReason::Timeout)), + LimitReached(Some(LimitKind::ModelCalls)), + Halted, + ProviderFailed(Some(FailoverReason::RateLimit)), + Paused, + Completed, + ]; + for (i, high) in order.iter().enumerate() { + for low in &order[i + 1..] { + let a = out(*high).merge(out(*low)); + let b = out(*low).merge(out(*high)); + assert_eq!(a.reason, *high, "{high:?} over {low:?}"); + assert_eq!(b.reason, *high, "{high:?} over {low:?} (swapped)"); + } + } + // Failures share a rank. + for failure in [ProviderFailed(None), ToolFailed, Internal] { + assert_eq!(out(Halted).merge(out(failure)).reason, Halted); + } +} + +#[test] +fn merge_ties_keep_the_earlier_and_or_provider_started() { + let first = TerminalOutcome::new(TerminalReason::ToolFailed, "first"); + let second = + TerminalOutcome::new(TerminalReason::Internal, "second").with_provider_started(true); + let merged = first.merge(second); + assert_eq!(merged.message, "first"); + assert!(merged.provider_started, "provider_started is OR-ed"); + + let cancel = TerminalOutcome::new(TerminalReason::Cancelled, "c"); + let merged = cancel + .merge(TerminalOutcome::new(TerminalReason::Completed, "").with_provider_started(true)); + assert_eq!(merged.reason, TerminalReason::Cancelled); + assert!(merged.provider_started); +} + +#[test] +fn serde_round_trip_and_wire_shape() { + let o = TerminalOutcome::from_error( + &TinyAgentsError::Timeout("t".into()), + TimeoutPhase::BeforeProvider, + ); + let json = serde_json::to_value(&o).unwrap(); + assert_eq!(json["reason"], "timeout"); + assert_eq!(json["class"], "timeout"); + assert_eq!(json["timeout_phase"], "before_provider"); + assert_eq!(json["provider_started"], false); + assert_eq!(serde_json::from_value::(json).unwrap(), o); + + let limit = TerminalOutcome::limit_reached(Some(LimitKind::ModelCalls), "cap"); + let json = serde_json::to_value(&limit).unwrap(); + assert_eq!( + json["reason"], + serde_json::json!({"limit_reached": "model_calls"}) + ); + assert!(json.get("timeout_phase").is_none()); + assert_eq!( + serde_json::from_value::(json).unwrap(), + limit + ); + + let provider = out(TerminalReason::ProviderFailed(Some( + FailoverReason::RateLimit, + ))); + let json = serde_json::to_value(&provider).unwrap(); + assert_eq!( + json["reason"], + serde_json::json!({"provider_failed": "rate_limit"}) + ); +} + +#[test] +fn a_call_timeout_before_dispatch_keeps_the_pre_provider_phase() { + let error = TinyAgentsError::CallTimeout("resolver".into()); + let before = TerminalOutcome::from_error(&error, TimeoutPhase::BeforeProvider); + assert_eq!(before.timeout_phase, Some(TimeoutPhase::BeforeProvider)); + assert!(!before.provider_started); + let during = TerminalOutcome::from_error(&error, TimeoutPhase::Provider); + assert_eq!(during.timeout_phase, Some(TimeoutPhase::Provider)); + assert!(during.provider_started); + let after = TerminalOutcome::from_error(&error, TimeoutPhase::AfterTurn); + assert_eq!(after.timeout_phase, Some(TimeoutPhase::AfterTurn)); +} + +#[test] +fn a_summarizer_failure_is_classified_from_its_inner_error() { + let error = TinyAgentsError::SummarizationUsage { + error: Box::new(TinyAgentsError::Model("HTTP 429 too many requests".into())), + usage: Default::default(), + }; + let o = TerminalOutcome::from_error(&error, TimeoutPhase::AfterTurn); + assert_eq!( + o.reason, + TerminalReason::ProviderFailed(Some(FailoverReason::RateLimit)) + ); +} diff --git a/crates/tinyagents-harness/src/testkit/mod_tests.rs b/crates/tinyagents-harness/src/testkit/mod_tests.rs index 2198b0212..022b4d414 100644 --- a/crates/tinyagents-harness/src/testkit/mod_tests.rs +++ b/crates/tinyagents-harness/src/testkit/mod_tests.rs @@ -275,6 +275,7 @@ fn event_recorder_kinds() { }); sink.emit(AgentEvent::RunCompleted { run_id: RunId::new("r1"), + outcome: None, }); let kinds = recorder.kinds(); @@ -352,6 +353,7 @@ fn make_trajectory() -> Vec { }, AgentEvent::RunCompleted { run_id: RunId::new("r1"), + outcome: None, }, ] } @@ -444,6 +446,7 @@ fn trajectory_failed_is_true_when_run_failed_present() { let events = vec![AgentEvent::RunFailed { run_id: RunId::new("r1"), error: "oops".into(), + outcome: None, }]; let traj = Trajectory::from_events(events); assert!(traj.failed()); diff --git a/crates/tinyagents-harness/src/testkit/types.rs b/crates/tinyagents-harness/src/testkit/types.rs index 7de347a44..00f01744a 100644 --- a/crates/tinyagents-harness/src/testkit/types.rs +++ b/crates/tinyagents-harness/src/testkit/types.rs @@ -267,7 +267,7 @@ pub struct EventRecorder { /// input: None, /// output: None, /// }, -/// AgentEvent::RunCompleted { run_id: RunId::new("r1") }, +/// AgentEvent::RunCompleted { run_id: RunId::new("r1"), outcome: None }, /// ]; /// let traj = Trajectory::from_events(events); /// assert_eq!(traj.model_call_count(), 1); diff --git a/crates/tinyagents-integration-tests/tests/e2e_registry_observability_contracts.rs b/crates/tinyagents-integration-tests/tests/e2e_registry_observability_contracts.rs index 213ac131d..d7e286183 100644 --- a/crates/tinyagents-integration-tests/tests/e2e_registry_observability_contracts.rs +++ b/crates/tinyagents-integration-tests/tests/e2e_registry_observability_contracts.rs @@ -249,10 +249,12 @@ fn component_metadata_and_event_kinds_are_stable_serializable_contracts() { AgentEvent::StreamClosed, AgentEvent::RunCompleted { run_id: RunId::new("run-1"), + outcome: None, }, AgentEvent::RunFailed { run_id: RunId::new("run-2"), error: "bad".into(), + outcome: None, }, ]; let kinds: Vec<_> = events.iter().map(AgentEvent::kind).collect(); @@ -289,6 +291,7 @@ async fn event_sinks_journals_and_status_stores_preserve_run_lineage() { }); let second = sink.emit(AgentEvent::RunCompleted { run_id: RunId::new("run-1"), + outcome: None, }); assert_eq!(first.offset, 0); assert_eq!(second.offset, 1); diff --git a/crates/tinyagents-integration-tests/tests/feature_infra_observability.rs b/crates/tinyagents-integration-tests/tests/feature_infra_observability.rs index 6fb34a4e7..57652b8d0 100644 --- a/crates/tinyagents-integration-tests/tests/feature_infra_observability.rs +++ b/crates/tinyagents-integration-tests/tests/feature_infra_observability.rs @@ -104,6 +104,7 @@ fn latency_metrics_correlate_started_and_completed_by_call_id() { 1_500, AgentEvent::RunCompleted { run_id: RunId::new("r1"), + outcome: None, }, ), ]; @@ -361,6 +362,7 @@ fn redacting_sink_with_no_secrets_is_pass_through() { 0, AgentEvent::RunCompleted { run_id: RunId::new("r"), + outcome: None, }, )); assert_eq!(downstream.events().len(), 1); diff --git a/crates/tinyagents-integration-tests/tests/loop_as_graph.rs b/crates/tinyagents-integration-tests/tests/loop_as_graph.rs index 919e0ac36..525337612 100644 --- a/crates/tinyagents-integration-tests/tests/loop_as_graph.rs +++ b/crates/tinyagents-integration-tests/tests/loop_as_graph.rs @@ -66,9 +66,15 @@ fn harness_for(execution: LoopExecution, model: Arc) -> AgentHarness< /// Asserts `expected` appears, in order, as a (not necessarily contiguous) /// subsequence of `actual` — the "same kind sequence, extra graph events /// allowed" contract. +/// +/// The turn/message lifecycle events (`turn.*`, `message.appended`) are emitted +/// by the direct loop only for now; the graph driver does not announce them, so +/// they are excluded from the expected sequence. fn assert_kinds_subsequence(expected: &[String], actual: &[String]) { let mut cursor = 0; - for kind in expected { + let lifecycle = + |kind: &&String| !(kind.starts_with("turn.") || kind.as_str() == "message.appended"); + for kind in expected.iter().filter(lifecycle) { let Some(offset) = actual[cursor..].iter().position(|k| k == kind) else { panic!( "expected event kind `{kind}` not found (in order) in graph run's kinds: \ @@ -629,3 +635,262 @@ async fn graph_reresolves_when_before_model_adds_a_model_hint() { assert_eq!(run.text(), Some("hinted".to_string())); } + +// ── Terminal outcome parity ───────────────────────────────────────────────── + +/// A `StopWithPartial` cap must be reported identically by both engines: a +/// limit outcome, not a plain completion. +#[tokio::test] +async fn model_cap_stop_reports_the_same_terminal_outcome_in_both_engines() { + use tinyagents_harness::events::LimitKind; + use tinyagents_harness::limits::{LimitBehavior, RunLimits}; + use tinyagents_harness::terminal::TerminalReason; + + for execution in [LoopExecution::Direct, LoopExecution::Graph] { + let model = Arc::new(MockModel::with_tool_call("spin", serde_json::json!({}))); + let mut harness = harness_for(execution, model); + harness.register_tool(Arc::new(tinyagents_harness::testkit::FakeTool::returning( + "spin", "again", + ))); + harness.with_policy(RunPolicy { + execution, + limits: RunLimits::default() + .with_max_model_calls(2) + .with_behavior(LimitBehavior::StopWithPartial), + ..RunPolicy::default() + }); + let run = harness + .invoke_default(&(), vec![Message::user("go")]) + .await + .expect("StopWithPartial completes the run"); + let outcome = run.terminal.expect("terminal outcome"); + assert_eq!( + outcome.reason, + TerminalReason::LimitReached(Some(LimitKind::ModelCalls)), + "{execution:?}" + ); + } +} + +/// The graph engine honors `StopWithPartial` for the tool-call cap (the direct +/// loop surfaces it as a `LimitExceeded` error carrying `ToolCalls`); the stop +/// must carry `ToolCalls`, not an untyped limit. +#[tokio::test] +async fn graph_tool_cap_stop_reports_a_typed_tool_calls_limit() { + use tinyagents_harness::events::LimitKind; + use tinyagents_harness::limits::{LimitBehavior, RunLimits}; + use tinyagents_harness::terminal::TerminalReason; + + for execution in [LoopExecution::Graph] { + let model = Arc::new(MockModel::with_tool_call("spin", serde_json::json!({}))); + let mut harness = harness_for(execution, model); + harness.register_tool(Arc::new(tinyagents_harness::testkit::FakeTool::returning( + "spin", "again", + ))); + harness.with_policy(RunPolicy { + execution, + limits: RunLimits::default() + .with_max_tool_calls(1) + .with_behavior(LimitBehavior::StopWithPartial), + ..RunPolicy::default() + }); + let run = harness + .invoke_default(&(), vec![Message::user("go")]) + .await + .expect("StopWithPartial completes the run"); + let outcome = run.terminal.expect("terminal outcome"); + assert_eq!( + outcome.reason, + TerminalReason::LimitReached(Some(LimitKind::ToolCalls)), + "{execution:?}" + ); + assert!( + outcome.message.contains("tool_calls"), + "{execution:?}: {outcome:?}" + ); + } +} + +struct BoomModel; + +#[async_trait::async_trait] +impl tinyinference_llm::model::ChatModel<()> for BoomModel { + async fn invoke( + &self, + _: &(), + _: tinyinference_llm::model::ModelRequest, + ) -> tinyinference_llm::Result { + Err(tinyinference_llm::Error::Model("model boom".into())) + } +} + +struct FailingAfterAgent; + +#[async_trait::async_trait] +impl tinyagents_harness::middleware::Middleware<(), ()> for FailingAfterAgent { + fn name(&self) -> &str { + "failing_after_agent" + } + + async fn after_agent( + &self, + _ctx: &mut RunContext<()>, + _state: &(), + _run: &mut tinyagents_harness::middleware::AgentRun, + ) -> tinyagents_harness::Result<()> { + Err(TinyAgentsError::Middleware("cleanup boom".into())) + } +} + +/// When the run already failed, a failing `after_agent` hook must not replace +/// the originating error, in either engine. +#[tokio::test] +async fn a_failing_after_agent_keeps_the_original_error_in_both_engines() { + for execution in [LoopExecution::Direct, LoopExecution::Graph] { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness + .register_model("mock", Arc::new(BoomModel)) + .set_default_model("mock"); + if matches!(execution, LoopExecution::Graph) { + harness.with_loop_driver(Arc::new(GraphLoopDriver::new())); + } + harness.with_policy(RunPolicy { + execution, + ..RunPolicy::default() + }); + harness.push_middleware(Arc::new(FailingAfterAgent)); + let error = harness + .invoke_default(&(), vec![Message::user("go")]) + .await + .expect_err("the run fails"); + assert!( + error.to_string().contains("model boom"), + "{execution:?}: {error}" + ); + } +} + +/// A failing `after_agent` hook on an otherwise successful run is the surfaced +/// error, so the partial run's terminal outcome must be that failure rather +/// than the stale completion recorded before the hook ran. +#[tokio::test] +async fn a_failing_after_agent_on_a_successful_run_records_a_failure_outcome() { + use tinyagents_harness::terminal::{TerminalClass, TerminalReason}; + + for execution in [LoopExecution::Direct, LoopExecution::Graph] { + let model = Arc::new(MockModel::with_responses(vec![ModelResponse::assistant( + "fine", + )])); + let mut harness = harness_for(execution, model); + harness.push_middleware(Arc::new(FailingAfterAgent)); + let ctx = RunContext::new(tinyagents_harness::context::RunConfig::new("aa"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("go")]) + .await; + let error = partial.error.expect("the hook error surfaces"); + assert!(error.to_string().contains("cleanup boom"), "{execution:?}"); + let outcome = partial.run.terminal.expect("terminal outcome"); + assert_ne!(outcome.reason, TerminalReason::Completed, "{execution:?}"); + assert_eq!(outcome.class, TerminalClass::Failure, "{execution:?}"); + } +} + +/// Under `LimitBehavior::Error` the graph engine's tool-cap failure still +/// carries the concrete `ToolCalls` kind on the partial run's outcome. +#[tokio::test] +async fn graph_tool_cap_error_carries_the_tool_calls_kind() { + use tinyagents_harness::events::LimitKind; + use tinyagents_harness::limits::RunLimits; + use tinyagents_harness::terminal::TerminalReason; + + let model = Arc::new(MockModel::with_tool_call("spin", serde_json::json!({}))); + let mut harness = harness_for(LoopExecution::Graph, model); + harness.register_tool(Arc::new(tinyagents_harness::testkit::FakeTool::returning( + "spin", "again", + ))); + harness.with_policy(RunPolicy { + execution: LoopExecution::Graph, + limits: RunLimits::default().with_max_tool_calls(1), + ..RunPolicy::default() + }); + let ctx = RunContext::new(tinyagents_harness::context::RunConfig::new("cap"), ()); + let partial = harness + .invoke_in_context_collecting_partial(&(), ctx, vec![Message::user("go")]) + .await; + assert!(partial.error.is_some()); + assert_eq!( + partial.run.terminal.expect("outcome").reason, + TerminalReason::LimitReached(Some(LimitKind::ToolCalls)) + ); +} + +struct LimitFromHook; + +#[async_trait::async_trait] +impl tinyagents_harness::middleware::Middleware<(), ()> for LimitFromHook { + fn name(&self) -> &str { + "limit_from_hook" + } + + async fn before_tool( + &self, + _ctx: &mut RunContext<()>, + _state: &(), + _call: &mut tinyinference_llm::tool::ToolCall, + ) -> tinyagents_harness::Result<()> { + Err(TinyAgentsError::LimitExceeded("policy budget".into())) + } +} + +/// A `LimitExceeded` raised by middleware is not the tool-call cap, so under +/// `StopWithPartial` it must fail the run in both engines instead of becoming +/// a partial stop labeled `ToolCalls`. +#[tokio::test] +async fn a_middleware_limit_error_is_not_a_tool_cap_partial_stop() { + use tinyagents_harness::limits::{LimitBehavior, RunLimits}; + + for execution in [LoopExecution::Direct, LoopExecution::Graph] { + let model = Arc::new(MockModel::with_tool_call("spin", serde_json::json!({}))); + let mut harness = harness_for(execution, model); + harness.register_tool(Arc::new(tinyagents_harness::testkit::FakeTool::returning( + "spin", "again", + ))); + harness.push_middleware(Arc::new(LimitFromHook)); + harness.with_policy(RunPolicy { + execution, + limits: RunLimits::default().with_behavior(LimitBehavior::StopWithPartial), + ..RunPolicy::default() + }); + let result = harness.invoke_default(&(), vec![Message::user("go")]).await; + assert!( + matches!(result, Err(TinyAgentsError::LimitExceeded(_))), + "{execution:?}: {result:?}" + ); + } +} + +/// The turn/message lifecycle events are direct-loop only for now (a documented +/// follow-up). Pin that so the gap is explicit and this parity file's filter +/// cannot silently hide a change in either direction. +#[tokio::test] +async fn lifecycle_events_are_direct_loop_only_until_the_graph_driver_emits_them() { + for (execution, expect_lifecycle) in + [(LoopExecution::Direct, true), (LoopExecution::Graph, false)] + { + let model = Arc::new(MockModel::with_responses(vec![ModelResponse::assistant( + "done", + )])); + let harness = harness_for(execution, model); + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("lc"), ()).with_events(recorder.sink()); + harness + .invoke_in_context(&(), ctx, vec![Message::user("hi")]) + .await + .unwrap(); + let has_lifecycle = recorder + .events() + .iter() + .any(|event| event.kind() == "turn.started" || event.kind() == "message.appended"); + assert_eq!(has_lifecycle, expect_lifecycle, "{execution:?}"); + } +} diff --git a/crates/tinyagents-orchestration/tests/e2e_orchestrator_subagents.rs b/crates/tinyagents-orchestration/tests/e2e_orchestrator_subagents.rs index 970fd0415..9a250e92c 100644 --- a/crates/tinyagents-orchestration/tests/e2e_orchestrator_subagents.rs +++ b/crates/tinyagents-orchestration/tests/e2e_orchestrator_subagents.rs @@ -241,6 +241,7 @@ async fn orchestrator_resolves_and_runs_only_the_chosen_subagents() -> Result<() sink.emit(AgentEvent::RunCompleted { run_id: RunId::new("orchestrator"), + outcome: None, }); // 4. Compose the resolved sub-agents' outputs into one final answer. diff --git a/crates/tinyagents-runtime/src/driver.rs b/crates/tinyagents-runtime/src/driver.rs index 8cc75b2ec..be5888004 100644 --- a/crates/tinyagents-runtime/src/driver.rs +++ b/crates/tinyagents-runtime/src/driver.rs @@ -29,6 +29,11 @@ pub struct DriverOutcome { pub partial: Option, /// Whether the driver ended at an interruptible point. pub interrupted: bool, + /// Typed classification of how the run ended, when the driver has one + /// ([`HarnessDriver`] always does). Preferred by the session over the + /// outcome it would derive from `interrupted`, so a run that stopped on a + /// cap or a deferral is not reported as `Completed`. + pub outcome: Option, } /// A driver error which may retain model history and display-only partial text. @@ -38,6 +43,11 @@ pub struct DriverFailure { pub error: RuntimeError, /// Work produced before failure, safe for partial transcript persistence. pub partial: Option, + /// Typed classification of the failure, when the driver has one + /// ([`HarnessDriver`] always does). Delivered to + /// [`crate::SessionHooks::on_terminal_outcome`]; `None` falls back to a + /// classification derived from the error. + pub outcome: Option, } /// Object-safe model/tool-loop invocation boundary. @@ -80,6 +90,7 @@ impl SessionDriver request.tools.specs(), ) { return Err(DriverFailure { + outcome: None, error: RuntimeError::ToolSnapshotMismatch, partial: None, }); @@ -110,6 +121,7 @@ impl SessionDriver .iter() .rev() .find_map(|message| matches!(message, Message::Assistant(_)).then(|| message.text())); + let terminal = partial.run.terminal.clone(); let outcome = DriverOutcome { history: partial.run.messages, output: output.clone(), @@ -122,9 +134,11 @@ impl SessionDriver .and(output) .map(TranscriptPartial::new), interrupted: partial.run.paused.is_some(), + outcome: terminal.clone(), }; match partial.error { Some(error) => Err(DriverFailure { + outcome: terminal, error: RuntimeError::Driver(error.to_string()), partial: Some(outcome), }), diff --git a/crates/tinyagents-runtime/src/hooks.rs b/crates/tinyagents-runtime/src/hooks.rs index dbd626427..567945ed8 100644 --- a/crates/tinyagents-runtime/src/hooks.rs +++ b/crates/tinyagents-runtime/src/hooks.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use tinyagents_harness::terminal::TerminalOutcome; use crate::{ CommitReceipt, ResumePreparation, RuntimeError, SessionStateView, SessionTerminal, @@ -46,6 +47,14 @@ pub trait SessionHooks: Send + Sync { async fn after_commit(&self, _: CommitReceipt) -> Result<(), RuntimeError> { Ok(()) } + /// Receives the typed [`TerminalOutcome`] of the turn, exactly once, + /// immediately before [`Self::on_terminal`]. A driver failure carries the + /// driver's own classification (timeout, provider failure, ...); + /// everything else is derived with [`SessionTerminal::outcome`]. The + /// default ignores it, so existing hooks are unaffected. + async fn on_terminal_outcome(&self, _: TerminalOutcome) -> Result<(), RuntimeError> { + Ok(()) + } /// Runs exactly once for every terminal turn result. async fn on_terminal(&self, terminal: SessionTerminal) -> Result<(), RuntimeError>; } diff --git a/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs b/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs index 4f0139f31..08db4dce2 100644 --- a/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs +++ b/crates/tinyagents-runtime/src/lib_prefix_refresh_tests.rs @@ -149,11 +149,13 @@ async fn uncommitted_refresh_errors_preserve_prefix_history_and_accept_old_froze ]; let result = if failure == "driver" { Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("failed".into()), partial: None, }) } else if failure.starts_with("partial") { Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("partial".into()), partial: Some(outcome(refreshed)), }) @@ -298,6 +300,7 @@ async fn successfully_persisted_partial_refresh_keeps_new_prefix() { let driver = Arc::new(Driver::new(vec![ Ok(outcome(old())), Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("partial".into()), partial: Some(outcome(partial_history.clone())), }), @@ -422,6 +425,7 @@ async fn prefix_extension_containing_prior_rows_forces_successor_for_normal_and_ ]; let result = if partial { Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("partial".into()), partial: Some(outcome(expected.clone())), }) diff --git a/crates/tinyagents-runtime/src/lib_tests.rs b/crates/tinyagents-runtime/src/lib_tests.rs index 723de5f7d..a5cc9ebb4 100644 --- a/crates/tinyagents-runtime/src/lib_tests.rs +++ b/crates/tinyagents-runtime/src/lib_tests.rs @@ -80,6 +80,7 @@ impl Tool for RegisteredTool { fn outcome(history: Vec) -> DriverOutcome { DriverOutcome { + outcome: None, history, output: Some("ok".into()), partial: None, @@ -599,12 +600,14 @@ async fn persistence_failure_rolls_back_and_a_partial_never_falls_back_to_two_wr let (locator, history) = locator(None); let partial = DriverOutcome { + outcome: None, history: vec![Message::assistant("recoverable")], output: None, partial: Some(crate::TranscriptPartial::new("display only")), interrupted: true, }; let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("interrupted".into()), partial: Some(partial), })]))) @@ -766,12 +769,14 @@ async fn file_history_commits_partial_model_history_and_display_only_partial_tog ..Default::default() }); let partial = DriverOutcome { + outcome: None, history: vec![Message::assistant("recoverable")], output: None, partial: Some(crate::TranscriptPartial::new("display partial")), interrupted: true, }; let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("interrupted".into()), partial: Some(partial), })]))) @@ -864,12 +869,14 @@ async fn partial_usage_error_leaves_the_session_and_target_entirely_uncommitted( ..Default::default() }); let partial = DriverOutcome { + outcome: None, history: vec![Message::assistant("recoverable")], output: None, partial: Some(crate::TranscriptPartial::new("display partial")), interrupted: true, }; let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("driver interrupted".into()), partial: Some(partial), })]))) @@ -2023,6 +2030,7 @@ async fn receipt_reports_a_compaction_as_replacement_not_an_append_range() { async fn failure_and_cancellation_do_not_run_after_commit_and_emit_one_terminal() { let (failure_hook, events) = hook(vec![]); let mut failed = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: None, error: RuntimeError::Driver("no".into()), partial: None, })]))) @@ -2433,6 +2441,7 @@ fn session_turn_options(resume: ResumeMode, thread: &str) -> TurnOptions { fn session_outcome(history: Vec, output: &str) -> DriverOutcome { DriverOutcome { + outcome: None, history, output: Some(output.into()), partial: None, @@ -4191,6 +4200,181 @@ async fn refreshing_initial_prefix_accepts_an_identical_frozen_preparation_after #[path = "lib_prefix_refresh_tests.rs"] mod prefix_refresh_tests; +struct OutcomeHook { + outcomes: Mutex>, + order: Mutex>, +} + +#[async_trait] +impl SessionHooks for OutcomeHook { + async fn before_turn( + &self, + _: &mut SessionTurnRequest, + _: &mut TurnOptions, + _: SessionStateView<'_>, + ) -> Result { + Ok(TurnPreparation::default()) + } + async fn before_commit( + &self, + _: &SessionTurnOutcome, + _: &crate::TranscriptTurnOptions, + ) -> Result<(), RuntimeError> { + Ok(()) + } + async fn on_terminal_outcome( + &self, + outcome: tinyagents_harness::terminal::TerminalOutcome, + ) -> Result<(), RuntimeError> { + self.order.lock().unwrap().push("outcome"); + self.outcomes.lock().unwrap().push(outcome); + Ok(()) + } + async fn on_terminal(&self, _: SessionTerminal) -> Result<(), RuntimeError> { + self.order.lock().unwrap().push("terminal"); + Ok(()) + } +} + +/// Waits (bounded) until the detached terminal task has recorded `count` +/// outcomes, instead of relying on scheduler order. +async fn wait_for_outcomes(hook: &OutcomeHook, count: usize) { + for _ in 0..500 { + if hook.outcomes.lock().unwrap().len() >= count { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } +} + +/// Waits (bounded) until `count` hook callbacks have been recorded in order. +async fn wait_for_order(hook: &OutcomeHook, count: usize) { + for _ in 0..500 { + if hook.order.lock().unwrap().len() >= count { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } +} + +fn outcome_hook() -> Arc { + Arc::new(OutcomeHook { + outcomes: Mutex::default(), + order: Mutex::default(), + }) +} + +#[tokio::test] +async fn driver_failures_deliver_their_typed_outcome_before_the_terminal() { + use tinyagents_harness::terminal::{TerminalClass, TerminalOutcome, TerminalReason}; + let hook = outcome_hook(); + let typed = TerminalOutcome::new(TerminalReason::Timeout, "deadline"); + let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: Some(typed.clone()), + error: RuntimeError::Driver("deadline".into()), + partial: None, + })]))) + .hooks(hook.clone()) + .build() + .unwrap(); + let _ = session + .turn( + SessionTurnRequest::new(Message::user("x")), + TurnOptions::default(), + ) + .await; + wait_for_outcomes(&hook, 1).await; + wait_for_order(&hook, 2).await; + assert_eq!(hook.outcomes.lock().unwrap().as_slice(), [typed]); + assert_eq!( + hook.order.lock().unwrap().as_slice(), + ["outcome", "terminal"] + ); + + // An untyped failure falls back to a derived Failure-class outcome. + let hook = outcome_hook(); + let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Err(DriverFailure { + outcome: None, + error: RuntimeError::Driver("plain".into()), + partial: None, + })]))) + .hooks(hook.clone()) + .build() + .unwrap(); + let _ = session + .turn( + SessionTurnRequest::new(Message::user("x")), + TurnOptions::default(), + ) + .await; + wait_for_outcomes(&hook, 1).await; + let outcomes = hook.outcomes.lock().unwrap(); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].class, TerminalClass::Failure); +} + +#[tokio::test] +async fn completed_turns_report_a_completed_outcome() { + use tinyagents_harness::terminal::TerminalReason; + let hook = outcome_hook(); + let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(outcome(vec![ + Message::assistant("hi"), + ]))]))) + .hooks(hook.clone()) + .build() + .unwrap(); + session + .turn( + SessionTurnRequest::new(Message::user("x")), + TurnOptions::default(), + ) + .await + .unwrap(); + wait_for_outcomes(&hook, 1).await; + let outcomes = hook.outcomes.lock().unwrap(); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].reason, TerminalReason::Completed); +} + +#[test] +fn session_terminal_derives_an_outcome() { + use tinyagents_harness::terminal::TerminalClass; + assert_eq!( + SessionTerminal::Cancelled.outcome().class, + TerminalClass::Cancellation + ); + let failed = SessionTerminal::Failed("x".into()).outcome(); + assert_eq!(failed.class, TerminalClass::Failure); + assert_eq!(failed.message, "x"); +} + +#[tokio::test] +async fn a_drivers_typed_success_outcome_is_not_flattened_to_completed() { + use tinyagents_harness::terminal::{TerminalOutcome, TerminalReason}; + let hook = outcome_hook(); + let capped = TerminalOutcome::limit_reached( + Some(tinyagents_harness::events::LimitKind::ModelCalls), + "stopped with the partial run", + ); + let mut driver_outcome = outcome(vec![Message::assistant("partial")]); + driver_outcome.outcome = Some(capped.clone()); + let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(driver_outcome)]))) + .hooks(hook.clone()) + .build() + .unwrap(); + session + .turn( + SessionTurnRequest::new(Message::user("x")), + TurnOptions::default(), + ) + .await + .unwrap(); + wait_for_outcomes(&hook, 1).await; + let outcomes = hook.outcomes.lock().unwrap(); + assert_eq!(outcomes.as_slice(), [capped]); + assert_ne!(outcomes[0].reason, TerminalReason::Completed); +} + #[test] fn tool_snapshot_retaining_keeps_matching_declarations_and_exactness() { let spec = |name: &str| ToolSpec { @@ -4207,3 +4391,89 @@ fn tool_snapshot_retaining_keeps_matching_declarations_and_exactness() { assert!(narrowed.is_exact(), "a one-off snapshot stays one-off"); assert_eq!(snapshot.specs().len(), 2, "the source is untouched"); } + +struct SlowOutcomeHook { + started: Arc, + release: Arc, + terminal: Arc, + terminals: Mutex>, +} + +#[async_trait] +impl SessionHooks for SlowOutcomeHook { + async fn before_turn( + &self, + _: &mut SessionTurnRequest, + _: &mut TurnOptions, + _: SessionStateView<'_>, + ) -> Result { + Err(RuntimeError::Cancelled) + } + + async fn before_commit( + &self, + _: &SessionTurnOutcome, + _: &TranscriptTurnOptions, + ) -> Result<(), RuntimeError> { + Ok(()) + } + + async fn after_commit(&self, _: CommitReceipt) -> Result<(), RuntimeError> { + Ok(()) + } + + async fn on_terminal_outcome( + &self, + _: tinyagents_harness::terminal::TerminalOutcome, + ) -> Result<(), RuntimeError> { + self.started.notify_waiters(); + self.release.notified().await; + Ok(()) + } + + async fn on_terminal(&self, terminal: SessionTerminal) -> Result<(), RuntimeError> { + self.terminals.lock().unwrap().push(terminal); + self.terminal.notify_waiters(); + Ok(()) + } +} + +#[tokio::test] +async fn dropping_the_turn_while_the_outcome_hook_is_pending_still_delivers_on_terminal() { + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let terminal = Arc::new(tokio::sync::Notify::new()); + let hook = Arc::new(SlowOutcomeHook { + started: started.clone(), + release: release.clone(), + terminal: terminal.clone(), + terminals: Mutex::new(Vec::new()), + }); + let (locator, _history) = locator(None); + let mut session = SessionBuilder::new(Arc::new(Driver::new(vec![Ok(outcome(vec![ + Message::assistant("never"), + ]))]))) + .codec(Arc::new(Codec::default())) + .transcript(locator, "agent", meta()) + .hooks(hook.clone()) + .build() + .unwrap(); + let observed_terminal = terminal.notified(); + let observed_start = started.notified(); + let turn = tokio::spawn(async move { + session + .turn( + SessionTurnRequest::new(Message::user("x")), + TurnOptions::default(), + ) + .await + }); + observed_start.await; + turn.abort(); + let _ = turn.await; + release.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(5), observed_terminal) + .await + .expect("on_terminal must still be delivered"); + assert_eq!(hook.terminals.lock().unwrap().len(), 1); +} diff --git a/crates/tinyagents-runtime/src/session.rs b/crates/tinyagents-runtime/src/session.rs index 3f98c3d4f..bab2b9022 100644 --- a/crates/tinyagents-runtime/src/session.rs +++ b/crates/tinyagents-runtime/src/session.rs @@ -1,6 +1,7 @@ use std::{future::Future, sync::Arc}; use tinyagents_harness::CancellationToken; +use tinyagents_harness::terminal::{TerminalOutcome, TerminalReason}; use tinyagents_session::transcript::{ SessionRef, SessionTurnGuard, TranscriptHistory, TranscriptMessage, TranscriptPartial, TranscriptTurn, TurnUsage, lock_session_turn, session_stem, @@ -602,7 +603,7 @@ impl Session { _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled), result = self.driver.execute(DriverRequest { history: input, tools, run_context, stream }) => result, }; - let outcome = match driver_result { + let mut outcome = match driver_result { Ok(outcome) => outcome, Err(failure) => { if let Some(partial) = failure.partial { @@ -628,9 +629,13 @@ impl Session { self.committed_turns += 1; } } + terminal_guard.set_outcome(failure.outcome); return Err(failure.error); } }; + // Kept for `finalize_commit`: only a *committed* turn reports the + // driver's success classification; a later failure uses its own. + terminal_guard.driver_success_outcome = outcome.outcome.take(); let candidate = Self::with_prefix_snapshot(&prefix, outcome.history); let committed = SessionTurnOutcome { history: candidate.clone(), @@ -1050,6 +1055,14 @@ impl Session { struct TerminalGuard { hooks: Arc>, terminal: Option, + /// The driver's own typed classification of a failure, when it supplied one. + outcome: Option, + /// The driver's typed classification of a run it returned successfully; + /// used only once the turn commits. + driver_success_outcome: Option, + /// `true` until a real terminal is set: the drop-time default means the + /// caller abandoned the turn, which is a cancellation. + abandoned: bool, committed: bool, } @@ -1058,16 +1071,42 @@ impl TerminalGuard { Self { hooks, terminal: Some(SessionTerminal::Failed("session turn dropped".into())), + outcome: None, + driver_success_outcome: None, + abandoned: true, committed: false, } } fn set(&mut self, terminal: SessionTerminal) { self.terminal = Some(terminal); + self.abandoned = false; + } + + fn set_outcome(&mut self, outcome: Option) { + self.outcome = outcome; + } + + /// Takes the pending terminal with its typed outcome: the driver's if it + /// gave one, a cancellation if the turn was abandoned, else derived. + fn take_pending(&mut self) -> Option<(SessionTerminal, TerminalOutcome)> { + let terminal = self.terminal.take()?; + let outcome = self.outcome.take().unwrap_or_else(|| { + if self.abandoned { + TerminalOutcome::new(TerminalReason::Cancelled, "session turn dropped") + } else { + terminal.outcome() + } + }); + Some((terminal, outcome)) } fn finalize_commit(&mut self, receipt: CommitReceipt) -> tokio::task::JoinHandle<()> { let terminal = SessionTerminal::Completed(receipt.outcome.clone()); + let outcome = self + .driver_success_outcome + .take() + .unwrap_or_else(|| terminal.outcome()); // Removing the guard's terminal transfers exactly-once ownership to // the finalizer. `finish` and `Drop` then become no-ops for this turn. self.terminal = None; @@ -1075,6 +1114,7 @@ impl TerminalGuard { let hooks = self.hooks.clone(); tokio::spawn(async move { let _ = hooks.after_commit(receipt).await; + let _ = hooks.on_terminal_outcome(outcome).await; let _ = hooks.on_terminal(terminal).await; }) } @@ -1084,21 +1124,35 @@ impl TerminalGuard { } async fn finish(mut self) -> Result<(), RuntimeError> { - let Some(terminal) = self.terminal.take() else { + let Some((terminal, outcome)) = self.take_pending() else { return Ok(()); }; - self.hooks.on_terminal(terminal).await + // The terminal is already removed from the guard, so `Drop` can no + // longer deliver it. Run the hooks on their own task: if the caller's + // future is dropped while a hook is pending, the remaining hooks still + // run exactly once. + let hooks = self.hooks.clone(); + let task = tokio::spawn(async move { + let _ = hooks.on_terminal_outcome(outcome).await; + hooks.on_terminal(terminal).await + }); + match task.await { + Ok(result) => result, + Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), + Err(_) => Ok(()), + } } } impl Drop for TerminalGuard { fn drop(&mut self) { - let Some(terminal) = self.terminal.take() else { + let Some((terminal, outcome)) = self.take_pending() else { return; }; let hooks = self.hooks.clone(); if let Ok(runtime) = tokio::runtime::Handle::try_current() { runtime.spawn(async move { + let _ = hooks.on_terminal_outcome(outcome).await; let _ = hooks.on_terminal(terminal).await; }); } diff --git a/crates/tinyagents-runtime/src/session_tests.rs b/crates/tinyagents-runtime/src/session_tests.rs index 6653fd911..d3c0f842e 100644 --- a/crates/tinyagents-runtime/src/session_tests.rs +++ b/crates/tinyagents-runtime/src/session_tests.rs @@ -83,6 +83,7 @@ impl SessionDriver for GatedDriver { let mut history = request.history; history.push(Message::assistant("Sunny.")); Ok(DriverOutcome { + outcome: None, history, output: Some("Sunny.".into()), partial: None, diff --git a/crates/tinyagents-runtime/src/types.rs b/crates/tinyagents-runtime/src/types.rs index dc7095a4f..0361bc032 100644 --- a/crates/tinyagents-runtime/src/types.rs +++ b/crates/tinyagents-runtime/src/types.rs @@ -344,3 +344,25 @@ pub enum SessionTerminal { /// The turn ended with an error after any recoverable partial persistence. Failed(String), } + +impl SessionTerminal { + /// A best-effort typed outcome derived from this terminal alone. + /// + /// A failure carries only its message here, so it classifies as + /// `Internal`. The precise outcome (timeout, provider failure, ...) is + /// delivered separately through + /// [`SessionHooks::on_terminal_outcome`][crate::SessionHooks::on_terminal_outcome]. + pub fn outcome(&self) -> tinyagents_harness::terminal::TerminalOutcome { + use tinyagents_harness::terminal::{TerminalOutcome, TerminalReason}; + match self { + Self::Completed(turn) if turn.interrupted => { + TerminalOutcome::new(TerminalReason::Paused, "turn interrupted before completion") + } + Self::Completed(_) => TerminalOutcome::completed(), + Self::Cancelled => TerminalOutcome::new(TerminalReason::Cancelled, "turn cancelled"), + Self::Failed(message) => { + TerminalOutcome::new(TerminalReason::Internal, message.clone()) + } + } + } +} diff --git a/docs/modules/harness/README.md b/docs/modules/harness/README.md index 67011815a..7b2eeb482 100644 --- a/docs/modules/harness/README.md +++ b/docs/modules/harness/README.md @@ -245,6 +245,7 @@ Feature details: - [Tool dialects](tool-dialect.md) - [Middleware feature](middleware.md) - [Repeat-progress guard](repeat-progress.md) +- [Terminal outcome and turn/message lifecycle events](terminal-outcome.md) - [Sub-agent and orchestrator steering](subagent-steering.md) - [Structured output feature](structured-output.md) - [Limits, retry, fallback, and rate limiting](limits-retry.md) diff --git a/docs/modules/harness/observability-overview.md b/docs/modules/harness/observability-overview.md index a900bb191..b2f14c7ec 100644 --- a/docs/modules/harness/observability-overview.md +++ b/docs/modules/harness/observability-overview.md @@ -49,11 +49,21 @@ pub enum AgentEvent { MiddlewareCompleted { name: String, call_id: Option }, RetryScheduled { call_id: CallId, attempt: usize }, Custom { name: String, payload: serde_json::Value }, - RunCompleted { run_id: RunId }, - RunFailed { run_id: RunId, error: String }, + TurnStarted { turn: u32 }, + TurnCompleted { turn: u32, tool_result_count: usize, tool_call_ids: Vec }, + MessageAppended { role: String, index: usize, call_id: Option, message: Option }, + RunCompleted { run_id: RunId, outcome: Option }, + RunFailed { run_id: RunId, error: String, outcome: Option }, } ``` +`outcome` on `RunCompleted` / `RunFailed` is a typed `TerminalOutcome` (a +`reason`, a coarse `class`, `provider_started`, and a `timeout_phase` for +timeouts). It is `Some` for every run this crate ends and `None` only when +replaying journals written before the field existed. The reason mapping, +lifecycle events and `MessageRetracted` / `TranscriptRewritten` semantics are in +[terminal outcome and turn lifecycle](terminal-outcome.md). + Streaming modes: - `messages`: model deltas and final messages diff --git a/docs/modules/harness/observability.md b/docs/modules/harness/observability.md index a2d61f331..2ba42287b 100644 --- a/docs/modules/harness/observability.md +++ b/docs/modules/harness/observability.md @@ -123,6 +123,7 @@ Event kinds should include: - `run.started` - `run.completed` +- `turn.started` / `turn.completed` / `message.appended` (see [terminal outcome and turn lifecycle](terminal-outcome.md)) - `run.failed` - `model.started` - `model.delta` diff --git a/docs/modules/harness/runtime.md b/docs/modules/harness/runtime.md index 423bd0c49..9fa031ab5 100644 --- a/docs/modules/harness/runtime.md +++ b/docs/modules/harness/runtime.md @@ -232,7 +232,9 @@ run is in flight, and the loop drains them only at safe boundaries: `RunPolicy::queue_mode` picks how many items a boundary takes: `QueueMode::All` (default) applies every pending item; `OneAtATime` applies the oldest and leaves the rest for the next boundary. Each application emits -`AgentEvent::QueuedMessageApplied { lane, count }`. A "natural finish" is +`AgentEvent::QueuedMessageApplied { lane, count, first_index, messages }` (`messages` follows the +capture policy per message: `tool` messages under `PayloadCapture::tool_io`, +all others under `PayloadCapture::model_io`). A "natural finish" is the model producing a final answer (including a structured-output finish under `EndStrategy::Early`/`Graceful`); a middleware `StopWithFinal` / `JumpTo(End)`, a limit stop, a pause, or a deferral is terminal and leaves diff --git a/docs/modules/harness/terminal-outcome.md b/docs/modules/harness/terminal-outcome.md new file mode 100644 index 000000000..a25c66b7c --- /dev/null +++ b/docs/modules/harness/terminal-outcome.md @@ -0,0 +1,41 @@ +# Terminal outcomes and lifecycle events + +`TerminalOutcome` is the typed explanation for how an agent run ended. It is +attached to completed and failed run events and is available on `AgentRun`. +The `reason` identifies the specific cause, while `class` provides a stable +coarse-grained category for hosts that need only success, timeout, +cancellation, failure, or suspension. + +Outcomes preserve the legacy human-readable message. Timeout outcomes also +record whether the deadline landed before a provider call, during one, or +after a call. Provider-start state is tracked separately so a host can tell a +first-call failure from a preflight failure. + +The direct and graph loop drivers publish the outcome on `AgentRun` before +running `after_agent` middleware. This lets middleware observe the same +classification as the terminal event. Session hooks receive the typed outcome +before the legacy terminal notification. + +Turn lifecycle events announce appended messages and retract only messages +that were previously announced. Initial input messages are treated as a seed +prefix and are not re-announced on the first turn. + +Mutations that are not appends are explicit. `MessageRetracted { index }` is +emitted (highest index first) when an announced message is popped, for example +an unusable assistant reply dropped before a retry; it always precedes the +`MessageAppended` of its replacement. `TranscriptRewritten { len, reason }` is +emitted when the transcript is rewritten in place (a tool-set change folded +into the leading system message); a mirror should resynchronise and count later +`MessageAppended` indices from `len`. Nested tool calls (a tool calling another +tool) produce `ToolStarted` / `ToolCompleted` events with a `parent_call_id` +but no transcript rows, so they never emit `MessageAppended`. + +On `RunCompleted` / `RunFailed` the `outcome` field is `Some` for every run this +crate ends; it is `None` only when deserializing journals written before the +field existed. + +Scope: the typed outcome is published by both the direct and the graph loop +driver, but the turn and message lifecycle events (`TurnStarted`, +`MessageAppended`, `MessageRetracted`, ...) are emitted by the direct loop only. +The graph driver does not emit them yet, so graph-engine hosts should not rely +on them.