Skip to main content

mojentic/llm/tools/
runner.rs

1//! Tool runner abstraction for executing batches of tool calls.
2//!
3//! Provides pluggable execution strategies (serial, parallel) so brokers can
4//! stay independent of concurrency policy. Mirrors the Python and TypeScript
5//! `ToolRunner` design.
6
7use crate::llm::tools::tool::{LlmTool, ToolRunCtx};
8use async_trait::async_trait;
9use futures::stream::{FuturesUnordered, StreamExt};
10use serde_json::Value;
11use std::collections::HashMap;
12use std::sync::Arc;
13use std::time::Instant;
14use tokio::sync::Semaphore;
15use tokio_util::sync::CancellationToken;
16
17/// A single tool call to execute, identified by an opaque id.
18///
19/// The id is preserved on the matching [`ToolCallOutcome`] so callers can
20/// pair calls and outcomes deterministically.
21#[derive(Debug, Clone)]
22pub struct ToolCallExecution {
23    pub id: String,
24    pub name: String,
25    pub args: HashMap<String, Value>,
26}
27
28/// Outcome of executing a single tool call.
29#[derive(Debug, Clone)]
30pub struct ToolCallOutcome {
31    pub id: String,
32    pub name: String,
33    pub ok: bool,
34    pub result: Option<Value>,
35    pub error: Option<String>,
36    pub duration_ms: u64,
37}
38
39/// Strategy for executing a batch of tool calls.
40///
41/// Implementations decide concurrency, ordering, and cancellation semantics.
42/// Output order must match input order regardless of execution order.
43#[async_trait]
44pub trait ToolRunner: Send + Sync {
45    async fn run_batch(
46        &self,
47        calls: &[ToolCallExecution],
48        tools: &[Box<dyn LlmTool>],
49        ctx: &ToolRunCtx,
50    ) -> Vec<ToolCallOutcome>;
51}
52
53/// Executes tool calls one at a time in input order.
54///
55/// Default for [`crate::llm::broker::LlmBroker`] — predictable, stepwise
56/// debugging and zero behaviour change for callers who don't opt in.
57#[derive(Debug, Default, Clone)]
58pub struct SerialToolRunner;
59
60#[async_trait]
61impl ToolRunner for SerialToolRunner {
62    async fn run_batch(
63        &self,
64        calls: &[ToolCallExecution],
65        tools: &[Box<dyn LlmTool>],
66        ctx: &ToolRunCtx,
67    ) -> Vec<ToolCallOutcome> {
68        let mut outcomes = Vec::with_capacity(calls.len());
69        for call in calls {
70            if ctx.cancel.is_cancelled() {
71                outcomes.push(aborted_outcome(call));
72                continue;
73            }
74            outcomes.push(execute_one(call, tools, ctx).await);
75        }
76        outcomes
77    }
78}
79
80/// Executes tool calls concurrently with a bounded fan-out.
81///
82/// `max_concurrency` defaults to 4 — high enough to win on typical realtime
83/// turns, low enough that unbounded fan-out into rate-limited APIs doesn't
84/// punish users.
85#[derive(Debug, Clone)]
86pub struct ParallelToolRunner {
87    pub max_concurrency: usize,
88}
89
90impl Default for ParallelToolRunner {
91    fn default() -> Self {
92        Self { max_concurrency: 4 }
93    }
94}
95
96impl ParallelToolRunner {
97    pub fn new(max_concurrency: usize) -> Self {
98        assert!(max_concurrency >= 1, "max_concurrency must be >= 1");
99        Self { max_concurrency }
100    }
101}
102
103#[async_trait]
104impl ToolRunner for ParallelToolRunner {
105    async fn run_batch(
106        &self,
107        calls: &[ToolCallExecution],
108        tools: &[Box<dyn LlmTool>],
109        ctx: &ToolRunCtx,
110    ) -> Vec<ToolCallOutcome> {
111        if calls.is_empty() {
112            return Vec::new();
113        }
114
115        let semaphore = Arc::new(Semaphore::new(self.max_concurrency));
116        let mut futures = FuturesUnordered::new();
117
118        for (idx, call) in calls.iter().enumerate() {
119            let permit_sem = Arc::clone(&semaphore);
120            let cancel = ctx.cancel.clone();
121            futures.push(async move {
122                if cancel.is_cancelled() {
123                    return (idx, aborted_outcome(call));
124                }
125                let _permit = permit_sem.acquire().await.expect("semaphore closed");
126                if cancel.is_cancelled() {
127                    return (idx, aborted_outcome(call));
128                }
129                (idx, execute_one(call, tools, ctx).await)
130            });
131        }
132
133        let mut indexed: Vec<(usize, ToolCallOutcome)> = Vec::with_capacity(calls.len());
134        while let Some(pair) = futures.next().await {
135            indexed.push(pair);
136        }
137        indexed.sort_by_key(|(idx, _)| *idx);
138        indexed.into_iter().map(|(_, o)| o).collect()
139    }
140}
141
142fn aborted_outcome(call: &ToolCallExecution) -> ToolCallOutcome {
143    ToolCallOutcome {
144        id: call.id.clone(),
145        name: call.name.clone(),
146        ok: false,
147        result: None,
148        error: Some(crate::error::MojenticError::Cancelled.to_string()),
149        duration_ms: 0,
150    }
151}
152
153async fn execute_one(
154    call: &ToolCallExecution,
155    tools: &[Box<dyn LlmTool>],
156    ctx: &ToolRunCtx,
157) -> ToolCallOutcome {
158    let start = Instant::now();
159    let tool = tools.iter().find(|t| t.matches(&call.name));
160    let outcome = match tool {
161        None => ToolCallOutcome {
162            id: call.id.clone(),
163            name: call.name.clone(),
164            ok: false,
165            result: None,
166            error: Some(format!("Tool {:?} not found", call.name)),
167            duration_ms: start.elapsed().as_millis() as u64,
168        },
169        Some(tool) => match tool.run(&call.args, ctx).await {
170            Ok(result) => ToolCallOutcome {
171                id: call.id.clone(),
172                name: call.name.clone(),
173                ok: true,
174                result: Some(result),
175                error: None,
176                duration_ms: start.elapsed().as_millis() as u64,
177            },
178            Err(err) => ToolCallOutcome {
179                id: call.id.clone(),
180                name: call.name.clone(),
181                ok: false,
182                result: None,
183                error: Some(format!("{err}")),
184                duration_ms: start.elapsed().as_millis() as u64,
185            },
186        },
187    };
188    outcome
189}
190
191/// Convenience: build a fresh cancellation token for callers that don't have
192/// one already (used by the sync broker code path).
193pub fn fresh_cancel_token() -> CancellationToken {
194    CancellationToken::new()
195}
196
197#[cfg(test)]
198mod tests {
199    use super::*;
200    use crate::llm::tools::tool::{FunctionDescriptor, ToolDescriptor};
201    use serde_json::json;
202
203    struct EchoTool;
204
205    #[async_trait]
206    impl LlmTool for EchoTool {
207        async fn run(
208            &self,
209            args: &HashMap<String, Value>,
210            _ctx: &ToolRunCtx,
211        ) -> crate::error::Result<Value> {
212            Ok(json!({ "echo": args.get("value").cloned().unwrap_or_default() }))
213        }
214
215        fn descriptor(&self) -> ToolDescriptor {
216            ToolDescriptor {
217                r#type: "function".to_string(),
218                function: FunctionDescriptor {
219                    name: "echo".to_string(),
220                    description: "Echo".to_string(),
221                    parameters: json!({}),
222                },
223            }
224        }
225
226        fn clone_box(&self) -> Box<dyn LlmTool> {
227            Box::new(EchoTool)
228        }
229    }
230
231    fn exec(id: &str, name: &str, value: &str) -> ToolCallExecution {
232        let mut args = HashMap::new();
233        args.insert("value".to_string(), json!(value));
234        ToolCallExecution {
235            id: id.to_string(),
236            name: name.to_string(),
237            args,
238        }
239    }
240
241    #[tokio::test]
242    async fn serial_preserves_input_order() {
243        let runner = SerialToolRunner;
244        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
245        let calls = vec![
246            exec("1", "echo", "a"),
247            exec("2", "echo", "b"),
248            exec("3", "echo", "c"),
249        ];
250        let ctx = ToolRunCtx::default();
251
252        let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
253
254        assert_eq!(outcomes.iter().map(|o| o.id.as_str()).collect::<Vec<_>>(), vec!["1", "2", "3"]);
255        assert!(outcomes.iter().all(|o| o.ok));
256    }
257
258    #[tokio::test]
259    async fn parallel_preserves_output_order() {
260        let runner = ParallelToolRunner::new(4);
261        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
262        let calls = vec![
263            exec("a", "echo", "1"),
264            exec("b", "echo", "2"),
265            exec("c", "echo", "3"),
266        ];
267        let ctx = ToolRunCtx::default();
268
269        let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
270
271        assert_eq!(outcomes.iter().map(|o| o.id.as_str()).collect::<Vec<_>>(), vec!["a", "b", "c"]);
272    }
273
274    #[tokio::test]
275    async fn unknown_tools_return_not_found() {
276        let runner = SerialToolRunner;
277        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
278        let calls = vec![exec("1", "missing", "x")];
279
280        let outcomes = runner.run_batch(&calls, &tools, &ToolRunCtx::default()).await;
281
282        assert!(!outcomes[0].ok);
283        assert!(outcomes[0].error.as_ref().map(|s| s.contains("missing")).unwrap_or(false));
284    }
285
286    #[tokio::test]
287    async fn pre_cancelled_batch_skips_dispatch() {
288        let runner = ParallelToolRunner::new(2);
289        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
290        let calls = vec![exec("1", "echo", "x"), exec("2", "echo", "y")];
291        let ctx = ToolRunCtx::default();
292        ctx.cancel.cancel();
293
294        let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
295
296        assert!(outcomes.iter().all(|o| !o.ok));
297        assert!(outcomes.iter().all(|o| o.error.as_deref() == Some("Cancelled")));
298    }
299}