1use crate::llm::tools::tool::{LlmTool, ToolRunCtx};
8use async_trait::async_trait;
9use futures::stream::{FuturesUnordered, StreamExt};
10use serde_json::Value;
11use std::collections::HashMap;
12use std::sync::Arc;
13use std::time::Instant;
14use tokio::sync::Semaphore;
15use tokio_util::sync::CancellationToken;
16
17#[derive(Debug, Clone)]
22pub struct ToolCallExecution {
23 pub id: String,
24 pub name: String,
25 pub args: HashMap<String, Value>,
26}
27
28#[derive(Debug, Clone)]
30pub struct ToolCallOutcome {
31 pub id: String,
32 pub name: String,
33 pub ok: bool,
34 pub result: Option<Value>,
35 pub error: Option<String>,
36 pub duration_ms: u64,
37}
38
39#[async_trait]
44pub trait ToolRunner: Send + Sync {
45 async fn run_batch(
46 &self,
47 calls: &[ToolCallExecution],
48 tools: &[Box<dyn LlmTool>],
49 ctx: &ToolRunCtx,
50 ) -> Vec<ToolCallOutcome>;
51}
52
53#[derive(Debug, Default, Clone)]
58pub struct SerialToolRunner;
59
60#[async_trait]
61impl ToolRunner for SerialToolRunner {
62 async fn run_batch(
63 &self,
64 calls: &[ToolCallExecution],
65 tools: &[Box<dyn LlmTool>],
66 ctx: &ToolRunCtx,
67 ) -> Vec<ToolCallOutcome> {
68 let mut outcomes = Vec::with_capacity(calls.len());
69 for call in calls {
70 if ctx.cancel.is_cancelled() {
71 outcomes.push(aborted_outcome(call));
72 continue;
73 }
74 outcomes.push(execute_one(call, tools, ctx).await);
75 }
76 outcomes
77 }
78}
79
80#[derive(Debug, Clone)]
86pub struct ParallelToolRunner {
87 pub max_concurrency: usize,
88}
89
90impl Default for ParallelToolRunner {
91 fn default() -> Self {
92 Self { max_concurrency: 4 }
93 }
94}
95
96impl ParallelToolRunner {
97 pub fn new(max_concurrency: usize) -> Self {
98 assert!(max_concurrency >= 1, "max_concurrency must be >= 1");
99 Self { max_concurrency }
100 }
101}
102
103#[async_trait]
104impl ToolRunner for ParallelToolRunner {
105 async fn run_batch(
106 &self,
107 calls: &[ToolCallExecution],
108 tools: &[Box<dyn LlmTool>],
109 ctx: &ToolRunCtx,
110 ) -> Vec<ToolCallOutcome> {
111 if calls.is_empty() {
112 return Vec::new();
113 }
114
115 let semaphore = Arc::new(Semaphore::new(self.max_concurrency));
116 let mut futures = FuturesUnordered::new();
117
118 for (idx, call) in calls.iter().enumerate() {
119 let permit_sem = Arc::clone(&semaphore);
120 let cancel = ctx.cancel.clone();
121 futures.push(async move {
122 if cancel.is_cancelled() {
123 return (idx, aborted_outcome(call));
124 }
125 let _permit = permit_sem.acquire().await.expect("semaphore closed");
126 if cancel.is_cancelled() {
127 return (idx, aborted_outcome(call));
128 }
129 (idx, execute_one(call, tools, ctx).await)
130 });
131 }
132
133 let mut indexed: Vec<(usize, ToolCallOutcome)> = Vec::with_capacity(calls.len());
134 while let Some(pair) = futures.next().await {
135 indexed.push(pair);
136 }
137 indexed.sort_by_key(|(idx, _)| *idx);
138 indexed.into_iter().map(|(_, o)| o).collect()
139 }
140}
141
142fn aborted_outcome(call: &ToolCallExecution) -> ToolCallOutcome {
143 ToolCallOutcome {
144 id: call.id.clone(),
145 name: call.name.clone(),
146 ok: false,
147 result: None,
148 error: Some(crate::error::MojenticError::Cancelled.to_string()),
149 duration_ms: 0,
150 }
151}
152
153async fn execute_one(
154 call: &ToolCallExecution,
155 tools: &[Box<dyn LlmTool>],
156 ctx: &ToolRunCtx,
157) -> ToolCallOutcome {
158 let start = Instant::now();
159 let tool = tools.iter().find(|t| t.matches(&call.name));
160 let outcome = match tool {
161 None => ToolCallOutcome {
162 id: call.id.clone(),
163 name: call.name.clone(),
164 ok: false,
165 result: None,
166 error: Some(format!("Tool {:?} not found", call.name)),
167 duration_ms: start.elapsed().as_millis() as u64,
168 },
169 Some(tool) => match tool.run(&call.args, ctx).await {
170 Ok(result) => ToolCallOutcome {
171 id: call.id.clone(),
172 name: call.name.clone(),
173 ok: true,
174 result: Some(result),
175 error: None,
176 duration_ms: start.elapsed().as_millis() as u64,
177 },
178 Err(err) => ToolCallOutcome {
179 id: call.id.clone(),
180 name: call.name.clone(),
181 ok: false,
182 result: None,
183 error: Some(format!("{err}")),
184 duration_ms: start.elapsed().as_millis() as u64,
185 },
186 },
187 };
188 outcome
189}
190
191pub fn fresh_cancel_token() -> CancellationToken {
194 CancellationToken::new()
195}
196
197#[cfg(test)]
198mod tests {
199 use super::*;
200 use crate::llm::tools::tool::{FunctionDescriptor, ToolDescriptor};
201 use serde_json::json;
202
203 struct EchoTool;
204
205 #[async_trait]
206 impl LlmTool for EchoTool {
207 async fn run(
208 &self,
209 args: &HashMap<String, Value>,
210 _ctx: &ToolRunCtx,
211 ) -> crate::error::Result<Value> {
212 Ok(json!({ "echo": args.get("value").cloned().unwrap_or_default() }))
213 }
214
215 fn descriptor(&self) -> ToolDescriptor {
216 ToolDescriptor {
217 r#type: "function".to_string(),
218 function: FunctionDescriptor {
219 name: "echo".to_string(),
220 description: "Echo".to_string(),
221 parameters: json!({}),
222 },
223 }
224 }
225
226 fn clone_box(&self) -> Box<dyn LlmTool> {
227 Box::new(EchoTool)
228 }
229 }
230
231 fn exec(id: &str, name: &str, value: &str) -> ToolCallExecution {
232 let mut args = HashMap::new();
233 args.insert("value".to_string(), json!(value));
234 ToolCallExecution {
235 id: id.to_string(),
236 name: name.to_string(),
237 args,
238 }
239 }
240
241 #[tokio::test]
242 async fn serial_preserves_input_order() {
243 let runner = SerialToolRunner;
244 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
245 let calls = vec![
246 exec("1", "echo", "a"),
247 exec("2", "echo", "b"),
248 exec("3", "echo", "c"),
249 ];
250 let ctx = ToolRunCtx::default();
251
252 let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
253
254 assert_eq!(outcomes.iter().map(|o| o.id.as_str()).collect::<Vec<_>>(), vec!["1", "2", "3"]);
255 assert!(outcomes.iter().all(|o| o.ok));
256 }
257
258 #[tokio::test]
259 async fn parallel_preserves_output_order() {
260 let runner = ParallelToolRunner::new(4);
261 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
262 let calls = vec![
263 exec("a", "echo", "1"),
264 exec("b", "echo", "2"),
265 exec("c", "echo", "3"),
266 ];
267 let ctx = ToolRunCtx::default();
268
269 let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
270
271 assert_eq!(outcomes.iter().map(|o| o.id.as_str()).collect::<Vec<_>>(), vec!["a", "b", "c"]);
272 }
273
274 #[tokio::test]
275 async fn unknown_tools_return_not_found() {
276 let runner = SerialToolRunner;
277 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
278 let calls = vec![exec("1", "missing", "x")];
279
280 let outcomes = runner.run_batch(&calls, &tools, &ToolRunCtx::default()).await;
281
282 assert!(!outcomes[0].ok);
283 assert!(outcomes[0].error.as_ref().map(|s| s.contains("missing")).unwrap_or(false));
284 }
285
286 #[tokio::test]
287 async fn pre_cancelled_batch_skips_dispatch() {
288 let runner = ParallelToolRunner::new(2);
289 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(EchoTool)];
290 let calls = vec![exec("1", "echo", "x"), exec("2", "echo", "y")];
291 let ctx = ToolRunCtx::default();
292 ctx.cancel.cancel();
293
294 let outcomes = runner.run_batch(&calls, &tools, &ctx).await;
295
296 assert!(outcomes.iter().all(|o| !o.ok));
297 assert!(outcomes.iter().all(|o| o.error.as_deref() == Some("Cancelled")));
298 }
299}