Skip to main content

mojentic/llm/
broker.rs

1use crate::error::{MojenticError, Result};
2use crate::llm::gateway::{CompletionConfig, LlmGateway, StreamChunk};
3use crate::llm::models::{LlmGatewayResponse, LlmMessage, MessageRole, ResponseEvidence};
4use crate::llm::stream_events::{StreamEvent, StreamEventError, StreamEventStream};
5use crate::llm::tools::{LlmTool, SerialToolRunner, ToolCallExecution, ToolRunCtx, ToolRunner};
6use crate::tracer::TracerSystem;
7use futures::stream::{Stream, StreamExt};
8use serde::{Deserialize, Serialize};
9use std::pin::Pin;
10use std::sync::Arc;
11use tracing::{info, warn};
12use uuid::Uuid;
13
14/// Main interface for LLM interactions
15#[derive(Clone)]
16pub struct LlmBroker {
17    model: String,
18    gateway: Arc<dyn LlmGateway>,
19    tracer: Option<Arc<TracerSystem>>,
20    tool_runner: Arc<dyn ToolRunner>,
21}
22
23impl LlmBroker {
24    /// Create a new LLM broker
25    ///
26    /// Uses [`SerialToolRunner`] as the tool execution strategy. Callers that
27    /// want parallel fan-out should use [`LlmBroker::with_tool_runner`].
28    ///
29    /// # Arguments
30    ///
31    /// * `model` - The name of the LLM model to use
32    /// * `gateway` - The gateway to use for LLM communication
33    /// * `tracer` - Optional tracer system for observability
34    pub fn new(
35        model: impl Into<String>,
36        gateway: Arc<dyn LlmGateway>,
37        tracer: Option<Arc<TracerSystem>>,
38    ) -> Self {
39        Self {
40            model: model.into(),
41            gateway,
42            tracer,
43            tool_runner: Arc::new(SerialToolRunner),
44        }
45    }
46
47    /// Create a new LLM broker with an explicit tool runner.
48    ///
49    /// Use this when you want [`crate::llm::tools::ParallelToolRunner`] (or a
50    /// custom strategy) instead of the default serial runner.
51    pub fn with_tool_runner(
52        model: impl Into<String>,
53        gateway: Arc<dyn LlmGateway>,
54        tracer: Option<Arc<TracerSystem>>,
55        tool_runner: Arc<dyn ToolRunner>,
56    ) -> Self {
57        Self {
58            model: model.into(),
59            gateway,
60            tracer,
61            tool_runner,
62        }
63    }
64
65    /// Return one native provider response without executing tools or extending history.
66    /// The caller supplies the complete context for each request.
67    ///
68    /// # Errors
69    /// Returns gateway errors unchanged.
70    pub async fn generate_response(
71        &self,
72        messages: &[LlmMessage],
73        tools: Option<&[Box<dyn LlmTool>]>,
74        config: Option<CompletionConfig>,
75        correlation_id: Option<String>,
76    ) -> Result<LlmGatewayResponse> {
77        let config = config.unwrap_or_default();
78        let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
79        // Record LLM call
80        if let Some(tracer) = &self.tracer {
81            let messages_json = messages_for_trace(messages);
82
83            let tools_json = tools.map(|t| {
84                t.iter()
85                    .map(|tool| {
86                        let desc = tool.descriptor();
87                        let mut map = std::collections::HashMap::new();
88                        map.insert("name".to_string(), serde_json::json!(desc.function.name));
89                        map.insert(
90                            "description".to_string(),
91                            serde_json::json!(desc.function.description),
92                        );
93                        map
94                    })
95                    .collect()
96            });
97
98            tracer.record_llm_call(
99                &self.model,
100                messages_json,
101                config.temperature.map(f64::from),
102                tools_json,
103                "LlmBroker",
104                &correlation_id,
105            );
106        }
107
108        // Measure call duration
109        let start = std::time::Instant::now();
110
111        // Make initial LLM call
112        let response = self.gateway.complete(&self.model, messages, tools, &config).await?;
113
114        let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
115
116        // Record LLM response
117        if let Some(tracer) = &self.tracer {
118            let tool_calls_json = if !response.tool_calls.is_empty() {
119                Some(
120                    response
121                        .tool_calls
122                        .iter()
123                        .map(|tc| {
124                            let mut map = std::collections::HashMap::new();
125                            map.insert("name".to_string(), serde_json::json!(&tc.name));
126                            if let Some(id) = &tc.id {
127                                map.insert("id".to_string(), serde_json::json!(id));
128                            }
129                            map
130                        })
131                        .collect(),
132                )
133            } else {
134                None
135            };
136
137            tracer.record_llm_response_with_evidence(
138                &self.model,
139                response.content.as_ref().unwrap_or(&String::new()),
140                tool_calls_json,
141                Some(call_duration_ms),
142                response.evidence.clone(),
143                "LlmBroker",
144                &correlation_id,
145            );
146        }
147
148        Ok(response)
149    }
150
151    /// Generate text response from LLM
152    ///
153    /// # Arguments
154    ///
155    /// * `messages` - The messages to send to the LLM
156    /// * `tools` - Optional tools available to the LLM
157    /// * `config` - Optional completion configuration
158    /// * `correlation_id` - Optional correlation ID for tracing (generates UUID if None)
159    pub async fn generate(
160        &self,
161        messages: &[LlmMessage],
162        tools: Option<&[Box<dyn LlmTool>]>,
163        config: Option<CompletionConfig>,
164        correlation_id: Option<String>,
165    ) -> Result<String> {
166        let config = config.unwrap_or_default();
167        let current_messages = messages.to_vec();
168        let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
169
170        let response = self
171            .generate_response(messages, tools, Some(config.clone()), Some(correlation_id.clone()))
172            .await?;
173
174        // Handle tool calls if present
175        if !response.tool_calls.is_empty() {
176            if let Some(tools) = tools {
177                return self
178                    .handle_tool_calls(
179                        current_messages,
180                        response,
181                        tools,
182                        &config,
183                        &correlation_id,
184                        0,
185                    )
186                    .await;
187            }
188        }
189
190        Ok(response.content.unwrap_or_default())
191    }
192
193    fn handle_tool_calls<'a>(
194        &'a self,
195        mut messages: Vec<LlmMessage>,
196        response: LlmGatewayResponse,
197        tools: &'a [Box<dyn LlmTool>],
198        config: &'a CompletionConfig,
199        correlation_id: &'a str,
200        iteration: usize,
201    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String>> + Send + 'a>> {
202        Box::pin(async move {
203            if config.tool_iteration_limit_reached(iteration) {
204                return Err(MojenticError::MaxToolIterationsExceeded {
205                    limit: config.max_tool_iterations,
206                });
207            }
208
209            info!("Tool calls requested: {}", response.tool_calls.len());
210
211            // Record the assistant turn that asked for tools so subsequent
212            // messages have a coherent history.
213            messages.push(LlmMessage {
214                role: MessageRole::Assistant,
215                content: response.content.clone(),
216                tool_calls: Some(response.tool_calls.clone()),
217                image_paths: None,
218                reasoning_content: config
219                    .preserve_reasoning
220                    .then(|| response.thinking.clone())
221                    .flatten(),
222            });
223
224            let outcomes = self
225                .run_tool_batch(&response.tool_calls, tools, correlation_id, "LlmBroker")
226                .await?;
227
228            for (call, outcome) in response.tool_calls.iter().zip(outcomes.iter()) {
229                let content = if outcome.ok {
230                    serde_json::to_string(
231                        outcome.result.as_ref().unwrap_or(&serde_json::Value::Null),
232                    )?
233                } else {
234                    serde_json::json!({
235                        "error": outcome.error.clone().unwrap_or_default(),
236                    })
237                    .to_string()
238                };
239                messages.push(LlmMessage {
240                    role: MessageRole::Tool,
241                    content: Some(content),
242                    tool_calls: Some(vec![call.clone()]),
243                    image_paths: None,
244                    reasoning_content: None,
245                });
246            }
247
248            let next_response = self
249                .generate_response(
250                    &messages,
251                    Some(tools),
252                    Some(config.clone()),
253                    Some(correlation_id.to_owned()),
254                )
255                .await?;
256
257            if !next_response.tool_calls.is_empty() {
258                return self
259                    .handle_tool_calls(
260                        messages,
261                        next_response,
262                        tools,
263                        config,
264                        correlation_id,
265                        iteration.saturating_add(1),
266                    )
267                    .await;
268            }
269
270            Ok(next_response.content.unwrap_or_default())
271        })
272    }
273
274    /// Dispatch a batch of tool calls through the configured [`ToolRunner`]
275    /// and record per-call + batch tracer events.
276    async fn run_tool_batch(
277        &self,
278        tool_calls: &[crate::llm::models::LlmToolCall],
279        tools: &[Box<dyn LlmTool>],
280        correlation_id: &str,
281        source: &'static str,
282    ) -> Result<Vec<crate::llm::tools::ToolCallOutcome>> {
283        let executions: Vec<ToolCallExecution> = tool_calls
284            .iter()
285            .enumerate()
286            .map(|(idx, tc)| ToolCallExecution {
287                id: tc.id.clone().unwrap_or_else(|| format!("call_{idx}")),
288                name: tc.name.clone(),
289                args: tc.arguments.clone(),
290            })
291            .collect();
292
293        let ctx = ToolRunCtx {
294            correlation_id: Some(correlation_id.to_string()),
295            source: Some(source.to_string()),
296            ..Default::default()
297        };
298
299        let batch_start = std::time::Instant::now();
300        let outcomes = self.tool_runner.run_batch(&executions, tools, &ctx).await;
301        let batch_duration_ms = batch_start.elapsed().as_secs_f64() * 1000.0;
302
303        if let Some(tracer) = &self.tracer {
304            for (exec, outcome) in executions.iter().zip(outcomes.iter()) {
305                tracer.record_tool_call(
306                    &outcome.name,
307                    exec.args.clone(),
308                    outcome.result.clone().unwrap_or(serde_json::Value::Null),
309                    Some(source.to_string()),
310                    Some(outcome.duration_ms as f64),
311                    source,
312                    correlation_id,
313                );
314            }
315            if outcomes.len() > 1 {
316                let ok = outcomes.iter().filter(|o| o.ok).count();
317                let fail = outcomes.len() - ok;
318                tracer.record_tool_batch(
319                    uuid::Uuid::new_v4().to_string(),
320                    outcomes.iter().map(|o| o.name.clone()).collect(),
321                    ok,
322                    fail,
323                    batch_duration_ms,
324                    Some(source.to_string()),
325                    source,
326                    correlation_id,
327                );
328            }
329        }
330
331        // Tool execution errors surface inside outcomes; we only short-circuit
332        // if no outcomes were produced (which the runner shouldn't do, but
333        // belt-and-braces).
334        if outcomes.len() != tool_calls.len() {
335            return Err(MojenticError::ToolError(format!(
336                "tool runner returned {} outcomes for {} calls",
337                outcomes.len(),
338                tool_calls.len()
339            )));
340        }
341
342        Ok(outcomes)
343    }
344
345    /// Generate structured object response from LLM
346    ///
347    /// # Arguments
348    ///
349    /// * `messages` - The messages to send to the LLM
350    /// * `config` - Optional completion configuration
351    /// * `correlation_id` - Optional correlation ID for tracing (generates UUID if None)
352    pub async fn generate_object<T>(
353        &self,
354        messages: &[LlmMessage],
355        config: Option<CompletionConfig>,
356        correlation_id: Option<String>,
357    ) -> Result<T>
358    where
359        T: for<'de> Deserialize<'de> + Serialize + schemars::JsonSchema + Send,
360    {
361        let config = config.unwrap_or_default();
362        let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
363
364        // Generate JSON schema for the type
365        let schema = serde_json::to_value(schemars::schema_for!(T))?;
366
367        // Record LLM call
368        if let Some(tracer) = &self.tracer {
369            let messages_json = messages_for_trace(messages);
370
371            tracer.record_llm_call(
372                &self.model,
373                messages_json,
374                config.temperature.map(f64::from),
375                None,
376                "LlmBroker::generate_object",
377                &correlation_id,
378            );
379        }
380
381        // Measure call duration
382        let start = std::time::Instant::now();
383
384        // Call the gateway with the schema
385        let response = self
386            .gateway
387            .complete_json_response(&self.model, messages, schema, &config)
388            .await?;
389
390        let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
391
392        let json_response = response.object.ok_or_else(|| {
393            MojenticError::GatewayError("Structured response carried no object".to_string())
394        })?;
395
396        // Record LLM response
397        if let Some(tracer) = &self.tracer {
398            let object_str = serde_json::to_string_pretty(&json_response).unwrap_or_default();
399            tracer.record_llm_response_with_evidence(
400                &self.model,
401                format!("Structured response: {}", object_str),
402                None,
403                Some(call_duration_ms),
404                response.evidence,
405                "LlmBroker::generate_object",
406                &correlation_id,
407            );
408        }
409
410        // Deserialize the JSON into the target type
411        let object: T = serde_json::from_value(json_response)?;
412
413        Ok(object)
414    }
415
416    /// Generate streaming text response from LLM
417    ///
418    /// Returns a stream that yields content chunks as they arrive. When tool calls
419    /// are detected, the broker executes them and recursively streams the LLM's
420    /// follow-up response.
421    ///
422    /// # Arguments
423    ///
424    /// * `messages` - The messages to send to the LLM
425    /// * `tools` - Optional tools available to the LLM
426    /// * `config` - Optional completion configuration
427    /// * `correlation_id` - Optional correlation ID for tracing (generates UUID if None)
428    ///
429    /// # Example
430    ///
431    /// ```ignore
432    /// use futures::stream::StreamExt;
433    ///
434    /// let broker = LlmBroker::new("qwen3:32b", gateway, None);
435    /// let messages = vec![LlmMessage::user("Tell me a story")];
436    ///
437    /// let mut stream = broker.generate_stream(&messages, None, None, None);
438    /// while let Some(result) = stream.next().await {
439    ///     match result {
440    ///         Ok(chunk) => print!("{}", chunk),
441    ///         Err(e) => eprintln!("Error: {}", e),
442    ///     }
443    /// }
444    /// ```
445    pub fn generate_stream<'a>(
446        &'a self,
447        messages: &'a [LlmMessage],
448        tools: Option<&'a [Box<dyn LlmTool>]>,
449        config: Option<CompletionConfig>,
450        correlation_id: Option<String>,
451    ) -> Pin<Box<dyn Stream<Item = Result<String>> + 'a>> {
452        let config = config.unwrap_or_default();
453        let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
454        Box::pin(self.generate_stream_with_depth(
455            messages.to_vec(),
456            tools,
457            config,
458            correlation_id,
459            0,
460        ))
461    }
462
463    fn generate_stream_with_depth<'a>(
464        &'a self,
465        current_messages: Vec<LlmMessage>,
466        tools: Option<&'a [Box<dyn LlmTool>]>,
467        config: CompletionConfig,
468        correlation_id: String,
469        depth: usize,
470    ) -> impl Stream<Item = Result<String>> + 'a {
471        async_stream::stream! {
472            if config.tool_iteration_limit_reached(depth) {
473                yield Err(MojenticError::MaxToolIterationsExceeded {
474                    limit: config.max_tool_iterations,
475                });
476                return;
477            }
478
479            // Record LLM call
480            if let Some(tracer) = &self.tracer {
481                let messages_json = messages_for_trace(&current_messages);
482
483                let tools_json = tools.map(|t| {
484                    t.iter()
485                        .map(|tool| {
486                            let desc = tool.descriptor();
487                            let mut map = std::collections::HashMap::new();
488                            map.insert("name".to_string(), serde_json::json!(desc.function.name));
489                            map.insert(
490                                "description".to_string(),
491                                serde_json::json!(desc.function.description),
492                            );
493                            map
494                        })
495                        .collect()
496                });
497
498                tracer.record_llm_call(
499                    &self.model,
500                    messages_json,
501                    config.temperature.map(f64::from),
502                    tools_json,
503                    "LlmBroker::generate_stream",
504                    &correlation_id,
505                );
506            }
507
508            let mut stream_evidence = ResponseEvidence::default();
509            let mut accumulated_content = String::new();
510            let mut accumulated_reasoning = String::new();
511            let mut accumulated_tool_calls = Vec::new();
512
513            // Measure stream duration
514            let start = std::time::Instant::now();
515
516            // Stream from gateway
517            let mut stream = self.gateway.complete_stream(
518                &self.model,
519                &current_messages,
520                tools,
521                &config,
522            );
523
524            let mut stream_error = None;
525            while let Some(chunk_result) = stream.next().await {
526                match chunk_result {
527                    Ok(StreamChunk::Content(content)) => {
528                        accumulated_content.push_str(&content);
529                        yield Ok(content);
530                    }
531                    Ok(StreamChunk::ToolCalls(tool_calls)) => {
532                        accumulated_tool_calls = tool_calls;
533                    }
534                    Ok(StreamChunk::Thinking(thinking)) => {
535                        if config.preserve_reasoning { accumulated_reasoning.push_str(&thinking); }
536                    }
537                    Ok(StreamChunk::Progress(_)) => {},
538                    Ok(StreamChunk::Metrics(metrics)) => {
539                        crate::llm::stream_timing::retain_timing(&mut stream_evidence, &metrics);
540                        stream_evidence.metadata.insert("stream_metrics".into(), serde_json::json!(metrics));
541                        stream_evidence.usage = metrics.provider_usage;
542                    }
543                    Err(e) => {
544                        stream_error = Some(e);
545                        break;
546                    }
547                }
548            }
549
550            let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
551
552            // Record LLM response
553            if let Some(tracer) = &self.tracer {
554                let tool_calls_json = if !accumulated_tool_calls.is_empty() {
555                    Some(
556                        accumulated_tool_calls
557                            .iter()
558                            .map(|tc| {
559                                let mut map = std::collections::HashMap::new();
560                                map.insert("name".to_string(), serde_json::json!(&tc.name));
561                                if let Some(id) = &tc.id {
562                                    map.insert("id".to_string(), serde_json::json!(id));
563                                }
564                                map
565                            })
566                            .collect(),
567                    )
568                } else {
569                    None
570                };
571
572                tracer.record_llm_response_with_evidence(
573                    &self.model,
574                    &accumulated_content,
575                    tool_calls_json,
576                    Some(call_duration_ms),
577                    stream_evidence,
578                    "LlmBroker::generate_stream",
579                    &correlation_id,
580                );
581            }
582
583            // Publish terminal timing before propagating the original error.
584            // Failed output must never reach tool execution or session history.
585            if let Some(error) = stream_error {
586                yield Err(error);
587                return;
588            }
589
590            // Handle tool calls if present
591            if !accumulated_tool_calls.is_empty() {
592                if let Some(tools) = tools {
593                    info!("Processing {} tool call(s) in stream", accumulated_tool_calls.len());
594
595                    // Build new messages with tool results
596                    let mut new_messages = current_messages.clone();
597
598                    // Add assistant message with tool calls
599                    new_messages.push(LlmMessage {
600                        role: MessageRole::Assistant,
601                        content: Some(accumulated_content),
602                        tool_calls: Some(accumulated_tool_calls.clone()),
603                        image_paths: None,
604                        reasoning_content: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning),
605                    });
606
607                    let outcomes = match self
608                        .run_tool_batch(
609                            &accumulated_tool_calls,
610                            tools,
611                            &correlation_id,
612                            "LlmBroker::generate_stream",
613                        )
614                        .await
615                    {
616                        Ok(o) => o,
617                        Err(e) => {
618                            yield Err(e);
619                            return;
620                        }
621                    };
622
623                    for (call, outcome) in accumulated_tool_calls.iter().zip(outcomes.iter()) {
624                        let content = if outcome.ok {
625                            match serde_json::to_string(
626                                outcome.result.as_ref().unwrap_or(&serde_json::Value::Null),
627                            ) {
628                                Ok(s) => s,
629                                Err(e) => {
630                                    yield Err(e.into());
631                                    return;
632                                }
633                            }
634                        } else {
635                            serde_json::json!({
636                                "error": outcome.error.clone().unwrap_or_default(),
637                            })
638                            .to_string()
639                        };
640                        new_messages.push(LlmMessage {
641                            role: MessageRole::Tool,
642                            content: Some(content),
643                            tool_calls: Some(vec![call.clone()]),
644                            image_paths: None,
645                            reasoning_content: None,
646                        });
647                    }
648
649                    // Continue streaming with updated messages, incrementing depth
650                    let mut recursive_stream = Box::pin(self.generate_stream_with_depth(
651                        new_messages,
652                        Some(tools),
653                        config.clone(),
654                        correlation_id.clone(),
655                        depth.saturating_add(1),
656                    ));
657
658                    while let Some(result) = recursive_stream.next().await {
659                        yield result;
660                    }
661                } else {
662                    warn!("LLM requested tool calls but no tools provided");
663                }
664            }
665        }
666    }
667
668    /// Stream one turn as [`StreamEvent`]s, ending with terminal completion evidence.
669    ///
670    /// The stream yields [`StreamEvent::Content`] in order, then exactly one
671    /// terminal event: [`StreamEvent::Completed`] when the provider finished
672    /// with `stop` and sent its terminal marker, or [`StreamEvent::Error`]
673    /// otherwise. Nothing follows the terminal event. Content that arrived
674    /// before an error is evidence, not a complete result.
675    ///
676    /// This sends one request with no tools. Tool iterations are forced to
677    /// zero, and there is no retry, continuation or recursion. Dropping the
678    /// stream (or breaking out of the loop that reads it) cancels the request.
679    ///
680    /// The tracer records the call when the request starts and the response,
681    /// with its evidence, when the terminal event arrives. A consumer that
682    /// stops early gets a traced call and no traced response. A gateway that
683    /// does not support this API yields a single
684    /// [`StreamEventError::StreamEventsUnsupported`] error, sends no request,
685    /// and records nothing.
686    ///
687    /// # Example
688    ///
689    /// ```no_run
690    /// use futures::StreamExt;
691    /// use mojentic::llm::gateways::OllamaGateway;
692    /// use mojentic::llm::{LlmBroker, LlmMessage, StreamEvent};
693    /// use std::sync::Arc;
694    ///
695    /// # async fn run() {
696    /// let broker = LlmBroker::new("qwen3:32b", Arc::new(OllamaGateway::new()), None);
697    /// let messages = vec![LlmMessage::user("Write a haiku about rivers.")];
698    ///
699    /// let mut events = broker.generate_stream_events(&messages, None, None);
700    /// while let Some(event) = events.next().await {
701    ///     match event {
702    ///         StreamEvent::Content(text) => print!("{text}"),
703    ///         StreamEvent::Completed(evidence) => println!("\nusage: {:?}", evidence.usage),
704    ///         StreamEvent::Error(error) => eprintln!("\nincomplete: {error}"),
705    ///         _ => {}
706    ///     }
707    /// }
708    /// # }
709    /// ```
710    pub fn generate_stream_events<'a>(
711        &'a self,
712        messages: &'a [LlmMessage],
713        config: Option<CompletionConfig>,
714        correlation_id: Option<String>,
715    ) -> StreamEventStream<'a> {
716        const SOURCE: &str = "LlmBroker::generate_stream_events";
717        let config = CompletionConfig {
718            max_tool_iterations: 0,
719            ..config.unwrap_or_default()
720        };
721        let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
722
723        Box::pin(async_stream::stream! {
724            let mut events =
725                match self.gateway.complete_stream_events(&self.model, messages, &config) {
726                    Ok(events) => events,
727                    Err(error) => {
728                        yield StreamEvent::Error(error);
729                        return;
730                    }
731                };
732
733            if let Some(tracer) = &self.tracer {
734                tracer.record_llm_call(
735                    &self.model,
736                    messages_for_trace(messages),
737                    config.temperature.map(f64::from),
738                    None,
739                    SOURCE,
740                    &correlation_id,
741                );
742            }
743            let start = std::time::Instant::now();
744
745            let mut content = String::new();
746            let terminal = loop {
747                match events.next().await {
748                    Some(StreamEvent::Content(text)) => {
749                        content.push_str(&text);
750                        yield StreamEvent::Content(text);
751                    }
752                    Some(StreamEvent::Metrics(metrics)) => yield StreamEvent::Metrics(metrics),
753                    Some(terminal) => break terminal,
754                    None => break StreamEvent::Error(StreamEventError::IncompleteStream(None)),
755                }
756            };
757
758            if let Some(tracer) = &self.tracer {
759                tracer.record_llm_response_with_evidence(
760                    &self.model,
761                    content,
762                    None,
763                    Some(start.elapsed().as_secs_f64() * 1000.0),
764                    terminal_evidence(&terminal),
765                    SOURCE,
766                    &correlation_id,
767                );
768            }
769            yield terminal;
770        })
771    }
772}
773
774/// The simplified message list recorded on an LLM call trace.
775fn messages_for_trace(
776    messages: &[LlmMessage],
777) -> Vec<std::collections::HashMap<String, serde_json::Value>> {
778    messages
779        .iter()
780        .map(|m| {
781            let mut map = std::collections::HashMap::new();
782            map.insert("role".to_string(), serde_json::json!(format!("{:?}", m.role)));
783            if let Some(content) = &m.content {
784                map.insert("content".to_string(), serde_json::json!(content));
785            }
786            map
787        })
788        .collect()
789}
790
791/// The provider evidence a terminal stream event carries, for the response trace.
792fn terminal_evidence(terminal: &StreamEvent) -> ResponseEvidence {
793    match terminal {
794        StreamEvent::Completed(evidence) => evidence.clone(),
795        StreamEvent::Error(StreamEventError::IncompleteCompletion(evidence))
796        | StreamEvent::Error(StreamEventError::IncompleteStream(Some(evidence))) => {
797            evidence.as_ref().clone()
798        }
799        _ => ResponseEvidence::default(),
800    }
801}
802
803#[cfg(test)]
804mod tests {
805    use super::*;
806    use crate::llm::models::LlmToolCall;
807    use crate::llm::tools::{FunctionDescriptor, ToolDescriptor};
808    use async_trait::async_trait;
809    use serde::{Deserialize, Serialize};
810    use serde_json::Value;
811    use std::collections::HashMap;
812
813    #[tokio::test]
814    async fn native_response_leaves_tool_dispatch_to_caller() {
815        let response = LlmGatewayResponse {
816            content: None,
817            object: None,
818            thinking: None,
819            tool_calls: vec![LlmToolCall {
820                id: Some("call-1".into()),
821                name: "unknown".into(),
822                arguments: HashMap::new(),
823            }],
824            ..Default::default()
825        };
826        let broker =
827            LlmBroker::new("test".to_string(), Arc::new(MockGateway::new(vec![response])), None);
828        let messages = vec![LlmMessage::user("hello")];
829        let actual = broker
830            .generate_response(&messages, Some(&[]), None, None)
831            .await
832            .expect("native response");
833        assert_eq!(actual.tool_calls[0].id.as_deref(), Some("call-1"));
834        assert_eq!(messages.len(), 1);
835    }
836
837    #[tokio::test]
838    async fn unlimited_execution_can_continue_past_default_rounds() {
839        let response = LlmGatewayResponse {
840            content: None,
841            object: None,
842            thinking: None,
843            tool_calls: vec![LlmToolCall {
844                id: Some("call".into()),
845                name: "unknown".into(),
846                arguments: HashMap::new(),
847            }],
848            ..Default::default()
849        };
850        let mut responses = vec![response; 12];
851        responses.push(LlmGatewayResponse {
852            content: Some("done".into()),
853            object: None,
854            thinking: None,
855            tool_calls: vec![],
856            ..Default::default()
857        });
858        let broker =
859            LlmBroker::new("test".to_string(), Arc::new(MockGateway::new(responses)), None);
860        let actual = broker
861            .generate(
862                &[LlmMessage::user("hello")],
863                Some(&[]),
864                Some(CompletionConfig::default().with_unlimited_tool_iterations()),
865                None,
866            )
867            .await
868            .expect("unlimited completion");
869        assert_eq!(actual, "done");
870    }
871
872    fn reported_evidence() -> crate::llm::models::ResponseEvidence {
873        crate::llm::models::ResponseEvidence {
874            usage: Some(serde_json::json!({"prompt_tokens": 9, "completion_tokens": 4})),
875            provider_model: Some("provider-model-2026".to_string()),
876            finish_reason: Some("stop".to_string()),
877            metadata: HashMap::from([("id".to_string(), serde_json::json!("resp-1"))]),
878        }
879    }
880
881    /// A gateway whose ordinary and structured responses carry fixed evidence.
882    struct EvidenceGateway {
883        evidence: crate::llm::models::ResponseEvidence,
884    }
885
886    #[async_trait::async_trait]
887    impl LlmGateway for EvidenceGateway {
888        async fn complete(
889            &self,
890            _model: &str,
891            _messages: &[LlmMessage],
892            _tools: Option<&[Box<dyn LlmTool>]>,
893            _config: &CompletionConfig,
894        ) -> Result<LlmGatewayResponse> {
895            Ok(LlmGatewayResponse {
896                content: Some("answer".to_string()),
897                evidence: self.evidence.clone(),
898                ..Default::default()
899            })
900        }
901
902        async fn complete_json(
903            &self,
904            _model: &str,
905            _messages: &[LlmMessage],
906            _schema: Value,
907            _config: &CompletionConfig,
908        ) -> Result<Value> {
909            unreachable!("the broker asks for the structured response with evidence")
910        }
911
912        async fn complete_json_response(
913            &self,
914            _model: &str,
915            _messages: &[LlmMessage],
916            _schema: Value,
917            _config: &CompletionConfig,
918        ) -> Result<LlmGatewayResponse<Value>> {
919            Ok(LlmGatewayResponse {
920                object: Some(serde_json::json!({"test": "value"})),
921                evidence: self.evidence.clone(),
922                ..Default::default()
923            })
924        }
925
926        async fn get_available_models(&self) -> Result<Vec<String>> {
927            Ok(vec![])
928        }
929
930        async fn calculate_embeddings(
931            &self,
932            _text: &str,
933            _model: Option<&str>,
934        ) -> Result<Vec<f32>> {
935            Ok(vec![])
936        }
937
938        fn complete_stream<'a>(
939            &'a self,
940            _model: &'a str,
941            _messages: &'a [LlmMessage],
942            _tools: Option<&'a [Box<dyn LlmTool>]>,
943            _config: &'a CompletionConfig,
944        ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
945            Box::pin(futures::stream::empty())
946        }
947    }
948
949    #[tokio::test]
950    async fn generate_response_traces_reported_evidence_unchanged() {
951        use crate::tracer::testing::{capturing_tracer, responses};
952
953        let (tracer, captured) = capturing_tracer();
954        let gateway = Arc::new(EvidenceGateway {
955            evidence: reported_evidence(),
956        });
957        let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
958
959        let response = broker
960            .generate_response(&[LlmMessage::user("q")], None, None, None)
961            .await
962            .unwrap();
963
964        assert_eq!(response.evidence, reported_evidence());
965        assert!(matches!(
966            captured.lock().unwrap().first(),
967            Some(crate::tracer::testing::CapturedLlmEvent::Call(call))
968                if call.model == "configured-model"
969        ));
970        let events = responses(&captured);
971        assert_eq!(events.len(), 1);
972        assert_eq!(events[0].model, "configured-model");
973        assert_eq!(events[0].evidence, reported_evidence());
974    }
975
976    #[tokio::test]
977    async fn generate_response_traces_unknown_usage_as_none() {
978        use crate::tracer::testing::{capturing_tracer, responses};
979
980        let (tracer, captured) = capturing_tracer();
981        let gateway = Arc::new(EvidenceGateway {
982            evidence: Default::default(),
983        });
984        let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
985
986        broker.generate(&[LlmMessage::user("q")], None, None, None).await.unwrap();
987
988        let events = responses(&captured);
989        assert_eq!(events.len(), 1);
990        assert_eq!(events[0].evidence.usage, None);
991        assert_eq!(events[0].evidence.provider_model, None);
992    }
993
994    #[tokio::test]
995    async fn generate_object_traces_reported_evidence_unchanged() {
996        use crate::tracer::testing::{capturing_tracer, responses};
997
998        #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
999        struct TestObject {
1000            test: String,
1001        }
1002
1003        let (tracer, captured) = capturing_tracer();
1004        let gateway = Arc::new(EvidenceGateway {
1005            evidence: reported_evidence(),
1006        });
1007        let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
1008
1009        let object: TestObject =
1010            broker.generate_object(&[LlmMessage::user("q")], None, None).await.unwrap();
1011
1012        assert_eq!(object.test, "value");
1013        let events = responses(&captured);
1014        assert_eq!(events.len(), 1);
1015        assert_eq!(events[0].evidence, reported_evidence());
1016    }
1017
1018    mod stream_events {
1019        use super::*;
1020        use crate::llm::models::ResponseEvidence;
1021        use crate::llm::stream_events::{StreamEvent, StreamEventError, StreamEventStream};
1022        use crate::tracer::testing::{capturing_tracer, responses, CapturedLlmEvent};
1023
1024        /// Scripted event factory: a fresh copy of the script on every call.
1025        type Script = fn() -> Vec<StreamEvent>;
1026
1027        /// A gateway that streams a fixed script of events and remembers the config it saw.
1028        struct ScriptedEventsGateway {
1029            script: Script,
1030            seen_tool_limit: std::sync::Mutex<Option<usize>>,
1031        }
1032
1033        impl ScriptedEventsGateway {
1034            fn new(script: Script) -> Self {
1035                Self {
1036                    script,
1037                    seen_tool_limit: std::sync::Mutex::new(None),
1038                }
1039            }
1040        }
1041
1042        #[async_trait::async_trait]
1043        impl LlmGateway for ScriptedEventsGateway {
1044            async fn complete(
1045                &self,
1046                _model: &str,
1047                _messages: &[LlmMessage],
1048                _tools: Option<&[Box<dyn LlmTool>]>,
1049                _config: &CompletionConfig,
1050            ) -> Result<LlmGatewayResponse> {
1051                unreachable!("the events API makes no ordinary completion")
1052            }
1053
1054            async fn complete_json(
1055                &self,
1056                _model: &str,
1057                _messages: &[LlmMessage],
1058                _schema: Value,
1059                _config: &CompletionConfig,
1060            ) -> Result<Value> {
1061                unreachable!("the events API makes no structured completion")
1062            }
1063
1064            async fn get_available_models(&self) -> Result<Vec<String>> {
1065                Ok(vec![])
1066            }
1067
1068            async fn calculate_embeddings(
1069                &self,
1070                _text: &str,
1071                _model: Option<&str>,
1072            ) -> Result<Vec<f32>> {
1073                Ok(vec![])
1074            }
1075
1076            fn complete_stream<'a>(
1077                &'a self,
1078                _model: &'a str,
1079                _messages: &'a [LlmMessage],
1080                _tools: Option<&'a [Box<dyn LlmTool>]>,
1081                _config: &'a CompletionConfig,
1082            ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1083                unreachable!("the events API does not use the legacy stream")
1084            }
1085
1086            fn complete_stream_events<'a>(
1087                &'a self,
1088                _model: &'a str,
1089                _messages: &'a [LlmMessage],
1090                config: &'a CompletionConfig,
1091            ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
1092                *self.seen_tool_limit.lock().unwrap() = Some(config.max_tool_iterations);
1093                Ok(Box::pin(futures::stream::iter((self.script)())))
1094            }
1095        }
1096
1097        fn stop_evidence() -> ResponseEvidence {
1098            ResponseEvidence {
1099                usage: Some(serde_json::json!({"total_tokens": 7})),
1100                provider_model: Some("provider-model".to_string()),
1101                finish_reason: Some("stop".to_string()),
1102                metadata: HashMap::from([("total_duration".to_string(), serde_json::json!(900))]),
1103            }
1104        }
1105
1106        fn broker_with(
1107            script: Script,
1108        ) -> (LlmBroker, Arc<ScriptedEventsGateway>, crate::tracer::testing::CapturedLlmEvents)
1109        {
1110            let gateway = Arc::new(ScriptedEventsGateway::new(script));
1111            let (tracer, captured) = capturing_tracer();
1112            let broker = LlmBroker::new("configured-model", gateway.clone(), Some(tracer));
1113            (broker, gateway, captured)
1114        }
1115
1116        #[tokio::test]
1117        async fn content_then_completed_is_forwarded_and_traced_with_evidence() {
1118            let (broker, _, captured) = broker_with(|| {
1119                vec![
1120                    StreamEvent::Content("Hel".into()),
1121                    StreamEvent::Content("lo".into()),
1122                    StreamEvent::Completed(stop_evidence()),
1123                ]
1124            });
1125            let messages = vec![LlmMessage::user("Hi")];
1126
1127            let events: Vec<StreamEvent> =
1128                broker.generate_stream_events(&messages, None, None).collect().await;
1129
1130            assert!(matches!(
1131                events.as_slice(),
1132                [StreamEvent::Content(a), StreamEvent::Content(b), StreamEvent::Completed(evidence)]
1133                    if a == "Hel" && b == "lo" && *evidence == stop_evidence()
1134            ));
1135            let recorded = captured.lock().unwrap().clone();
1136            assert!(matches!(
1137                recorded.as_slice(),
1138                [CapturedLlmEvent::Call(call), CapturedLlmEvent::Response(response)]
1139                    if call.model == "configured-model"
1140                        && call.tools.is_none()
1141                        && response.content == "Hello"
1142                        && response.model == "configured-model"
1143                        && response.evidence == stop_evidence()
1144            ));
1145        }
1146
1147        #[tokio::test]
1148        async fn incomplete_completion_is_traced_with_its_evidence() {
1149            let (broker, _, captured) = broker_with(|| {
1150                vec![
1151                    StreamEvent::Content("Part".into()),
1152                    StreamEvent::Error(StreamEventError::IncompleteCompletion(Box::new(
1153                        ResponseEvidence {
1154                            finish_reason: Some("length".into()),
1155                            usage: Some(serde_json::json!({"completion_tokens": 16})),
1156                            ..Default::default()
1157                        },
1158                    ))),
1159                ]
1160            });
1161            let messages = vec![LlmMessage::user("Hi")];
1162
1163            let events: Vec<StreamEvent> =
1164                broker.generate_stream_events(&messages, None, None).collect().await;
1165
1166            assert!(matches!(
1167                events.last(),
1168                Some(StreamEvent::Error(StreamEventError::IncompleteCompletion(_)))
1169            ));
1170            let traced = responses(&captured);
1171            assert_eq!(traced.len(), 1);
1172            assert_eq!(traced[0].content, "Part");
1173            assert_eq!(traced[0].evidence.finish_reason.as_deref(), Some("length"));
1174            assert_eq!(
1175                traced[0].evidence.usage,
1176                Some(serde_json::json!({"completion_tokens": 16}))
1177            );
1178        }
1179
1180        #[tokio::test]
1181        async fn incomplete_stream_is_traced_with_its_partial_evidence() {
1182            let (broker, _, captured) = broker_with(|| {
1183                vec![StreamEvent::Error(StreamEventError::IncompleteStream(
1184                    Some(Box::new(ResponseEvidence {
1185                        provider_model: Some("provider-model".into()),
1186                        ..Default::default()
1187                    })),
1188                ))]
1189            });
1190            let messages = vec![LlmMessage::user("Hi")];
1191
1192            let _: Vec<StreamEvent> =
1193                broker.generate_stream_events(&messages, None, None).collect().await;
1194
1195            let traced = responses(&captured);
1196            assert_eq!(traced[0].evidence.provider_model.as_deref(), Some("provider-model"));
1197        }
1198
1199        #[tokio::test]
1200        async fn a_gateway_stream_without_terminal_event_ends_incomplete() {
1201            let (broker, _, captured) = broker_with(|| vec![StreamEvent::Content("Hi".into())]);
1202            let messages = vec![LlmMessage::user("Hi")];
1203
1204            let events: Vec<StreamEvent> =
1205                broker.generate_stream_events(&messages, None, None).collect().await;
1206
1207            assert!(matches!(
1208                events.as_slice(),
1209                [
1210                    StreamEvent::Content(_),
1211                    StreamEvent::Error(StreamEventError::IncompleteStream(None))
1212                ]
1213            ));
1214            assert_eq!(responses(&captured).len(), 1);
1215        }
1216
1217        #[tokio::test]
1218        async fn nothing_follows_the_terminal_event() {
1219            let (broker, _, _) = broker_with(|| {
1220                vec![
1221                    StreamEvent::Completed(stop_evidence()),
1222                    StreamEvent::Content("late".into()),
1223                    StreamEvent::Completed(stop_evidence()),
1224                ]
1225            });
1226            let messages = vec![LlmMessage::user("Hi")];
1227
1228            let events: Vec<StreamEvent> =
1229                broker.generate_stream_events(&messages, None, None).collect().await;
1230
1231            assert!(matches!(events.as_slice(), [StreamEvent::Completed(_)]));
1232        }
1233
1234        #[tokio::test]
1235        async fn tool_iterations_are_forced_to_zero() {
1236            let (broker, gateway, _) =
1237                broker_with(|| vec![StreamEvent::Completed(stop_evidence())]);
1238            let messages = vec![LlmMessage::user("Hi")];
1239            let config = CompletionConfig::default().with_unlimited_tool_iterations();
1240
1241            let _: Vec<StreamEvent> =
1242                broker.generate_stream_events(&messages, Some(config), None).collect().await;
1243
1244            assert_eq!(*gateway.seen_tool_limit.lock().unwrap(), Some(0));
1245        }
1246
1247        #[tokio::test]
1248        async fn stopping_early_traces_the_call_but_no_response() {
1249            let (broker, _, captured) = broker_with(|| {
1250                vec![
1251                    StreamEvent::Content("Hi".into()),
1252                    StreamEvent::Completed(stop_evidence()),
1253                ]
1254            });
1255            let messages = vec![LlmMessage::user("Hi")];
1256
1257            let mut stream = broker.generate_stream_events(&messages, None, None);
1258            assert!(matches!(stream.next().await, Some(StreamEvent::Content(_))));
1259            drop(stream);
1260
1261            let recorded = captured.lock().unwrap().clone();
1262            assert!(matches!(recorded.as_slice(), [CapturedLlmEvent::Call(_)]));
1263        }
1264
1265        #[tokio::test]
1266        async fn an_unsupported_gateway_yields_one_error_without_a_trace() {
1267            let (tracer, captured) = capturing_tracer();
1268            let broker = LlmBroker::new(
1269                "configured-model",
1270                Arc::new(MockGateway::new(vec![])),
1271                Some(tracer),
1272            );
1273            let messages = vec![LlmMessage::user("Hi")];
1274
1275            let events: Vec<StreamEvent> =
1276                broker.generate_stream_events(&messages, None, None).collect().await;
1277
1278            assert!(matches!(
1279                events.as_slice(),
1280                [StreamEvent::Error(
1281                    StreamEventError::StreamEventsUnsupported
1282                )]
1283            ));
1284            assert!(captured.lock().unwrap().is_empty());
1285        }
1286
1287        #[tokio::test]
1288        async fn dropping_the_broker_stream_cancels_the_provider_request() {
1289            use crate::llm::gateways::OllamaGateway;
1290            use crate::llm::stream_events::testing::endless_stream_server;
1291
1292            let (url, closed) =
1293                endless_stream_server("{\"message\":{\"content\":\"Hi\"},\"done\":false}\n").await;
1294            let broker = LlmBroker::new("qwen3:32b", Arc::new(OllamaGateway::with_host(url)), None);
1295            let messages = vec![LlmMessage::user("Hi")];
1296
1297            let mut stream = broker.generate_stream_events(&messages, None, None);
1298            assert!(matches!(stream.next().await, Some(StreamEvent::Content(_))));
1299            drop(stream);
1300
1301            tokio::time::timeout(std::time::Duration::from_secs(5), closed)
1302                .await
1303                .expect("the request was not cancelled")
1304                .expect("server reported the disconnect");
1305        }
1306
1307        #[test]
1308        fn the_event_stream_can_move_across_threads() {
1309            fn assert_send<T: Send>(_: &T) {}
1310            let broker = LlmBroker::new("m", Arc::new(MockGateway::new(vec![])), None);
1311            let messages = vec![LlmMessage::user("Hi")];
1312
1313            assert_send(&broker.generate_stream_events(&messages, None, None));
1314        }
1315    }
1316
1317    // Mock gateway for testing
1318    struct MockGateway {
1319        responses: Vec<LlmGatewayResponse>,
1320        call_count: std::sync::Mutex<usize>,
1321    }
1322
1323    impl MockGateway {
1324        fn new(responses: Vec<LlmGatewayResponse>) -> Self {
1325            Self {
1326                responses,
1327                call_count: std::sync::Mutex::new(0),
1328            }
1329        }
1330    }
1331
1332    #[async_trait::async_trait]
1333    impl LlmGateway for MockGateway {
1334        async fn complete(
1335            &self,
1336            _model: &str,
1337            _messages: &[LlmMessage],
1338            _tools: Option<&[Box<dyn LlmTool>]>,
1339            _config: &CompletionConfig,
1340        ) -> Result<LlmGatewayResponse> {
1341            let mut count = self.call_count.lock().unwrap();
1342            let idx = *count;
1343            *count += 1;
1344
1345            if idx < self.responses.len() {
1346                Ok(self.responses[idx].clone())
1347            } else {
1348                Ok(LlmGatewayResponse {
1349                    content: Some("default response".to_string()),
1350                    object: None,
1351                    tool_calls: vec![],
1352                    thinking: None,
1353                    ..Default::default()
1354                })
1355            }
1356        }
1357
1358        async fn complete_json(
1359            &self,
1360            _model: &str,
1361            _messages: &[LlmMessage],
1362            _schema: Value,
1363            _config: &CompletionConfig,
1364        ) -> Result<Value> {
1365            Ok(serde_json::json!({"test": "value"}))
1366        }
1367
1368        async fn get_available_models(&self) -> Result<Vec<String>> {
1369            Ok(vec!["test-model".to_string()])
1370        }
1371
1372        async fn calculate_embeddings(
1373            &self,
1374            _text: &str,
1375            _model: Option<&str>,
1376        ) -> Result<Vec<f32>> {
1377            Ok(vec![0.1, 0.2, 0.3])
1378        }
1379
1380        fn complete_stream<'a>(
1381            &'a self,
1382            _model: &'a str,
1383            _messages: &'a [LlmMessage],
1384            _tools: Option<&'a [Box<dyn LlmTool>]>,
1385            _config: &'a CompletionConfig,
1386        ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1387            use futures::stream;
1388            Box::pin(stream::iter(vec![Ok(StreamChunk::Content("test".to_string()))]))
1389        }
1390    }
1391
1392    // Mock tool for testing
1393    struct MockTool {
1394        name: String,
1395        result: Value,
1396    }
1397
1398    #[async_trait]
1399
1400    impl LlmTool for MockTool {
1401        async fn run(
1402            &self,
1403            _args: &HashMap<String, Value>,
1404            _ctx: &crate::llm::tools::ToolRunCtx,
1405        ) -> Result<Value> {
1406            Ok(self.result.clone())
1407        }
1408
1409        fn descriptor(&self) -> ToolDescriptor {
1410            ToolDescriptor {
1411                r#type: "function".to_string(),
1412                function: FunctionDescriptor {
1413                    name: self.name.clone(),
1414                    description: "A mock tool".to_string(),
1415                    parameters: serde_json::json!({}),
1416                },
1417            }
1418        }
1419
1420        fn clone_box(&self) -> Box<dyn LlmTool> {
1421            Box::new(MockTool {
1422                name: self.name.clone(),
1423                result: self.result.clone(),
1424            })
1425        }
1426    }
1427
1428    #[tokio::test]
1429    async fn test_broker_new() {
1430        let gateway = Arc::new(MockGateway::new(vec![]));
1431        let broker = LlmBroker::new("test-model", gateway, None);
1432        assert_eq!(broker.model, "test-model");
1433        assert_eq!(CompletionConfig::default().max_tool_iterations, 10);
1434    }
1435
1436    #[tokio::test]
1437    async fn test_broker_with_max_tool_iterations() {
1438        assert_eq!(
1439            CompletionConfig {
1440                max_tool_iterations: 3,
1441                ..Default::default()
1442            }
1443            .max_tool_iterations,
1444            3
1445        );
1446    }
1447
1448    #[tokio::test]
1449    async fn test_generate_max_tool_iterations_exceeded() {
1450        // Every response requests a tool call — should hit the limit
1451        let tool_call = LlmToolCall {
1452            id: Some("call_loop".to_string()),
1453            name: "loop_tool".to_string(),
1454            arguments: HashMap::new(),
1455        };
1456
1457        // Supply more responses than the limit so the gateway never runs dry
1458        let responses: Vec<LlmGatewayResponse> = (0..15)
1459            .map(|_| LlmGatewayResponse {
1460                content: None,
1461                object: None,
1462                tool_calls: vec![tool_call.clone()],
1463                thinking: None,
1464                ..Default::default()
1465            })
1466            .collect();
1467
1468        let gateway = Arc::new(MockGateway::new(responses));
1469        let broker = LlmBroker::new("test-model", gateway, None);
1470
1471        let tool = MockTool {
1472            name: "loop_tool".to_string(),
1473            result: serde_json::json!({"result": "loop"}),
1474        };
1475        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1476
1477        let messages = vec![LlmMessage::user("Loop forever")];
1478        let result = broker
1479            .generate(
1480                &messages,
1481                Some(&tools),
1482                Some(CompletionConfig {
1483                    max_tool_iterations: 3,
1484                    ..Default::default()
1485                }),
1486                None,
1487            )
1488            .await;
1489
1490        assert!(result.is_err());
1491        match result.unwrap_err() {
1492            crate::error::MojenticError::MaxToolIterationsExceeded { limit } => {
1493                assert_eq!(limit, 3);
1494            }
1495            e => panic!("Expected MaxToolIterationsExceeded, got: {:?}", e),
1496        }
1497    }
1498
1499    #[tokio::test]
1500    async fn test_generate_stream_max_tool_iterations_exceeded() {
1501        use futures::stream;
1502
1503        struct InfiniteToolGateway;
1504
1505        #[async_trait::async_trait]
1506        impl LlmGateway for InfiniteToolGateway {
1507            async fn complete(
1508                &self,
1509                _model: &str,
1510                _messages: &[LlmMessage],
1511                _tools: Option<&[Box<dyn LlmTool>]>,
1512                _config: &CompletionConfig,
1513            ) -> Result<LlmGatewayResponse> {
1514                Ok(LlmGatewayResponse {
1515                    content: None,
1516                    object: None,
1517                    tool_calls: vec![],
1518                    thinking: None,
1519                    ..Default::default()
1520                })
1521            }
1522
1523            async fn complete_json(
1524                &self,
1525                _model: &str,
1526                _messages: &[LlmMessage],
1527                _schema: serde_json::Value,
1528                _config: &CompletionConfig,
1529            ) -> Result<serde_json::Value> {
1530                Ok(serde_json::json!({}))
1531            }
1532
1533            async fn get_available_models(&self) -> Result<Vec<String>> {
1534                Ok(vec![])
1535            }
1536
1537            async fn calculate_embeddings(
1538                &self,
1539                _text: &str,
1540                _model: Option<&str>,
1541            ) -> Result<Vec<f32>> {
1542                Ok(vec![])
1543            }
1544
1545            fn complete_stream<'a>(
1546                &'a self,
1547                _model: &'a str,
1548                _messages: &'a [LlmMessage],
1549                _tools: Option<&'a [Box<dyn LlmTool>]>,
1550                _config: &'a CompletionConfig,
1551            ) -> std::pin::Pin<
1552                Box<dyn futures::stream::Stream<Item = Result<StreamChunk>> + Send + 'a>,
1553            > {
1554                Box::pin(stream::iter(vec![Ok(StreamChunk::ToolCalls(vec![LlmToolCall {
1555                    id: Some("call_loop".to_string()),
1556                    name: "loop_tool".to_string(),
1557                    arguments: HashMap::new(),
1558                }]))]))
1559            }
1560        }
1561
1562        let gateway = Arc::new(InfiniteToolGateway);
1563        let broker = LlmBroker::new("test-model", gateway, None);
1564
1565        let tool = MockTool {
1566            name: "loop_tool".to_string(),
1567            result: serde_json::json!({"result": "loop"}),
1568        };
1569        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1570
1571        let messages = vec![LlmMessage::user("Loop forever")];
1572        let mut stream = broker.generate_stream(
1573            &messages,
1574            Some(&tools),
1575            Some(CompletionConfig {
1576                max_tool_iterations: 2,
1577                ..Default::default()
1578            }),
1579            None,
1580        );
1581
1582        let mut found_error = false;
1583        while let Some(result) = stream.next().await {
1584            if let Err(crate::error::MojenticError::MaxToolIterationsExceeded { .. }) = result {
1585                found_error = true;
1586                break;
1587            }
1588        }
1589        assert!(found_error, "Expected MaxToolIterationsExceeded error in stream");
1590    }
1591
1592    #[tokio::test]
1593    async fn test_broker_new_string_conversion() {
1594        let gateway = Arc::new(MockGateway::new(vec![]));
1595        let broker = LlmBroker::new(String::from("my-model"), gateway, None);
1596        assert_eq!(broker.model, "my-model");
1597    }
1598
1599    #[tokio::test]
1600    async fn test_generate_simple_response() {
1601        let response = LlmGatewayResponse {
1602            content: Some("Hello, World!".to_string()),
1603            object: None,
1604            tool_calls: vec![],
1605            thinking: None,
1606            ..Default::default()
1607        };
1608
1609        let gateway = Arc::new(MockGateway::new(vec![response]));
1610        let broker = LlmBroker::new("test-model", gateway, None);
1611
1612        let messages = vec![LlmMessage::user("Hi")];
1613        let result = broker.generate(&messages, None, None, None).await.unwrap();
1614
1615        assert_eq!(result, "Hello, World!");
1616    }
1617
1618    #[tokio::test]
1619    async fn test_generate_with_custom_config() {
1620        let response = LlmGatewayResponse {
1621            content: Some("Response".to_string()),
1622            object: None,
1623            tool_calls: vec![],
1624            thinking: None,
1625            ..Default::default()
1626        };
1627
1628        let gateway = Arc::new(MockGateway::new(vec![response]));
1629        let broker = LlmBroker::new("test-model", gateway, None);
1630
1631        let config = CompletionConfig {
1632            temperature: Some(0.5),
1633            num_ctx: Some(2048),
1634            max_tokens: Some(100),
1635            top_p: None,
1636            top_k: None,
1637            response_format: None,
1638            reasoning_effort: None,
1639            reasoning_profile: None,
1640            preserve_reasoning: false,
1641            max_tool_iterations: 10,
1642        };
1643
1644        let messages = vec![LlmMessage::user("Hi")];
1645        let result = broker.generate(&messages, None, Some(config), None).await.unwrap();
1646
1647        assert_eq!(result, "Response");
1648    }
1649
1650    #[tokio::test]
1651    async fn test_generate_empty_response_content() {
1652        let response = LlmGatewayResponse {
1653            content: None,
1654            object: None,
1655            tool_calls: vec![],
1656            thinking: None,
1657            ..Default::default()
1658        };
1659
1660        let gateway = Arc::new(MockGateway::new(vec![response]));
1661        let broker = LlmBroker::new("test-model", gateway, None);
1662
1663        let messages = vec![LlmMessage::user("Hi")];
1664        let result = broker.generate(&messages, None, None, None).await.unwrap();
1665
1666        assert_eq!(result, "");
1667    }
1668
1669    #[tokio::test]
1670    async fn test_generate_with_tool_call() {
1671        let tool_call = LlmToolCall {
1672            id: Some("call_1".to_string()),
1673            name: "test_tool".to_string(),
1674            arguments: HashMap::new(),
1675        };
1676
1677        let first_response = LlmGatewayResponse {
1678            content: None,
1679            object: None,
1680            tool_calls: vec![tool_call],
1681            thinking: None,
1682            ..Default::default()
1683        };
1684
1685        let second_response = LlmGatewayResponse {
1686            content: Some("After tool execution".to_string()),
1687            object: None,
1688            tool_calls: vec![],
1689            thinking: None,
1690            ..Default::default()
1691        };
1692
1693        let gateway = Arc::new(MockGateway::new(vec![first_response, second_response]));
1694        let broker = LlmBroker::new("test-model", gateway, None);
1695
1696        let tool = MockTool {
1697            name: "test_tool".to_string(),
1698            result: serde_json::json!({"result": "success"}),
1699        };
1700
1701        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1702
1703        let messages = vec![LlmMessage::user("Use the tool")];
1704        let result = broker.generate(&messages, Some(&tools), None, None).await.unwrap();
1705
1706        assert_eq!(result, "After tool execution");
1707    }
1708
1709    #[tokio::test]
1710    async fn test_generate_with_tool_call_no_tools_provided() {
1711        let tool_call = LlmToolCall {
1712            id: Some("call_1".to_string()),
1713            name: "test_tool".to_string(),
1714            arguments: HashMap::new(),
1715        };
1716
1717        let response = LlmGatewayResponse {
1718            content: Some("fallback".to_string()),
1719            object: None,
1720            tool_calls: vec![tool_call],
1721            thinking: None,
1722            ..Default::default()
1723        };
1724
1725        let gateway = Arc::new(MockGateway::new(vec![response]));
1726        let broker = LlmBroker::new("test-model", gateway, None);
1727
1728        let messages = vec![LlmMessage::user("Use the tool")];
1729        let result = broker.generate(&messages, None, None, None).await.unwrap();
1730
1731        assert_eq!(result, "fallback");
1732    }
1733
1734    #[tokio::test]
1735    async fn test_generate_object() {
1736        #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
1737        struct TestObject {
1738            test: String,
1739        }
1740
1741        let gateway = Arc::new(MockGateway::new(vec![]));
1742        let broker = LlmBroker::new("test-model", gateway, None);
1743
1744        let messages = vec![LlmMessage::user("Generate object")];
1745        let result: TestObject = broker.generate_object(&messages, None, None).await.unwrap();
1746
1747        assert_eq!(result.test, "value");
1748    }
1749
1750    #[tokio::test]
1751    async fn test_generate_object_with_config() {
1752        #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
1753        struct TestData {
1754            test: String,
1755        }
1756
1757        let gateway = Arc::new(MockGateway::new(vec![]));
1758        let broker = LlmBroker::new("test-model", gateway, None);
1759
1760        let config = CompletionConfig {
1761            temperature: Some(0.1),
1762            num_ctx: Some(1024),
1763            max_tokens: Some(50),
1764            top_p: None,
1765            top_k: None,
1766            response_format: None,
1767            reasoning_effort: None,
1768            reasoning_profile: None,
1769            preserve_reasoning: false,
1770            max_tool_iterations: 10,
1771        };
1772
1773        let messages = vec![LlmMessage::user("Generate")];
1774        let result: TestData = broker.generate_object(&messages, Some(config), None).await.unwrap();
1775
1776        assert_eq!(result.test, "value");
1777    }
1778
1779    #[tokio::test]
1780    async fn test_multiple_messages() {
1781        let response = LlmGatewayResponse {
1782            content: Some("Response to conversation".to_string()),
1783            object: None,
1784            tool_calls: vec![],
1785            thinking: None,
1786            ..Default::default()
1787        };
1788
1789        let gateway = Arc::new(MockGateway::new(vec![response]));
1790        let broker = LlmBroker::new("test-model", gateway, None);
1791
1792        let messages = vec![
1793            LlmMessage::system("You are helpful"),
1794            LlmMessage::user("First message"),
1795            LlmMessage::assistant("First response"),
1796            LlmMessage::user("Second message"),
1797        ];
1798
1799        let result = broker.generate(&messages, None, None, None).await.unwrap();
1800        assert_eq!(result, "Response to conversation");
1801    }
1802
1803    #[tokio::test]
1804    async fn test_generate_stream_basic() {
1805        use futures::stream;
1806
1807        // Mock gateway that returns a simple stream
1808        struct StreamingMockGateway;
1809
1810        #[async_trait::async_trait]
1811        impl LlmGateway for StreamingMockGateway {
1812            async fn complete(
1813                &self,
1814                _model: &str,
1815                _messages: &[LlmMessage],
1816                _tools: Option<&[Box<dyn LlmTool>]>,
1817                _config: &CompletionConfig,
1818            ) -> Result<LlmGatewayResponse> {
1819                Ok(LlmGatewayResponse {
1820                    content: Some("test".to_string()),
1821                    object: None,
1822                    tool_calls: vec![],
1823                    thinking: None,
1824                    ..Default::default()
1825                })
1826            }
1827
1828            async fn complete_json(
1829                &self,
1830                _model: &str,
1831                _messages: &[LlmMessage],
1832                _schema: Value,
1833                _config: &CompletionConfig,
1834            ) -> Result<Value> {
1835                Ok(serde_json::json!({}))
1836            }
1837
1838            async fn get_available_models(&self) -> Result<Vec<String>> {
1839                Ok(vec![])
1840            }
1841
1842            async fn calculate_embeddings(
1843                &self,
1844                _text: &str,
1845                _model: Option<&str>,
1846            ) -> Result<Vec<f32>> {
1847                Ok(vec![])
1848            }
1849
1850            fn complete_stream<'a>(
1851                &'a self,
1852                _model: &'a str,
1853                _messages: &'a [LlmMessage],
1854                _tools: Option<&'a [Box<dyn LlmTool>]>,
1855                _config: &'a CompletionConfig,
1856            ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1857                Box::pin(stream::iter(vec![
1858                    Ok(StreamChunk::Content("Hello".to_string())),
1859                    Ok(StreamChunk::Content(" ".to_string())),
1860                    Ok(StreamChunk::Content("World".to_string())),
1861                ]))
1862            }
1863        }
1864
1865        let gateway = Arc::new(StreamingMockGateway);
1866        let broker = LlmBroker::new("test-model", gateway, None);
1867        let messages = vec![LlmMessage::user("Hello")];
1868
1869        let mut stream = broker.generate_stream(&messages, None, None, None);
1870        let mut result = String::new();
1871
1872        while let Some(chunk) = stream.next().await {
1873            result.push_str(&chunk.unwrap());
1874        }
1875
1876        assert_eq!(result, "Hello World");
1877    }
1878
1879    #[tokio::test]
1880    async fn test_generate_stream_with_tool_calls() {
1881        use futures::stream;
1882
1883        // Mock gateway that returns tool calls
1884        struct ToolCallMockGateway {
1885            call_count: std::sync::Mutex<usize>,
1886        }
1887
1888        impl ToolCallMockGateway {
1889            fn new() -> Self {
1890                Self {
1891                    call_count: std::sync::Mutex::new(0),
1892                }
1893            }
1894        }
1895
1896        #[async_trait::async_trait]
1897        impl LlmGateway for ToolCallMockGateway {
1898            async fn complete(
1899                &self,
1900                _model: &str,
1901                _messages: &[LlmMessage],
1902                _tools: Option<&[Box<dyn LlmTool>]>,
1903                _config: &CompletionConfig,
1904            ) -> Result<LlmGatewayResponse> {
1905                Ok(LlmGatewayResponse {
1906                    content: Some("test".to_string()),
1907                    object: None,
1908                    tool_calls: vec![],
1909                    thinking: None,
1910                    ..Default::default()
1911                })
1912            }
1913
1914            async fn complete_json(
1915                &self,
1916                _model: &str,
1917                _messages: &[LlmMessage],
1918                _schema: Value,
1919                _config: &CompletionConfig,
1920            ) -> Result<Value> {
1921                Ok(serde_json::json!({}))
1922            }
1923
1924            async fn get_available_models(&self) -> Result<Vec<String>> {
1925                Ok(vec![])
1926            }
1927
1928            async fn calculate_embeddings(
1929                &self,
1930                _text: &str,
1931                _model: Option<&str>,
1932            ) -> Result<Vec<f32>> {
1933                Ok(vec![])
1934            }
1935
1936            fn complete_stream<'a>(
1937                &'a self,
1938                _model: &'a str,
1939                _messages: &'a [LlmMessage],
1940                _tools: Option<&'a [Box<dyn LlmTool>]>,
1941                _config: &'a CompletionConfig,
1942            ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1943                let mut count = self.call_count.lock().unwrap();
1944                let call_num = *count;
1945                *count += 1;
1946
1947                if call_num == 0 {
1948                    // First call: return content with tool call
1949                    Box::pin(stream::iter(vec![
1950                        Ok(StreamChunk::Content("Initial ".to_string())),
1951                        Ok(StreamChunk::Content("response".to_string())),
1952                        Ok(StreamChunk::ToolCalls(vec![LlmToolCall {
1953                            id: Some("call_1".to_string()),
1954                            name: "test_tool".to_string(),
1955                            arguments: HashMap::new(),
1956                        }])),
1957                    ]))
1958                } else {
1959                    // Second call (after tool execution): return final content
1960                    Box::pin(stream::iter(vec![
1961                        Ok(StreamChunk::Content("After ".to_string())),
1962                        Ok(StreamChunk::Content("tool".to_string())),
1963                    ]))
1964                }
1965            }
1966        }
1967
1968        let gateway = Arc::new(ToolCallMockGateway::new());
1969        let broker = LlmBroker::new("test-model", gateway, None);
1970
1971        let tool = MockTool {
1972            name: "test_tool".to_string(),
1973            result: serde_json::json!({"result": "success"}),
1974        };
1975        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1976
1977        let messages = vec![LlmMessage::user("Use the tool")];
1978        let mut stream = broker.generate_stream(&messages, Some(&tools), None, None);
1979
1980        let mut result = String::new();
1981        while let Some(chunk) = stream.next().await {
1982            result.push_str(&chunk.unwrap());
1983        }
1984
1985        // Should contain both initial response and post-tool response
1986        assert!(result.contains("Initial response"));
1987        assert!(result.contains("After tool"));
1988    }
1989
1990    #[tokio::test]
1991    async fn test_generate_stream_without_tools() {
1992        use futures::stream;
1993
1994        struct SimpleStreamGateway;
1995
1996        #[async_trait::async_trait]
1997        impl LlmGateway for SimpleStreamGateway {
1998            async fn complete(
1999                &self,
2000                _model: &str,
2001                _messages: &[LlmMessage],
2002                _tools: Option<&[Box<dyn LlmTool>]>,
2003                _config: &CompletionConfig,
2004            ) -> Result<LlmGatewayResponse> {
2005                Ok(LlmGatewayResponse {
2006                    content: Some("test".to_string()),
2007                    object: None,
2008                    tool_calls: vec![],
2009                    thinking: None,
2010                    ..Default::default()
2011                })
2012            }
2013
2014            async fn complete_json(
2015                &self,
2016                _model: &str,
2017                _messages: &[LlmMessage],
2018                _schema: Value,
2019                _config: &CompletionConfig,
2020            ) -> Result<Value> {
2021                Ok(serde_json::json!({}))
2022            }
2023
2024            async fn get_available_models(&self) -> Result<Vec<String>> {
2025                Ok(vec![])
2026            }
2027
2028            async fn calculate_embeddings(
2029                &self,
2030                _text: &str,
2031                _model: Option<&str>,
2032            ) -> Result<Vec<f32>> {
2033                Ok(vec![])
2034            }
2035
2036            fn complete_stream<'a>(
2037                &'a self,
2038                _model: &'a str,
2039                _messages: &'a [LlmMessage],
2040                _tools: Option<&'a [Box<dyn LlmTool>]>,
2041                _config: &'a CompletionConfig,
2042            ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
2043                // Simple stream with no tool calls
2044                Box::pin(stream::iter(vec![
2045                    Ok(StreamChunk::Content("Simple ".to_string())),
2046                    Ok(StreamChunk::Content("stream".to_string())),
2047                ]))
2048            }
2049        }
2050
2051        let gateway = Arc::new(SimpleStreamGateway);
2052        let broker = LlmBroker::new("test-model", gateway, None);
2053
2054        let messages = vec![LlmMessage::user("Test")];
2055        let mut stream = broker.generate_stream(&messages, None, None, None);
2056
2057        let mut result = String::new();
2058        while let Some(chunk) = stream.next().await {
2059            result.push_str(&chunk.unwrap());
2060        }
2061
2062        assert_eq!(result, "Simple stream");
2063    }
2064
2065    #[tokio::test]
2066    async fn test_tracer_integration() {
2067        use crate::tracer::TracerSystem;
2068
2069        let response = LlmGatewayResponse {
2070            content: Some("Test response".to_string()),
2071            object: None,
2072            tool_calls: vec![],
2073            thinking: None,
2074            ..Default::default()
2075        };
2076
2077        let gateway = Arc::new(MockGateway::new(vec![response]));
2078        let tracer = Arc::new(TracerSystem::default());
2079        let broker = LlmBroker::new("test-model", gateway, Some(tracer.clone()));
2080
2081        let messages = vec![LlmMessage::user("Test")];
2082        let correlation_id = "test-correlation-123";
2083
2084        let result = broker
2085            .generate(&messages, None, None, Some(correlation_id.to_string()))
2086            .await
2087            .unwrap();
2088
2089        assert_eq!(result, "Test response");
2090
2091        // Verify tracer recorded events
2092        assert_eq!(tracer.len(), 2); // One LLM call + one LLM response
2093
2094        // Verify correlation ID is preserved
2095        let summaries = tracer.get_event_summaries(None, None, None);
2096        assert!(summaries[0].contains(correlation_id));
2097        assert!(summaries[1].contains(correlation_id));
2098    }
2099
2100    #[tokio::test]
2101    async fn test_tracer_with_tool_calls() {
2102        use crate::tracer::TracerSystem;
2103
2104        let tool_call = LlmToolCall {
2105            id: Some("call_1".to_string()),
2106            name: "test_tool".to_string(),
2107            arguments: HashMap::new(),
2108        };
2109
2110        let first_response = LlmGatewayResponse {
2111            content: None,
2112            object: None,
2113            tool_calls: vec![tool_call],
2114            thinking: None,
2115            ..Default::default()
2116        };
2117
2118        let second_response = LlmGatewayResponse {
2119            content: Some("After tool".to_string()),
2120            object: None,
2121            tool_calls: vec![],
2122            thinking: None,
2123            ..Default::default()
2124        };
2125
2126        let gateway = Arc::new(MockGateway::new(vec![first_response, second_response]));
2127        let tracer = Arc::new(TracerSystem::default());
2128        let broker = LlmBroker::new("test-model", gateway, Some(tracer.clone()));
2129
2130        let tool = MockTool {
2131            name: "test_tool".to_string(),
2132            result: serde_json::json!({"result": "success"}),
2133        };
2134        let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
2135
2136        let messages = vec![LlmMessage::user("Use tool")];
2137        let correlation_id = "tool-test-456";
2138
2139        let result = broker
2140            .generate(&messages, Some(&tools), None, Some(correlation_id.to_string()))
2141            .await
2142            .unwrap();
2143
2144        assert_eq!(result, "After tool");
2145
2146        // Should have: 2 LLM calls, 2 LLM responses, 1 tool call
2147        assert_eq!(tracer.len(), 5);
2148
2149        // Verify all events share the same correlation ID
2150        let summaries = tracer.get_event_summaries(None, None, None);
2151        for summary in &summaries {
2152            assert!(summary.contains(correlation_id));
2153        }
2154    }
2155}