1use crate::error::{MojenticError, Result};
2use crate::llm::gateway::{CompletionConfig, LlmGateway, StreamChunk};
3use crate::llm::models::{LlmGatewayResponse, LlmMessage, MessageRole, ResponseEvidence};
4use crate::llm::stream_events::{StreamEvent, StreamEventError, StreamEventStream};
5use crate::llm::tools::{LlmTool, SerialToolRunner, ToolCallExecution, ToolRunCtx, ToolRunner};
6use crate::tracer::TracerSystem;
7use futures::stream::{Stream, StreamExt};
8use serde::{Deserialize, Serialize};
9use std::pin::Pin;
10use std::sync::Arc;
11use tracing::{info, warn};
12use uuid::Uuid;
13
14#[derive(Clone)]
16pub struct LlmBroker {
17 model: String,
18 gateway: Arc<dyn LlmGateway>,
19 tracer: Option<Arc<TracerSystem>>,
20 tool_runner: Arc<dyn ToolRunner>,
21}
22
23impl LlmBroker {
24 pub fn new(
35 model: impl Into<String>,
36 gateway: Arc<dyn LlmGateway>,
37 tracer: Option<Arc<TracerSystem>>,
38 ) -> Self {
39 Self {
40 model: model.into(),
41 gateway,
42 tracer,
43 tool_runner: Arc::new(SerialToolRunner),
44 }
45 }
46
47 pub fn with_tool_runner(
52 model: impl Into<String>,
53 gateway: Arc<dyn LlmGateway>,
54 tracer: Option<Arc<TracerSystem>>,
55 tool_runner: Arc<dyn ToolRunner>,
56 ) -> Self {
57 Self {
58 model: model.into(),
59 gateway,
60 tracer,
61 tool_runner,
62 }
63 }
64
65 pub async fn generate_response(
71 &self,
72 messages: &[LlmMessage],
73 tools: Option<&[Box<dyn LlmTool>]>,
74 config: Option<CompletionConfig>,
75 correlation_id: Option<String>,
76 ) -> Result<LlmGatewayResponse> {
77 let config = config.unwrap_or_default();
78 let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
79 if let Some(tracer) = &self.tracer {
81 let messages_json = messages_for_trace(messages);
82
83 let tools_json = tools.map(|t| {
84 t.iter()
85 .map(|tool| {
86 let desc = tool.descriptor();
87 let mut map = std::collections::HashMap::new();
88 map.insert("name".to_string(), serde_json::json!(desc.function.name));
89 map.insert(
90 "description".to_string(),
91 serde_json::json!(desc.function.description),
92 );
93 map
94 })
95 .collect()
96 });
97
98 tracer.record_llm_call(
99 &self.model,
100 messages_json,
101 config.temperature.map(f64::from),
102 tools_json,
103 "LlmBroker",
104 &correlation_id,
105 );
106 }
107
108 let start = std::time::Instant::now();
110
111 let response = self.gateway.complete(&self.model, messages, tools, &config).await?;
113
114 let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
115
116 if let Some(tracer) = &self.tracer {
118 let tool_calls_json = if !response.tool_calls.is_empty() {
119 Some(
120 response
121 .tool_calls
122 .iter()
123 .map(|tc| {
124 let mut map = std::collections::HashMap::new();
125 map.insert("name".to_string(), serde_json::json!(&tc.name));
126 if let Some(id) = &tc.id {
127 map.insert("id".to_string(), serde_json::json!(id));
128 }
129 map
130 })
131 .collect(),
132 )
133 } else {
134 None
135 };
136
137 tracer.record_llm_response_with_evidence(
138 &self.model,
139 response.content.as_ref().unwrap_or(&String::new()),
140 tool_calls_json,
141 Some(call_duration_ms),
142 response.evidence.clone(),
143 "LlmBroker",
144 &correlation_id,
145 );
146 }
147
148 Ok(response)
149 }
150
151 pub async fn generate(
160 &self,
161 messages: &[LlmMessage],
162 tools: Option<&[Box<dyn LlmTool>]>,
163 config: Option<CompletionConfig>,
164 correlation_id: Option<String>,
165 ) -> Result<String> {
166 let config = config.unwrap_or_default();
167 let current_messages = messages.to_vec();
168 let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
169
170 let response = self
171 .generate_response(messages, tools, Some(config.clone()), Some(correlation_id.clone()))
172 .await?;
173
174 if !response.tool_calls.is_empty() {
176 if let Some(tools) = tools {
177 return self
178 .handle_tool_calls(
179 current_messages,
180 response,
181 tools,
182 &config,
183 &correlation_id,
184 0,
185 )
186 .await;
187 }
188 }
189
190 Ok(response.content.unwrap_or_default())
191 }
192
193 fn handle_tool_calls<'a>(
194 &'a self,
195 mut messages: Vec<LlmMessage>,
196 response: LlmGatewayResponse,
197 tools: &'a [Box<dyn LlmTool>],
198 config: &'a CompletionConfig,
199 correlation_id: &'a str,
200 iteration: usize,
201 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String>> + Send + 'a>> {
202 Box::pin(async move {
203 if config.tool_iteration_limit_reached(iteration) {
204 return Err(MojenticError::MaxToolIterationsExceeded {
205 limit: config.max_tool_iterations,
206 });
207 }
208
209 info!("Tool calls requested: {}", response.tool_calls.len());
210
211 messages.push(LlmMessage {
214 role: MessageRole::Assistant,
215 content: response.content.clone(),
216 tool_calls: Some(response.tool_calls.clone()),
217 image_paths: None,
218 reasoning_content: config
219 .preserve_reasoning
220 .then(|| response.thinking.clone())
221 .flatten(),
222 });
223
224 let outcomes = self
225 .run_tool_batch(&response.tool_calls, tools, correlation_id, "LlmBroker")
226 .await?;
227
228 for (call, outcome) in response.tool_calls.iter().zip(outcomes.iter()) {
229 let content = if outcome.ok {
230 serde_json::to_string(
231 outcome.result.as_ref().unwrap_or(&serde_json::Value::Null),
232 )?
233 } else {
234 serde_json::json!({
235 "error": outcome.error.clone().unwrap_or_default(),
236 })
237 .to_string()
238 };
239 messages.push(LlmMessage {
240 role: MessageRole::Tool,
241 content: Some(content),
242 tool_calls: Some(vec![call.clone()]),
243 image_paths: None,
244 reasoning_content: None,
245 });
246 }
247
248 let next_response = self
249 .generate_response(
250 &messages,
251 Some(tools),
252 Some(config.clone()),
253 Some(correlation_id.to_owned()),
254 )
255 .await?;
256
257 if !next_response.tool_calls.is_empty() {
258 return self
259 .handle_tool_calls(
260 messages,
261 next_response,
262 tools,
263 config,
264 correlation_id,
265 iteration.saturating_add(1),
266 )
267 .await;
268 }
269
270 Ok(next_response.content.unwrap_or_default())
271 })
272 }
273
274 async fn run_tool_batch(
277 &self,
278 tool_calls: &[crate::llm::models::LlmToolCall],
279 tools: &[Box<dyn LlmTool>],
280 correlation_id: &str,
281 source: &'static str,
282 ) -> Result<Vec<crate::llm::tools::ToolCallOutcome>> {
283 let executions: Vec<ToolCallExecution> = tool_calls
284 .iter()
285 .enumerate()
286 .map(|(idx, tc)| ToolCallExecution {
287 id: tc.id.clone().unwrap_or_else(|| format!("call_{idx}")),
288 name: tc.name.clone(),
289 args: tc.arguments.clone(),
290 })
291 .collect();
292
293 let ctx = ToolRunCtx {
294 correlation_id: Some(correlation_id.to_string()),
295 source: Some(source.to_string()),
296 ..Default::default()
297 };
298
299 let batch_start = std::time::Instant::now();
300 let outcomes = self.tool_runner.run_batch(&executions, tools, &ctx).await;
301 let batch_duration_ms = batch_start.elapsed().as_secs_f64() * 1000.0;
302
303 if let Some(tracer) = &self.tracer {
304 for (exec, outcome) in executions.iter().zip(outcomes.iter()) {
305 tracer.record_tool_call(
306 &outcome.name,
307 exec.args.clone(),
308 outcome.result.clone().unwrap_or(serde_json::Value::Null),
309 Some(source.to_string()),
310 Some(outcome.duration_ms as f64),
311 source,
312 correlation_id,
313 );
314 }
315 if outcomes.len() > 1 {
316 let ok = outcomes.iter().filter(|o| o.ok).count();
317 let fail = outcomes.len() - ok;
318 tracer.record_tool_batch(
319 uuid::Uuid::new_v4().to_string(),
320 outcomes.iter().map(|o| o.name.clone()).collect(),
321 ok,
322 fail,
323 batch_duration_ms,
324 Some(source.to_string()),
325 source,
326 correlation_id,
327 );
328 }
329 }
330
331 if outcomes.len() != tool_calls.len() {
335 return Err(MojenticError::ToolError(format!(
336 "tool runner returned {} outcomes for {} calls",
337 outcomes.len(),
338 tool_calls.len()
339 )));
340 }
341
342 Ok(outcomes)
343 }
344
345 pub async fn generate_object<T>(
353 &self,
354 messages: &[LlmMessage],
355 config: Option<CompletionConfig>,
356 correlation_id: Option<String>,
357 ) -> Result<T>
358 where
359 T: for<'de> Deserialize<'de> + Serialize + schemars::JsonSchema + Send,
360 {
361 let config = config.unwrap_or_default();
362 let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
363
364 let schema = serde_json::to_value(schemars::schema_for!(T))?;
366
367 if let Some(tracer) = &self.tracer {
369 let messages_json = messages_for_trace(messages);
370
371 tracer.record_llm_call(
372 &self.model,
373 messages_json,
374 config.temperature.map(f64::from),
375 None,
376 "LlmBroker::generate_object",
377 &correlation_id,
378 );
379 }
380
381 let start = std::time::Instant::now();
383
384 let response = self
386 .gateway
387 .complete_json_response(&self.model, messages, schema, &config)
388 .await?;
389
390 let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
391
392 let json_response = response.object.ok_or_else(|| {
393 MojenticError::GatewayError("Structured response carried no object".to_string())
394 })?;
395
396 if let Some(tracer) = &self.tracer {
398 let object_str = serde_json::to_string_pretty(&json_response).unwrap_or_default();
399 tracer.record_llm_response_with_evidence(
400 &self.model,
401 format!("Structured response: {}", object_str),
402 None,
403 Some(call_duration_ms),
404 response.evidence,
405 "LlmBroker::generate_object",
406 &correlation_id,
407 );
408 }
409
410 let object: T = serde_json::from_value(json_response)?;
412
413 Ok(object)
414 }
415
416 pub fn generate_stream<'a>(
446 &'a self,
447 messages: &'a [LlmMessage],
448 tools: Option<&'a [Box<dyn LlmTool>]>,
449 config: Option<CompletionConfig>,
450 correlation_id: Option<String>,
451 ) -> Pin<Box<dyn Stream<Item = Result<String>> + 'a>> {
452 let config = config.unwrap_or_default();
453 let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
454 Box::pin(self.generate_stream_with_depth(
455 messages.to_vec(),
456 tools,
457 config,
458 correlation_id,
459 0,
460 ))
461 }
462
463 fn generate_stream_with_depth<'a>(
464 &'a self,
465 current_messages: Vec<LlmMessage>,
466 tools: Option<&'a [Box<dyn LlmTool>]>,
467 config: CompletionConfig,
468 correlation_id: String,
469 depth: usize,
470 ) -> impl Stream<Item = Result<String>> + 'a {
471 async_stream::stream! {
472 if config.tool_iteration_limit_reached(depth) {
473 yield Err(MojenticError::MaxToolIterationsExceeded {
474 limit: config.max_tool_iterations,
475 });
476 return;
477 }
478
479 if let Some(tracer) = &self.tracer {
481 let messages_json = messages_for_trace(¤t_messages);
482
483 let tools_json = tools.map(|t| {
484 t.iter()
485 .map(|tool| {
486 let desc = tool.descriptor();
487 let mut map = std::collections::HashMap::new();
488 map.insert("name".to_string(), serde_json::json!(desc.function.name));
489 map.insert(
490 "description".to_string(),
491 serde_json::json!(desc.function.description),
492 );
493 map
494 })
495 .collect()
496 });
497
498 tracer.record_llm_call(
499 &self.model,
500 messages_json,
501 config.temperature.map(f64::from),
502 tools_json,
503 "LlmBroker::generate_stream",
504 &correlation_id,
505 );
506 }
507
508 let mut stream_evidence = ResponseEvidence::default();
509 let mut accumulated_content = String::new();
510 let mut accumulated_reasoning = String::new();
511 let mut accumulated_tool_calls = Vec::new();
512
513 let start = std::time::Instant::now();
515
516 let mut stream = self.gateway.complete_stream(
518 &self.model,
519 ¤t_messages,
520 tools,
521 &config,
522 );
523
524 let mut stream_error = None;
525 while let Some(chunk_result) = stream.next().await {
526 match chunk_result {
527 Ok(StreamChunk::Content(content)) => {
528 accumulated_content.push_str(&content);
529 yield Ok(content);
530 }
531 Ok(StreamChunk::ToolCalls(tool_calls)) => {
532 accumulated_tool_calls = tool_calls;
533 }
534 Ok(StreamChunk::Thinking(thinking)) => {
535 if config.preserve_reasoning { accumulated_reasoning.push_str(&thinking); }
536 }
537 Ok(StreamChunk::Progress(_)) => {},
538 Ok(StreamChunk::Metrics(metrics)) => {
539 crate::llm::stream_timing::retain_timing(&mut stream_evidence, &metrics);
540 stream_evidence.metadata.insert("stream_metrics".into(), serde_json::json!(metrics));
541 stream_evidence.usage = metrics.provider_usage;
542 }
543 Err(e) => {
544 stream_error = Some(e);
545 break;
546 }
547 }
548 }
549
550 let call_duration_ms = start.elapsed().as_secs_f64() * 1000.0;
551
552 if let Some(tracer) = &self.tracer {
554 let tool_calls_json = if !accumulated_tool_calls.is_empty() {
555 Some(
556 accumulated_tool_calls
557 .iter()
558 .map(|tc| {
559 let mut map = std::collections::HashMap::new();
560 map.insert("name".to_string(), serde_json::json!(&tc.name));
561 if let Some(id) = &tc.id {
562 map.insert("id".to_string(), serde_json::json!(id));
563 }
564 map
565 })
566 .collect(),
567 )
568 } else {
569 None
570 };
571
572 tracer.record_llm_response_with_evidence(
573 &self.model,
574 &accumulated_content,
575 tool_calls_json,
576 Some(call_duration_ms),
577 stream_evidence,
578 "LlmBroker::generate_stream",
579 &correlation_id,
580 );
581 }
582
583 if let Some(error) = stream_error {
586 yield Err(error);
587 return;
588 }
589
590 if !accumulated_tool_calls.is_empty() {
592 if let Some(tools) = tools {
593 info!("Processing {} tool call(s) in stream", accumulated_tool_calls.len());
594
595 let mut new_messages = current_messages.clone();
597
598 new_messages.push(LlmMessage {
600 role: MessageRole::Assistant,
601 content: Some(accumulated_content),
602 tool_calls: Some(accumulated_tool_calls.clone()),
603 image_paths: None,
604 reasoning_content: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning),
605 });
606
607 let outcomes = match self
608 .run_tool_batch(
609 &accumulated_tool_calls,
610 tools,
611 &correlation_id,
612 "LlmBroker::generate_stream",
613 )
614 .await
615 {
616 Ok(o) => o,
617 Err(e) => {
618 yield Err(e);
619 return;
620 }
621 };
622
623 for (call, outcome) in accumulated_tool_calls.iter().zip(outcomes.iter()) {
624 let content = if outcome.ok {
625 match serde_json::to_string(
626 outcome.result.as_ref().unwrap_or(&serde_json::Value::Null),
627 ) {
628 Ok(s) => s,
629 Err(e) => {
630 yield Err(e.into());
631 return;
632 }
633 }
634 } else {
635 serde_json::json!({
636 "error": outcome.error.clone().unwrap_or_default(),
637 })
638 .to_string()
639 };
640 new_messages.push(LlmMessage {
641 role: MessageRole::Tool,
642 content: Some(content),
643 tool_calls: Some(vec![call.clone()]),
644 image_paths: None,
645 reasoning_content: None,
646 });
647 }
648
649 let mut recursive_stream = Box::pin(self.generate_stream_with_depth(
651 new_messages,
652 Some(tools),
653 config.clone(),
654 correlation_id.clone(),
655 depth.saturating_add(1),
656 ));
657
658 while let Some(result) = recursive_stream.next().await {
659 yield result;
660 }
661 } else {
662 warn!("LLM requested tool calls but no tools provided");
663 }
664 }
665 }
666 }
667
668 pub fn generate_stream_events<'a>(
711 &'a self,
712 messages: &'a [LlmMessage],
713 config: Option<CompletionConfig>,
714 correlation_id: Option<String>,
715 ) -> StreamEventStream<'a> {
716 const SOURCE: &str = "LlmBroker::generate_stream_events";
717 let config = CompletionConfig {
718 max_tool_iterations: 0,
719 ..config.unwrap_or_default()
720 };
721 let correlation_id = correlation_id.unwrap_or_else(|| Uuid::new_v4().to_string());
722
723 Box::pin(async_stream::stream! {
724 let mut events =
725 match self.gateway.complete_stream_events(&self.model, messages, &config) {
726 Ok(events) => events,
727 Err(error) => {
728 yield StreamEvent::Error(error);
729 return;
730 }
731 };
732
733 if let Some(tracer) = &self.tracer {
734 tracer.record_llm_call(
735 &self.model,
736 messages_for_trace(messages),
737 config.temperature.map(f64::from),
738 None,
739 SOURCE,
740 &correlation_id,
741 );
742 }
743 let start = std::time::Instant::now();
744
745 let mut content = String::new();
746 let terminal = loop {
747 match events.next().await {
748 Some(StreamEvent::Content(text)) => {
749 content.push_str(&text);
750 yield StreamEvent::Content(text);
751 }
752 Some(StreamEvent::Metrics(metrics)) => yield StreamEvent::Metrics(metrics),
753 Some(terminal) => break terminal,
754 None => break StreamEvent::Error(StreamEventError::IncompleteStream(None)),
755 }
756 };
757
758 if let Some(tracer) = &self.tracer {
759 tracer.record_llm_response_with_evidence(
760 &self.model,
761 content,
762 None,
763 Some(start.elapsed().as_secs_f64() * 1000.0),
764 terminal_evidence(&terminal),
765 SOURCE,
766 &correlation_id,
767 );
768 }
769 yield terminal;
770 })
771 }
772}
773
774fn messages_for_trace(
776 messages: &[LlmMessage],
777) -> Vec<std::collections::HashMap<String, serde_json::Value>> {
778 messages
779 .iter()
780 .map(|m| {
781 let mut map = std::collections::HashMap::new();
782 map.insert("role".to_string(), serde_json::json!(format!("{:?}", m.role)));
783 if let Some(content) = &m.content {
784 map.insert("content".to_string(), serde_json::json!(content));
785 }
786 map
787 })
788 .collect()
789}
790
791fn terminal_evidence(terminal: &StreamEvent) -> ResponseEvidence {
793 match terminal {
794 StreamEvent::Completed(evidence) => evidence.clone(),
795 StreamEvent::Error(StreamEventError::IncompleteCompletion(evidence))
796 | StreamEvent::Error(StreamEventError::IncompleteStream(Some(evidence))) => {
797 evidence.as_ref().clone()
798 }
799 _ => ResponseEvidence::default(),
800 }
801}
802
803#[cfg(test)]
804mod tests {
805 use super::*;
806 use crate::llm::models::LlmToolCall;
807 use crate::llm::tools::{FunctionDescriptor, ToolDescriptor};
808 use async_trait::async_trait;
809 use serde::{Deserialize, Serialize};
810 use serde_json::Value;
811 use std::collections::HashMap;
812
813 #[tokio::test]
814 async fn native_response_leaves_tool_dispatch_to_caller() {
815 let response = LlmGatewayResponse {
816 content: None,
817 object: None,
818 thinking: None,
819 tool_calls: vec![LlmToolCall {
820 id: Some("call-1".into()),
821 name: "unknown".into(),
822 arguments: HashMap::new(),
823 }],
824 ..Default::default()
825 };
826 let broker =
827 LlmBroker::new("test".to_string(), Arc::new(MockGateway::new(vec![response])), None);
828 let messages = vec![LlmMessage::user("hello")];
829 let actual = broker
830 .generate_response(&messages, Some(&[]), None, None)
831 .await
832 .expect("native response");
833 assert_eq!(actual.tool_calls[0].id.as_deref(), Some("call-1"));
834 assert_eq!(messages.len(), 1);
835 }
836
837 #[tokio::test]
838 async fn unlimited_execution_can_continue_past_default_rounds() {
839 let response = LlmGatewayResponse {
840 content: None,
841 object: None,
842 thinking: None,
843 tool_calls: vec![LlmToolCall {
844 id: Some("call".into()),
845 name: "unknown".into(),
846 arguments: HashMap::new(),
847 }],
848 ..Default::default()
849 };
850 let mut responses = vec![response; 12];
851 responses.push(LlmGatewayResponse {
852 content: Some("done".into()),
853 object: None,
854 thinking: None,
855 tool_calls: vec![],
856 ..Default::default()
857 });
858 let broker =
859 LlmBroker::new("test".to_string(), Arc::new(MockGateway::new(responses)), None);
860 let actual = broker
861 .generate(
862 &[LlmMessage::user("hello")],
863 Some(&[]),
864 Some(CompletionConfig::default().with_unlimited_tool_iterations()),
865 None,
866 )
867 .await
868 .expect("unlimited completion");
869 assert_eq!(actual, "done");
870 }
871
872 fn reported_evidence() -> crate::llm::models::ResponseEvidence {
873 crate::llm::models::ResponseEvidence {
874 usage: Some(serde_json::json!({"prompt_tokens": 9, "completion_tokens": 4})),
875 provider_model: Some("provider-model-2026".to_string()),
876 finish_reason: Some("stop".to_string()),
877 metadata: HashMap::from([("id".to_string(), serde_json::json!("resp-1"))]),
878 }
879 }
880
881 struct EvidenceGateway {
883 evidence: crate::llm::models::ResponseEvidence,
884 }
885
886 #[async_trait::async_trait]
887 impl LlmGateway for EvidenceGateway {
888 async fn complete(
889 &self,
890 _model: &str,
891 _messages: &[LlmMessage],
892 _tools: Option<&[Box<dyn LlmTool>]>,
893 _config: &CompletionConfig,
894 ) -> Result<LlmGatewayResponse> {
895 Ok(LlmGatewayResponse {
896 content: Some("answer".to_string()),
897 evidence: self.evidence.clone(),
898 ..Default::default()
899 })
900 }
901
902 async fn complete_json(
903 &self,
904 _model: &str,
905 _messages: &[LlmMessage],
906 _schema: Value,
907 _config: &CompletionConfig,
908 ) -> Result<Value> {
909 unreachable!("the broker asks for the structured response with evidence")
910 }
911
912 async fn complete_json_response(
913 &self,
914 _model: &str,
915 _messages: &[LlmMessage],
916 _schema: Value,
917 _config: &CompletionConfig,
918 ) -> Result<LlmGatewayResponse<Value>> {
919 Ok(LlmGatewayResponse {
920 object: Some(serde_json::json!({"test": "value"})),
921 evidence: self.evidence.clone(),
922 ..Default::default()
923 })
924 }
925
926 async fn get_available_models(&self) -> Result<Vec<String>> {
927 Ok(vec![])
928 }
929
930 async fn calculate_embeddings(
931 &self,
932 _text: &str,
933 _model: Option<&str>,
934 ) -> Result<Vec<f32>> {
935 Ok(vec![])
936 }
937
938 fn complete_stream<'a>(
939 &'a self,
940 _model: &'a str,
941 _messages: &'a [LlmMessage],
942 _tools: Option<&'a [Box<dyn LlmTool>]>,
943 _config: &'a CompletionConfig,
944 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
945 Box::pin(futures::stream::empty())
946 }
947 }
948
949 #[tokio::test]
950 async fn generate_response_traces_reported_evidence_unchanged() {
951 use crate::tracer::testing::{capturing_tracer, responses};
952
953 let (tracer, captured) = capturing_tracer();
954 let gateway = Arc::new(EvidenceGateway {
955 evidence: reported_evidence(),
956 });
957 let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
958
959 let response = broker
960 .generate_response(&[LlmMessage::user("q")], None, None, None)
961 .await
962 .unwrap();
963
964 assert_eq!(response.evidence, reported_evidence());
965 assert!(matches!(
966 captured.lock().unwrap().first(),
967 Some(crate::tracer::testing::CapturedLlmEvent::Call(call))
968 if call.model == "configured-model"
969 ));
970 let events = responses(&captured);
971 assert_eq!(events.len(), 1);
972 assert_eq!(events[0].model, "configured-model");
973 assert_eq!(events[0].evidence, reported_evidence());
974 }
975
976 #[tokio::test]
977 async fn generate_response_traces_unknown_usage_as_none() {
978 use crate::tracer::testing::{capturing_tracer, responses};
979
980 let (tracer, captured) = capturing_tracer();
981 let gateway = Arc::new(EvidenceGateway {
982 evidence: Default::default(),
983 });
984 let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
985
986 broker.generate(&[LlmMessage::user("q")], None, None, None).await.unwrap();
987
988 let events = responses(&captured);
989 assert_eq!(events.len(), 1);
990 assert_eq!(events[0].evidence.usage, None);
991 assert_eq!(events[0].evidence.provider_model, None);
992 }
993
994 #[tokio::test]
995 async fn generate_object_traces_reported_evidence_unchanged() {
996 use crate::tracer::testing::{capturing_tracer, responses};
997
998 #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
999 struct TestObject {
1000 test: String,
1001 }
1002
1003 let (tracer, captured) = capturing_tracer();
1004 let gateway = Arc::new(EvidenceGateway {
1005 evidence: reported_evidence(),
1006 });
1007 let broker = LlmBroker::new("configured-model", gateway, Some(tracer));
1008
1009 let object: TestObject =
1010 broker.generate_object(&[LlmMessage::user("q")], None, None).await.unwrap();
1011
1012 assert_eq!(object.test, "value");
1013 let events = responses(&captured);
1014 assert_eq!(events.len(), 1);
1015 assert_eq!(events[0].evidence, reported_evidence());
1016 }
1017
1018 mod stream_events {
1019 use super::*;
1020 use crate::llm::models::ResponseEvidence;
1021 use crate::llm::stream_events::{StreamEvent, StreamEventError, StreamEventStream};
1022 use crate::tracer::testing::{capturing_tracer, responses, CapturedLlmEvent};
1023
1024 type Script = fn() -> Vec<StreamEvent>;
1026
1027 struct ScriptedEventsGateway {
1029 script: Script,
1030 seen_tool_limit: std::sync::Mutex<Option<usize>>,
1031 }
1032
1033 impl ScriptedEventsGateway {
1034 fn new(script: Script) -> Self {
1035 Self {
1036 script,
1037 seen_tool_limit: std::sync::Mutex::new(None),
1038 }
1039 }
1040 }
1041
1042 #[async_trait::async_trait]
1043 impl LlmGateway for ScriptedEventsGateway {
1044 async fn complete(
1045 &self,
1046 _model: &str,
1047 _messages: &[LlmMessage],
1048 _tools: Option<&[Box<dyn LlmTool>]>,
1049 _config: &CompletionConfig,
1050 ) -> Result<LlmGatewayResponse> {
1051 unreachable!("the events API makes no ordinary completion")
1052 }
1053
1054 async fn complete_json(
1055 &self,
1056 _model: &str,
1057 _messages: &[LlmMessage],
1058 _schema: Value,
1059 _config: &CompletionConfig,
1060 ) -> Result<Value> {
1061 unreachable!("the events API makes no structured completion")
1062 }
1063
1064 async fn get_available_models(&self) -> Result<Vec<String>> {
1065 Ok(vec![])
1066 }
1067
1068 async fn calculate_embeddings(
1069 &self,
1070 _text: &str,
1071 _model: Option<&str>,
1072 ) -> Result<Vec<f32>> {
1073 Ok(vec![])
1074 }
1075
1076 fn complete_stream<'a>(
1077 &'a self,
1078 _model: &'a str,
1079 _messages: &'a [LlmMessage],
1080 _tools: Option<&'a [Box<dyn LlmTool>]>,
1081 _config: &'a CompletionConfig,
1082 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1083 unreachable!("the events API does not use the legacy stream")
1084 }
1085
1086 fn complete_stream_events<'a>(
1087 &'a self,
1088 _model: &'a str,
1089 _messages: &'a [LlmMessage],
1090 config: &'a CompletionConfig,
1091 ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
1092 *self.seen_tool_limit.lock().unwrap() = Some(config.max_tool_iterations);
1093 Ok(Box::pin(futures::stream::iter((self.script)())))
1094 }
1095 }
1096
1097 fn stop_evidence() -> ResponseEvidence {
1098 ResponseEvidence {
1099 usage: Some(serde_json::json!({"total_tokens": 7})),
1100 provider_model: Some("provider-model".to_string()),
1101 finish_reason: Some("stop".to_string()),
1102 metadata: HashMap::from([("total_duration".to_string(), serde_json::json!(900))]),
1103 }
1104 }
1105
1106 fn broker_with(
1107 script: Script,
1108 ) -> (LlmBroker, Arc<ScriptedEventsGateway>, crate::tracer::testing::CapturedLlmEvents)
1109 {
1110 let gateway = Arc::new(ScriptedEventsGateway::new(script));
1111 let (tracer, captured) = capturing_tracer();
1112 let broker = LlmBroker::new("configured-model", gateway.clone(), Some(tracer));
1113 (broker, gateway, captured)
1114 }
1115
1116 #[tokio::test]
1117 async fn content_then_completed_is_forwarded_and_traced_with_evidence() {
1118 let (broker, _, captured) = broker_with(|| {
1119 vec![
1120 StreamEvent::Content("Hel".into()),
1121 StreamEvent::Content("lo".into()),
1122 StreamEvent::Completed(stop_evidence()),
1123 ]
1124 });
1125 let messages = vec![LlmMessage::user("Hi")];
1126
1127 let events: Vec<StreamEvent> =
1128 broker.generate_stream_events(&messages, None, None).collect().await;
1129
1130 assert!(matches!(
1131 events.as_slice(),
1132 [StreamEvent::Content(a), StreamEvent::Content(b), StreamEvent::Completed(evidence)]
1133 if a == "Hel" && b == "lo" && *evidence == stop_evidence()
1134 ));
1135 let recorded = captured.lock().unwrap().clone();
1136 assert!(matches!(
1137 recorded.as_slice(),
1138 [CapturedLlmEvent::Call(call), CapturedLlmEvent::Response(response)]
1139 if call.model == "configured-model"
1140 && call.tools.is_none()
1141 && response.content == "Hello"
1142 && response.model == "configured-model"
1143 && response.evidence == stop_evidence()
1144 ));
1145 }
1146
1147 #[tokio::test]
1148 async fn incomplete_completion_is_traced_with_its_evidence() {
1149 let (broker, _, captured) = broker_with(|| {
1150 vec![
1151 StreamEvent::Content("Part".into()),
1152 StreamEvent::Error(StreamEventError::IncompleteCompletion(Box::new(
1153 ResponseEvidence {
1154 finish_reason: Some("length".into()),
1155 usage: Some(serde_json::json!({"completion_tokens": 16})),
1156 ..Default::default()
1157 },
1158 ))),
1159 ]
1160 });
1161 let messages = vec![LlmMessage::user("Hi")];
1162
1163 let events: Vec<StreamEvent> =
1164 broker.generate_stream_events(&messages, None, None).collect().await;
1165
1166 assert!(matches!(
1167 events.last(),
1168 Some(StreamEvent::Error(StreamEventError::IncompleteCompletion(_)))
1169 ));
1170 let traced = responses(&captured);
1171 assert_eq!(traced.len(), 1);
1172 assert_eq!(traced[0].content, "Part");
1173 assert_eq!(traced[0].evidence.finish_reason.as_deref(), Some("length"));
1174 assert_eq!(
1175 traced[0].evidence.usage,
1176 Some(serde_json::json!({"completion_tokens": 16}))
1177 );
1178 }
1179
1180 #[tokio::test]
1181 async fn incomplete_stream_is_traced_with_its_partial_evidence() {
1182 let (broker, _, captured) = broker_with(|| {
1183 vec![StreamEvent::Error(StreamEventError::IncompleteStream(
1184 Some(Box::new(ResponseEvidence {
1185 provider_model: Some("provider-model".into()),
1186 ..Default::default()
1187 })),
1188 ))]
1189 });
1190 let messages = vec![LlmMessage::user("Hi")];
1191
1192 let _: Vec<StreamEvent> =
1193 broker.generate_stream_events(&messages, None, None).collect().await;
1194
1195 let traced = responses(&captured);
1196 assert_eq!(traced[0].evidence.provider_model.as_deref(), Some("provider-model"));
1197 }
1198
1199 #[tokio::test]
1200 async fn a_gateway_stream_without_terminal_event_ends_incomplete() {
1201 let (broker, _, captured) = broker_with(|| vec![StreamEvent::Content("Hi".into())]);
1202 let messages = vec![LlmMessage::user("Hi")];
1203
1204 let events: Vec<StreamEvent> =
1205 broker.generate_stream_events(&messages, None, None).collect().await;
1206
1207 assert!(matches!(
1208 events.as_slice(),
1209 [
1210 StreamEvent::Content(_),
1211 StreamEvent::Error(StreamEventError::IncompleteStream(None))
1212 ]
1213 ));
1214 assert_eq!(responses(&captured).len(), 1);
1215 }
1216
1217 #[tokio::test]
1218 async fn nothing_follows_the_terminal_event() {
1219 let (broker, _, _) = broker_with(|| {
1220 vec![
1221 StreamEvent::Completed(stop_evidence()),
1222 StreamEvent::Content("late".into()),
1223 StreamEvent::Completed(stop_evidence()),
1224 ]
1225 });
1226 let messages = vec![LlmMessage::user("Hi")];
1227
1228 let events: Vec<StreamEvent> =
1229 broker.generate_stream_events(&messages, None, None).collect().await;
1230
1231 assert!(matches!(events.as_slice(), [StreamEvent::Completed(_)]));
1232 }
1233
1234 #[tokio::test]
1235 async fn tool_iterations_are_forced_to_zero() {
1236 let (broker, gateway, _) =
1237 broker_with(|| vec![StreamEvent::Completed(stop_evidence())]);
1238 let messages = vec![LlmMessage::user("Hi")];
1239 let config = CompletionConfig::default().with_unlimited_tool_iterations();
1240
1241 let _: Vec<StreamEvent> =
1242 broker.generate_stream_events(&messages, Some(config), None).collect().await;
1243
1244 assert_eq!(*gateway.seen_tool_limit.lock().unwrap(), Some(0));
1245 }
1246
1247 #[tokio::test]
1248 async fn stopping_early_traces_the_call_but_no_response() {
1249 let (broker, _, captured) = broker_with(|| {
1250 vec![
1251 StreamEvent::Content("Hi".into()),
1252 StreamEvent::Completed(stop_evidence()),
1253 ]
1254 });
1255 let messages = vec![LlmMessage::user("Hi")];
1256
1257 let mut stream = broker.generate_stream_events(&messages, None, None);
1258 assert!(matches!(stream.next().await, Some(StreamEvent::Content(_))));
1259 drop(stream);
1260
1261 let recorded = captured.lock().unwrap().clone();
1262 assert!(matches!(recorded.as_slice(), [CapturedLlmEvent::Call(_)]));
1263 }
1264
1265 #[tokio::test]
1266 async fn an_unsupported_gateway_yields_one_error_without_a_trace() {
1267 let (tracer, captured) = capturing_tracer();
1268 let broker = LlmBroker::new(
1269 "configured-model",
1270 Arc::new(MockGateway::new(vec![])),
1271 Some(tracer),
1272 );
1273 let messages = vec![LlmMessage::user("Hi")];
1274
1275 let events: Vec<StreamEvent> =
1276 broker.generate_stream_events(&messages, None, None).collect().await;
1277
1278 assert!(matches!(
1279 events.as_slice(),
1280 [StreamEvent::Error(
1281 StreamEventError::StreamEventsUnsupported
1282 )]
1283 ));
1284 assert!(captured.lock().unwrap().is_empty());
1285 }
1286
1287 #[tokio::test]
1288 async fn dropping_the_broker_stream_cancels_the_provider_request() {
1289 use crate::llm::gateways::OllamaGateway;
1290 use crate::llm::stream_events::testing::endless_stream_server;
1291
1292 let (url, closed) =
1293 endless_stream_server("{\"message\":{\"content\":\"Hi\"},\"done\":false}\n").await;
1294 let broker = LlmBroker::new("qwen3:32b", Arc::new(OllamaGateway::with_host(url)), None);
1295 let messages = vec![LlmMessage::user("Hi")];
1296
1297 let mut stream = broker.generate_stream_events(&messages, None, None);
1298 assert!(matches!(stream.next().await, Some(StreamEvent::Content(_))));
1299 drop(stream);
1300
1301 tokio::time::timeout(std::time::Duration::from_secs(5), closed)
1302 .await
1303 .expect("the request was not cancelled")
1304 .expect("server reported the disconnect");
1305 }
1306
1307 #[test]
1308 fn the_event_stream_can_move_across_threads() {
1309 fn assert_send<T: Send>(_: &T) {}
1310 let broker = LlmBroker::new("m", Arc::new(MockGateway::new(vec![])), None);
1311 let messages = vec![LlmMessage::user("Hi")];
1312
1313 assert_send(&broker.generate_stream_events(&messages, None, None));
1314 }
1315 }
1316
1317 struct MockGateway {
1319 responses: Vec<LlmGatewayResponse>,
1320 call_count: std::sync::Mutex<usize>,
1321 }
1322
1323 impl MockGateway {
1324 fn new(responses: Vec<LlmGatewayResponse>) -> Self {
1325 Self {
1326 responses,
1327 call_count: std::sync::Mutex::new(0),
1328 }
1329 }
1330 }
1331
1332 #[async_trait::async_trait]
1333 impl LlmGateway for MockGateway {
1334 async fn complete(
1335 &self,
1336 _model: &str,
1337 _messages: &[LlmMessage],
1338 _tools: Option<&[Box<dyn LlmTool>]>,
1339 _config: &CompletionConfig,
1340 ) -> Result<LlmGatewayResponse> {
1341 let mut count = self.call_count.lock().unwrap();
1342 let idx = *count;
1343 *count += 1;
1344
1345 if idx < self.responses.len() {
1346 Ok(self.responses[idx].clone())
1347 } else {
1348 Ok(LlmGatewayResponse {
1349 content: Some("default response".to_string()),
1350 object: None,
1351 tool_calls: vec![],
1352 thinking: None,
1353 ..Default::default()
1354 })
1355 }
1356 }
1357
1358 async fn complete_json(
1359 &self,
1360 _model: &str,
1361 _messages: &[LlmMessage],
1362 _schema: Value,
1363 _config: &CompletionConfig,
1364 ) -> Result<Value> {
1365 Ok(serde_json::json!({"test": "value"}))
1366 }
1367
1368 async fn get_available_models(&self) -> Result<Vec<String>> {
1369 Ok(vec!["test-model".to_string()])
1370 }
1371
1372 async fn calculate_embeddings(
1373 &self,
1374 _text: &str,
1375 _model: Option<&str>,
1376 ) -> Result<Vec<f32>> {
1377 Ok(vec![0.1, 0.2, 0.3])
1378 }
1379
1380 fn complete_stream<'a>(
1381 &'a self,
1382 _model: &'a str,
1383 _messages: &'a [LlmMessage],
1384 _tools: Option<&'a [Box<dyn LlmTool>]>,
1385 _config: &'a CompletionConfig,
1386 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1387 use futures::stream;
1388 Box::pin(stream::iter(vec![Ok(StreamChunk::Content("test".to_string()))]))
1389 }
1390 }
1391
1392 struct MockTool {
1394 name: String,
1395 result: Value,
1396 }
1397
1398 #[async_trait]
1399
1400 impl LlmTool for MockTool {
1401 async fn run(
1402 &self,
1403 _args: &HashMap<String, Value>,
1404 _ctx: &crate::llm::tools::ToolRunCtx,
1405 ) -> Result<Value> {
1406 Ok(self.result.clone())
1407 }
1408
1409 fn descriptor(&self) -> ToolDescriptor {
1410 ToolDescriptor {
1411 r#type: "function".to_string(),
1412 function: FunctionDescriptor {
1413 name: self.name.clone(),
1414 description: "A mock tool".to_string(),
1415 parameters: serde_json::json!({}),
1416 },
1417 }
1418 }
1419
1420 fn clone_box(&self) -> Box<dyn LlmTool> {
1421 Box::new(MockTool {
1422 name: self.name.clone(),
1423 result: self.result.clone(),
1424 })
1425 }
1426 }
1427
1428 #[tokio::test]
1429 async fn test_broker_new() {
1430 let gateway = Arc::new(MockGateway::new(vec![]));
1431 let broker = LlmBroker::new("test-model", gateway, None);
1432 assert_eq!(broker.model, "test-model");
1433 assert_eq!(CompletionConfig::default().max_tool_iterations, 10);
1434 }
1435
1436 #[tokio::test]
1437 async fn test_broker_with_max_tool_iterations() {
1438 assert_eq!(
1439 CompletionConfig {
1440 max_tool_iterations: 3,
1441 ..Default::default()
1442 }
1443 .max_tool_iterations,
1444 3
1445 );
1446 }
1447
1448 #[tokio::test]
1449 async fn test_generate_max_tool_iterations_exceeded() {
1450 let tool_call = LlmToolCall {
1452 id: Some("call_loop".to_string()),
1453 name: "loop_tool".to_string(),
1454 arguments: HashMap::new(),
1455 };
1456
1457 let responses: Vec<LlmGatewayResponse> = (0..15)
1459 .map(|_| LlmGatewayResponse {
1460 content: None,
1461 object: None,
1462 tool_calls: vec![tool_call.clone()],
1463 thinking: None,
1464 ..Default::default()
1465 })
1466 .collect();
1467
1468 let gateway = Arc::new(MockGateway::new(responses));
1469 let broker = LlmBroker::new("test-model", gateway, None);
1470
1471 let tool = MockTool {
1472 name: "loop_tool".to_string(),
1473 result: serde_json::json!({"result": "loop"}),
1474 };
1475 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1476
1477 let messages = vec![LlmMessage::user("Loop forever")];
1478 let result = broker
1479 .generate(
1480 &messages,
1481 Some(&tools),
1482 Some(CompletionConfig {
1483 max_tool_iterations: 3,
1484 ..Default::default()
1485 }),
1486 None,
1487 )
1488 .await;
1489
1490 assert!(result.is_err());
1491 match result.unwrap_err() {
1492 crate::error::MojenticError::MaxToolIterationsExceeded { limit } => {
1493 assert_eq!(limit, 3);
1494 }
1495 e => panic!("Expected MaxToolIterationsExceeded, got: {:?}", e),
1496 }
1497 }
1498
1499 #[tokio::test]
1500 async fn test_generate_stream_max_tool_iterations_exceeded() {
1501 use futures::stream;
1502
1503 struct InfiniteToolGateway;
1504
1505 #[async_trait::async_trait]
1506 impl LlmGateway for InfiniteToolGateway {
1507 async fn complete(
1508 &self,
1509 _model: &str,
1510 _messages: &[LlmMessage],
1511 _tools: Option<&[Box<dyn LlmTool>]>,
1512 _config: &CompletionConfig,
1513 ) -> Result<LlmGatewayResponse> {
1514 Ok(LlmGatewayResponse {
1515 content: None,
1516 object: None,
1517 tool_calls: vec![],
1518 thinking: None,
1519 ..Default::default()
1520 })
1521 }
1522
1523 async fn complete_json(
1524 &self,
1525 _model: &str,
1526 _messages: &[LlmMessage],
1527 _schema: serde_json::Value,
1528 _config: &CompletionConfig,
1529 ) -> Result<serde_json::Value> {
1530 Ok(serde_json::json!({}))
1531 }
1532
1533 async fn get_available_models(&self) -> Result<Vec<String>> {
1534 Ok(vec![])
1535 }
1536
1537 async fn calculate_embeddings(
1538 &self,
1539 _text: &str,
1540 _model: Option<&str>,
1541 ) -> Result<Vec<f32>> {
1542 Ok(vec![])
1543 }
1544
1545 fn complete_stream<'a>(
1546 &'a self,
1547 _model: &'a str,
1548 _messages: &'a [LlmMessage],
1549 _tools: Option<&'a [Box<dyn LlmTool>]>,
1550 _config: &'a CompletionConfig,
1551 ) -> std::pin::Pin<
1552 Box<dyn futures::stream::Stream<Item = Result<StreamChunk>> + Send + 'a>,
1553 > {
1554 Box::pin(stream::iter(vec![Ok(StreamChunk::ToolCalls(vec![LlmToolCall {
1555 id: Some("call_loop".to_string()),
1556 name: "loop_tool".to_string(),
1557 arguments: HashMap::new(),
1558 }]))]))
1559 }
1560 }
1561
1562 let gateway = Arc::new(InfiniteToolGateway);
1563 let broker = LlmBroker::new("test-model", gateway, None);
1564
1565 let tool = MockTool {
1566 name: "loop_tool".to_string(),
1567 result: serde_json::json!({"result": "loop"}),
1568 };
1569 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1570
1571 let messages = vec![LlmMessage::user("Loop forever")];
1572 let mut stream = broker.generate_stream(
1573 &messages,
1574 Some(&tools),
1575 Some(CompletionConfig {
1576 max_tool_iterations: 2,
1577 ..Default::default()
1578 }),
1579 None,
1580 );
1581
1582 let mut found_error = false;
1583 while let Some(result) = stream.next().await {
1584 if let Err(crate::error::MojenticError::MaxToolIterationsExceeded { .. }) = result {
1585 found_error = true;
1586 break;
1587 }
1588 }
1589 assert!(found_error, "Expected MaxToolIterationsExceeded error in stream");
1590 }
1591
1592 #[tokio::test]
1593 async fn test_broker_new_string_conversion() {
1594 let gateway = Arc::new(MockGateway::new(vec![]));
1595 let broker = LlmBroker::new(String::from("my-model"), gateway, None);
1596 assert_eq!(broker.model, "my-model");
1597 }
1598
1599 #[tokio::test]
1600 async fn test_generate_simple_response() {
1601 let response = LlmGatewayResponse {
1602 content: Some("Hello, World!".to_string()),
1603 object: None,
1604 tool_calls: vec![],
1605 thinking: None,
1606 ..Default::default()
1607 };
1608
1609 let gateway = Arc::new(MockGateway::new(vec![response]));
1610 let broker = LlmBroker::new("test-model", gateway, None);
1611
1612 let messages = vec![LlmMessage::user("Hi")];
1613 let result = broker.generate(&messages, None, None, None).await.unwrap();
1614
1615 assert_eq!(result, "Hello, World!");
1616 }
1617
1618 #[tokio::test]
1619 async fn test_generate_with_custom_config() {
1620 let response = LlmGatewayResponse {
1621 content: Some("Response".to_string()),
1622 object: None,
1623 tool_calls: vec![],
1624 thinking: None,
1625 ..Default::default()
1626 };
1627
1628 let gateway = Arc::new(MockGateway::new(vec![response]));
1629 let broker = LlmBroker::new("test-model", gateway, None);
1630
1631 let config = CompletionConfig {
1632 temperature: Some(0.5),
1633 num_ctx: Some(2048),
1634 max_tokens: Some(100),
1635 top_p: None,
1636 top_k: None,
1637 response_format: None,
1638 reasoning_effort: None,
1639 reasoning_profile: None,
1640 preserve_reasoning: false,
1641 max_tool_iterations: 10,
1642 };
1643
1644 let messages = vec![LlmMessage::user("Hi")];
1645 let result = broker.generate(&messages, None, Some(config), None).await.unwrap();
1646
1647 assert_eq!(result, "Response");
1648 }
1649
1650 #[tokio::test]
1651 async fn test_generate_empty_response_content() {
1652 let response = LlmGatewayResponse {
1653 content: None,
1654 object: None,
1655 tool_calls: vec![],
1656 thinking: None,
1657 ..Default::default()
1658 };
1659
1660 let gateway = Arc::new(MockGateway::new(vec![response]));
1661 let broker = LlmBroker::new("test-model", gateway, None);
1662
1663 let messages = vec![LlmMessage::user("Hi")];
1664 let result = broker.generate(&messages, None, None, None).await.unwrap();
1665
1666 assert_eq!(result, "");
1667 }
1668
1669 #[tokio::test]
1670 async fn test_generate_with_tool_call() {
1671 let tool_call = LlmToolCall {
1672 id: Some("call_1".to_string()),
1673 name: "test_tool".to_string(),
1674 arguments: HashMap::new(),
1675 };
1676
1677 let first_response = LlmGatewayResponse {
1678 content: None,
1679 object: None,
1680 tool_calls: vec![tool_call],
1681 thinking: None,
1682 ..Default::default()
1683 };
1684
1685 let second_response = LlmGatewayResponse {
1686 content: Some("After tool execution".to_string()),
1687 object: None,
1688 tool_calls: vec![],
1689 thinking: None,
1690 ..Default::default()
1691 };
1692
1693 let gateway = Arc::new(MockGateway::new(vec![first_response, second_response]));
1694 let broker = LlmBroker::new("test-model", gateway, None);
1695
1696 let tool = MockTool {
1697 name: "test_tool".to_string(),
1698 result: serde_json::json!({"result": "success"}),
1699 };
1700
1701 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1702
1703 let messages = vec![LlmMessage::user("Use the tool")];
1704 let result = broker.generate(&messages, Some(&tools), None, None).await.unwrap();
1705
1706 assert_eq!(result, "After tool execution");
1707 }
1708
1709 #[tokio::test]
1710 async fn test_generate_with_tool_call_no_tools_provided() {
1711 let tool_call = LlmToolCall {
1712 id: Some("call_1".to_string()),
1713 name: "test_tool".to_string(),
1714 arguments: HashMap::new(),
1715 };
1716
1717 let response = LlmGatewayResponse {
1718 content: Some("fallback".to_string()),
1719 object: None,
1720 tool_calls: vec![tool_call],
1721 thinking: None,
1722 ..Default::default()
1723 };
1724
1725 let gateway = Arc::new(MockGateway::new(vec![response]));
1726 let broker = LlmBroker::new("test-model", gateway, None);
1727
1728 let messages = vec![LlmMessage::user("Use the tool")];
1729 let result = broker.generate(&messages, None, None, None).await.unwrap();
1730
1731 assert_eq!(result, "fallback");
1732 }
1733
1734 #[tokio::test]
1735 async fn test_generate_object() {
1736 #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
1737 struct TestObject {
1738 test: String,
1739 }
1740
1741 let gateway = Arc::new(MockGateway::new(vec![]));
1742 let broker = LlmBroker::new("test-model", gateway, None);
1743
1744 let messages = vec![LlmMessage::user("Generate object")];
1745 let result: TestObject = broker.generate_object(&messages, None, None).await.unwrap();
1746
1747 assert_eq!(result.test, "value");
1748 }
1749
1750 #[tokio::test]
1751 async fn test_generate_object_with_config() {
1752 #[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
1753 struct TestData {
1754 test: String,
1755 }
1756
1757 let gateway = Arc::new(MockGateway::new(vec![]));
1758 let broker = LlmBroker::new("test-model", gateway, None);
1759
1760 let config = CompletionConfig {
1761 temperature: Some(0.1),
1762 num_ctx: Some(1024),
1763 max_tokens: Some(50),
1764 top_p: None,
1765 top_k: None,
1766 response_format: None,
1767 reasoning_effort: None,
1768 reasoning_profile: None,
1769 preserve_reasoning: false,
1770 max_tool_iterations: 10,
1771 };
1772
1773 let messages = vec![LlmMessage::user("Generate")];
1774 let result: TestData = broker.generate_object(&messages, Some(config), None).await.unwrap();
1775
1776 assert_eq!(result.test, "value");
1777 }
1778
1779 #[tokio::test]
1780 async fn test_multiple_messages() {
1781 let response = LlmGatewayResponse {
1782 content: Some("Response to conversation".to_string()),
1783 object: None,
1784 tool_calls: vec![],
1785 thinking: None,
1786 ..Default::default()
1787 };
1788
1789 let gateway = Arc::new(MockGateway::new(vec![response]));
1790 let broker = LlmBroker::new("test-model", gateway, None);
1791
1792 let messages = vec![
1793 LlmMessage::system("You are helpful"),
1794 LlmMessage::user("First message"),
1795 LlmMessage::assistant("First response"),
1796 LlmMessage::user("Second message"),
1797 ];
1798
1799 let result = broker.generate(&messages, None, None, None).await.unwrap();
1800 assert_eq!(result, "Response to conversation");
1801 }
1802
1803 #[tokio::test]
1804 async fn test_generate_stream_basic() {
1805 use futures::stream;
1806
1807 struct StreamingMockGateway;
1809
1810 #[async_trait::async_trait]
1811 impl LlmGateway for StreamingMockGateway {
1812 async fn complete(
1813 &self,
1814 _model: &str,
1815 _messages: &[LlmMessage],
1816 _tools: Option<&[Box<dyn LlmTool>]>,
1817 _config: &CompletionConfig,
1818 ) -> Result<LlmGatewayResponse> {
1819 Ok(LlmGatewayResponse {
1820 content: Some("test".to_string()),
1821 object: None,
1822 tool_calls: vec![],
1823 thinking: None,
1824 ..Default::default()
1825 })
1826 }
1827
1828 async fn complete_json(
1829 &self,
1830 _model: &str,
1831 _messages: &[LlmMessage],
1832 _schema: Value,
1833 _config: &CompletionConfig,
1834 ) -> Result<Value> {
1835 Ok(serde_json::json!({}))
1836 }
1837
1838 async fn get_available_models(&self) -> Result<Vec<String>> {
1839 Ok(vec![])
1840 }
1841
1842 async fn calculate_embeddings(
1843 &self,
1844 _text: &str,
1845 _model: Option<&str>,
1846 ) -> Result<Vec<f32>> {
1847 Ok(vec![])
1848 }
1849
1850 fn complete_stream<'a>(
1851 &'a self,
1852 _model: &'a str,
1853 _messages: &'a [LlmMessage],
1854 _tools: Option<&'a [Box<dyn LlmTool>]>,
1855 _config: &'a CompletionConfig,
1856 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1857 Box::pin(stream::iter(vec![
1858 Ok(StreamChunk::Content("Hello".to_string())),
1859 Ok(StreamChunk::Content(" ".to_string())),
1860 Ok(StreamChunk::Content("World".to_string())),
1861 ]))
1862 }
1863 }
1864
1865 let gateway = Arc::new(StreamingMockGateway);
1866 let broker = LlmBroker::new("test-model", gateway, None);
1867 let messages = vec![LlmMessage::user("Hello")];
1868
1869 let mut stream = broker.generate_stream(&messages, None, None, None);
1870 let mut result = String::new();
1871
1872 while let Some(chunk) = stream.next().await {
1873 result.push_str(&chunk.unwrap());
1874 }
1875
1876 assert_eq!(result, "Hello World");
1877 }
1878
1879 #[tokio::test]
1880 async fn test_generate_stream_with_tool_calls() {
1881 use futures::stream;
1882
1883 struct ToolCallMockGateway {
1885 call_count: std::sync::Mutex<usize>,
1886 }
1887
1888 impl ToolCallMockGateway {
1889 fn new() -> Self {
1890 Self {
1891 call_count: std::sync::Mutex::new(0),
1892 }
1893 }
1894 }
1895
1896 #[async_trait::async_trait]
1897 impl LlmGateway for ToolCallMockGateway {
1898 async fn complete(
1899 &self,
1900 _model: &str,
1901 _messages: &[LlmMessage],
1902 _tools: Option<&[Box<dyn LlmTool>]>,
1903 _config: &CompletionConfig,
1904 ) -> Result<LlmGatewayResponse> {
1905 Ok(LlmGatewayResponse {
1906 content: Some("test".to_string()),
1907 object: None,
1908 tool_calls: vec![],
1909 thinking: None,
1910 ..Default::default()
1911 })
1912 }
1913
1914 async fn complete_json(
1915 &self,
1916 _model: &str,
1917 _messages: &[LlmMessage],
1918 _schema: Value,
1919 _config: &CompletionConfig,
1920 ) -> Result<Value> {
1921 Ok(serde_json::json!({}))
1922 }
1923
1924 async fn get_available_models(&self) -> Result<Vec<String>> {
1925 Ok(vec![])
1926 }
1927
1928 async fn calculate_embeddings(
1929 &self,
1930 _text: &str,
1931 _model: Option<&str>,
1932 ) -> Result<Vec<f32>> {
1933 Ok(vec![])
1934 }
1935
1936 fn complete_stream<'a>(
1937 &'a self,
1938 _model: &'a str,
1939 _messages: &'a [LlmMessage],
1940 _tools: Option<&'a [Box<dyn LlmTool>]>,
1941 _config: &'a CompletionConfig,
1942 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
1943 let mut count = self.call_count.lock().unwrap();
1944 let call_num = *count;
1945 *count += 1;
1946
1947 if call_num == 0 {
1948 Box::pin(stream::iter(vec![
1950 Ok(StreamChunk::Content("Initial ".to_string())),
1951 Ok(StreamChunk::Content("response".to_string())),
1952 Ok(StreamChunk::ToolCalls(vec![LlmToolCall {
1953 id: Some("call_1".to_string()),
1954 name: "test_tool".to_string(),
1955 arguments: HashMap::new(),
1956 }])),
1957 ]))
1958 } else {
1959 Box::pin(stream::iter(vec![
1961 Ok(StreamChunk::Content("After ".to_string())),
1962 Ok(StreamChunk::Content("tool".to_string())),
1963 ]))
1964 }
1965 }
1966 }
1967
1968 let gateway = Arc::new(ToolCallMockGateway::new());
1969 let broker = LlmBroker::new("test-model", gateway, None);
1970
1971 let tool = MockTool {
1972 name: "test_tool".to_string(),
1973 result: serde_json::json!({"result": "success"}),
1974 };
1975 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
1976
1977 let messages = vec![LlmMessage::user("Use the tool")];
1978 let mut stream = broker.generate_stream(&messages, Some(&tools), None, None);
1979
1980 let mut result = String::new();
1981 while let Some(chunk) = stream.next().await {
1982 result.push_str(&chunk.unwrap());
1983 }
1984
1985 assert!(result.contains("Initial response"));
1987 assert!(result.contains("After tool"));
1988 }
1989
1990 #[tokio::test]
1991 async fn test_generate_stream_without_tools() {
1992 use futures::stream;
1993
1994 struct SimpleStreamGateway;
1995
1996 #[async_trait::async_trait]
1997 impl LlmGateway for SimpleStreamGateway {
1998 async fn complete(
1999 &self,
2000 _model: &str,
2001 _messages: &[LlmMessage],
2002 _tools: Option<&[Box<dyn LlmTool>]>,
2003 _config: &CompletionConfig,
2004 ) -> Result<LlmGatewayResponse> {
2005 Ok(LlmGatewayResponse {
2006 content: Some("test".to_string()),
2007 object: None,
2008 tool_calls: vec![],
2009 thinking: None,
2010 ..Default::default()
2011 })
2012 }
2013
2014 async fn complete_json(
2015 &self,
2016 _model: &str,
2017 _messages: &[LlmMessage],
2018 _schema: Value,
2019 _config: &CompletionConfig,
2020 ) -> Result<Value> {
2021 Ok(serde_json::json!({}))
2022 }
2023
2024 async fn get_available_models(&self) -> Result<Vec<String>> {
2025 Ok(vec![])
2026 }
2027
2028 async fn calculate_embeddings(
2029 &self,
2030 _text: &str,
2031 _model: Option<&str>,
2032 ) -> Result<Vec<f32>> {
2033 Ok(vec![])
2034 }
2035
2036 fn complete_stream<'a>(
2037 &'a self,
2038 _model: &'a str,
2039 _messages: &'a [LlmMessage],
2040 _tools: Option<&'a [Box<dyn LlmTool>]>,
2041 _config: &'a CompletionConfig,
2042 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
2043 Box::pin(stream::iter(vec![
2045 Ok(StreamChunk::Content("Simple ".to_string())),
2046 Ok(StreamChunk::Content("stream".to_string())),
2047 ]))
2048 }
2049 }
2050
2051 let gateway = Arc::new(SimpleStreamGateway);
2052 let broker = LlmBroker::new("test-model", gateway, None);
2053
2054 let messages = vec![LlmMessage::user("Test")];
2055 let mut stream = broker.generate_stream(&messages, None, None, None);
2056
2057 let mut result = String::new();
2058 while let Some(chunk) = stream.next().await {
2059 result.push_str(&chunk.unwrap());
2060 }
2061
2062 assert_eq!(result, "Simple stream");
2063 }
2064
2065 #[tokio::test]
2066 async fn test_tracer_integration() {
2067 use crate::tracer::TracerSystem;
2068
2069 let response = LlmGatewayResponse {
2070 content: Some("Test response".to_string()),
2071 object: None,
2072 tool_calls: vec![],
2073 thinking: None,
2074 ..Default::default()
2075 };
2076
2077 let gateway = Arc::new(MockGateway::new(vec![response]));
2078 let tracer = Arc::new(TracerSystem::default());
2079 let broker = LlmBroker::new("test-model", gateway, Some(tracer.clone()));
2080
2081 let messages = vec![LlmMessage::user("Test")];
2082 let correlation_id = "test-correlation-123";
2083
2084 let result = broker
2085 .generate(&messages, None, None, Some(correlation_id.to_string()))
2086 .await
2087 .unwrap();
2088
2089 assert_eq!(result, "Test response");
2090
2091 assert_eq!(tracer.len(), 2); let summaries = tracer.get_event_summaries(None, None, None);
2096 assert!(summaries[0].contains(correlation_id));
2097 assert!(summaries[1].contains(correlation_id));
2098 }
2099
2100 #[tokio::test]
2101 async fn test_tracer_with_tool_calls() {
2102 use crate::tracer::TracerSystem;
2103
2104 let tool_call = LlmToolCall {
2105 id: Some("call_1".to_string()),
2106 name: "test_tool".to_string(),
2107 arguments: HashMap::new(),
2108 };
2109
2110 let first_response = LlmGatewayResponse {
2111 content: None,
2112 object: None,
2113 tool_calls: vec![tool_call],
2114 thinking: None,
2115 ..Default::default()
2116 };
2117
2118 let second_response = LlmGatewayResponse {
2119 content: Some("After tool".to_string()),
2120 object: None,
2121 tool_calls: vec![],
2122 thinking: None,
2123 ..Default::default()
2124 };
2125
2126 let gateway = Arc::new(MockGateway::new(vec![first_response, second_response]));
2127 let tracer = Arc::new(TracerSystem::default());
2128 let broker = LlmBroker::new("test-model", gateway, Some(tracer.clone()));
2129
2130 let tool = MockTool {
2131 name: "test_tool".to_string(),
2132 result: serde_json::json!({"result": "success"}),
2133 };
2134 let tools: Vec<Box<dyn LlmTool>> = vec![Box::new(tool)];
2135
2136 let messages = vec![LlmMessage::user("Use tool")];
2137 let correlation_id = "tool-test-456";
2138
2139 let result = broker
2140 .generate(&messages, Some(&tools), None, Some(correlation_id.to_string()))
2141 .await
2142 .unwrap();
2143
2144 assert_eq!(result, "After tool");
2145
2146 assert_eq!(tracer.len(), 5);
2148
2149 let summaries = tracer.get_event_summaries(None, None, None);
2151 for summary in &summaries {
2152 assert!(summary.contains(correlation_id));
2153 }
2154 }
2155}