1use crate::error::Result;
7use crate::llm::broker::LlmBroker;
8use crate::llm::gateway::CompletionConfig;
9use crate::llm::gateways::TokenizerGateway;
10use crate::llm::models::{LlmMessage, MessageRole};
11use crate::llm::tools::LlmTool;
12use futures::stream::{Stream, StreamExt};
13use serde::{Deserialize, Serialize};
14use std::pin::Pin;
15
16#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct SizedLlmMessage {
22 #[serde(flatten)]
23 pub message: LlmMessage,
24 pub token_length: usize,
25}
26
27impl SizedLlmMessage {
28 pub fn new(message: LlmMessage, token_length: usize) -> Self {
30 Self {
31 message,
32 token_length,
33 }
34 }
35
36 pub fn role(&self) -> MessageRole {
38 self.message.role
39 }
40
41 pub fn content(&self) -> Option<&str> {
43 self.message.content.as_deref()
44 }
45}
46
47pub struct ChatSession {
73 broker: LlmBroker,
74 messages: Vec<SizedLlmMessage>,
75 tools: Option<Vec<Box<dyn LlmTool>>>,
76 max_context: usize,
77 tokenizer_gateway: TokenizerGateway,
78 completion_config: CompletionConfig,
79}
80
81impl ChatSession {
82 pub fn new(broker: LlmBroker) -> Self {
100 Self::builder(broker).build()
101 }
102
103 pub fn builder(broker: LlmBroker) -> ChatSessionBuilder {
121 ChatSessionBuilder::new(broker)
122 }
123
124 pub async fn send(&mut self, query: &str) -> Result<String> {
147 self.insert_message(LlmMessage::user(query));
149
150 let messages: Vec<LlmMessage> = self.messages.iter().map(|m| m.message.clone()).collect();
152 let config = self.completion_config.clone();
153
154 let response = self
155 .broker
156 .generate(&messages, self.tools.as_deref(), Some(config), None)
157 .await?;
158
159 self.ensure_all_messages_are_sized();
161
162 self.insert_message(LlmMessage::assistant(&response));
164
165 Ok(response)
166 }
167
168 pub fn send_stream<'a>(
195 &'a mut self,
196 query: &str,
197 ) -> Pin<Box<dyn Stream<Item = Result<String>> + 'a>> {
198 self.insert_message(LlmMessage::user(query));
200
201 let messages: Vec<LlmMessage> = self.messages.iter().map(|m| m.message.clone()).collect();
203 let config = self.completion_config.clone();
204
205 Box::pin(async_stream::stream! {
206 let mut accumulated = Vec::new();
207 let tools_ref = self.tools.as_deref();
208 let mut inner_stream = self.broker.generate_stream(&messages, tools_ref, Some(config), None);
209
210 while let Some(result) = inner_stream.next().await {
211 match &result {
212 Ok(chunk) => {
213 accumulated.push(chunk.clone());
214 yield result;
215 }
216 Err(_) => {
217 yield result;
218 return;
219 }
220 }
221 }
222
223 drop(inner_stream);
225 self.ensure_all_messages_are_sized();
226 let full_response = accumulated.join("");
227 self.insert_message(LlmMessage::assistant(&full_response));
228 })
229 }
230
231 pub fn insert_message(&mut self, message: LlmMessage) {
241 let sized_message = self.build_sized_message(message);
242 self.messages.push(sized_message);
243
244 let mut total_length: usize = self.messages.iter().map(|m| m.token_length).sum();
246
247 while total_length > self.max_context && self.messages.len() > 1 {
248 let removed = self.messages.remove(1);
250 total_length -= removed.token_length;
251 }
252 }
253
254 pub fn messages(&self) -> &[SizedLlmMessage] {
256 &self.messages
257 }
258
259 pub fn total_tokens(&self) -> usize {
261 self.messages.iter().map(|m| m.token_length).sum()
262 }
263
264 fn build_sized_message(&self, message: LlmMessage) -> SizedLlmMessage {
266 let token_length = if let Some(content) = &message.content {
267 self.tokenizer_gateway.encode(content).len()
268 } else {
269 0
270 };
271
272 SizedLlmMessage::new(message, token_length)
273 }
274
275 fn ensure_all_messages_are_sized(&mut self) {
277 for i in 0..self.messages.len() {
278 if self.messages[i].token_length == 0 && self.messages[i].message.content.is_some() {
279 let content = self.messages[i].message.content.clone().unwrap();
280 let token_length = self.tokenizer_gateway.encode(&content).len();
281 self.messages[i].token_length = token_length;
282 }
283 }
284 }
285}
286
287pub struct ChatSessionBuilder {
289 broker: LlmBroker,
290 system_prompt: String,
291 tools: Option<Vec<Box<dyn LlmTool>>>,
292 max_context: usize,
293 tokenizer_gateway: Option<TokenizerGateway>,
294 completion_config: CompletionConfig,
295}
296
297impl ChatSessionBuilder {
298 fn new(broker: LlmBroker) -> Self {
300 Self {
301 broker,
302 system_prompt: "You are a helpful assistant.".to_string(),
303 tools: None,
304 max_context: 32768,
305 tokenizer_gateway: None,
306 completion_config: CompletionConfig::default(),
307 }
308 }
309
310 pub fn system_prompt(mut self, prompt: impl Into<String>) -> Self {
312 self.system_prompt = prompt.into();
313 self
314 }
315
316 pub fn tools(mut self, tools: Vec<Box<dyn LlmTool>>) -> Self {
318 self.tools = Some(tools);
319 self
320 }
321
322 pub fn max_context(mut self, max_context: usize) -> Self {
324 self.max_context = max_context;
325 self
326 }
327
328 pub fn tokenizer_gateway(mut self, gateway: TokenizerGateway) -> Self {
330 self.tokenizer_gateway = Some(gateway);
331 self
332 }
333
334 pub fn completion_config(mut self, config: CompletionConfig) -> Self {
339 self.completion_config = config;
340 self
341 }
342
343 pub fn temperature(mut self, temperature: f32) -> Self {
345 self.completion_config.temperature = Some(temperature);
346 self
347 }
348
349 pub fn build(self) -> ChatSession {
351 let tokenizer_gateway = self.tokenizer_gateway.unwrap_or_default();
352 let system_message = LlmMessage::system(&self.system_prompt);
353 let token_length = tokenizer_gateway.encode(&self.system_prompt).len();
354
355 ChatSession {
356 broker: self.broker,
357 messages: vec![SizedLlmMessage::new(system_message, token_length)],
358 tools: self.tools,
359 max_context: self.max_context,
360 tokenizer_gateway,
361 completion_config: self.completion_config,
362 }
363 }
364}
365
366#[cfg(test)]
367mod tests {
368 use super::*;
369 use crate::llm::gateway::{LlmGateway, StreamChunk};
370 use crate::llm::models::LlmGatewayResponse;
371 use crate::llm::tools::{FunctionDescriptor, ToolDescriptor};
372 use async_trait::async_trait;
373 use futures::stream::{self, Stream};
374 use serde_json::{json, Value};
375 use std::collections::HashMap;
376 use std::pin::Pin;
377 use std::sync::{Arc, Mutex};
378
379 struct MockGateway {
381 responses: Vec<String>,
382 call_count: Mutex<usize>,
383 }
384
385 impl MockGateway {
386 fn new(responses: Vec<String>) -> Self {
387 Self {
388 responses,
389 call_count: Mutex::new(0),
390 }
391 }
392 }
393
394 #[async_trait::async_trait]
395 impl LlmGateway for MockGateway {
396 async fn complete(
397 &self,
398 _model: &str,
399 _messages: &[LlmMessage],
400 _tools: Option<&[Box<dyn LlmTool>]>,
401 _config: &CompletionConfig,
402 ) -> Result<LlmGatewayResponse> {
403 let mut count = self.call_count.lock().unwrap();
404 let idx = *count;
405 *count += 1;
406
407 let content = if idx < self.responses.len() {
408 self.responses[idx].clone()
409 } else {
410 "default response".to_string()
411 };
412
413 Ok(LlmGatewayResponse {
414 content: Some(content),
415 object: None,
416 tool_calls: vec![],
417 thinking: None,
418 ..Default::default()
419 })
420 }
421
422 async fn complete_json(
423 &self,
424 _model: &str,
425 _messages: &[LlmMessage],
426 _schema: Value,
427 _config: &CompletionConfig,
428 ) -> Result<Value> {
429 Ok(json!({}))
430 }
431
432 async fn get_available_models(&self) -> Result<Vec<String>> {
433 Ok(vec!["test-model".to_string()])
434 }
435
436 async fn calculate_embeddings(
437 &self,
438 _text: &str,
439 _model: Option<&str>,
440 ) -> Result<Vec<f32>> {
441 Ok(vec![0.1, 0.2, 0.3])
442 }
443
444 fn complete_stream<'a>(
445 &'a self,
446 _model: &'a str,
447 _messages: &'a [LlmMessage],
448 _tools: Option<&'a [Box<dyn LlmTool>]>,
449 _config: &'a CompletionConfig,
450 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
451 Box::pin(stream::iter(vec![Ok(StreamChunk::Content("test".to_string()))]))
452 }
453 }
454
455 #[derive(Clone)]
457 struct MockTool {
458 name: String,
459 }
460
461 #[async_trait]
462
463 impl LlmTool for MockTool {
464 async fn run(
465 &self,
466 _args: &HashMap<String, Value>,
467 _ctx: &crate::llm::tools::ToolRunCtx,
468 ) -> Result<Value> {
469 Ok(json!({"result": "success"}))
470 }
471
472 fn descriptor(&self) -> ToolDescriptor {
473 ToolDescriptor {
474 r#type: "function".to_string(),
475 function: FunctionDescriptor {
476 name: self.name.clone(),
477 description: "A mock tool".to_string(),
478 parameters: json!({}),
479 },
480 }
481 }
482
483 fn clone_box(&self) -> Box<dyn LlmTool> {
484 Box::new(self.clone())
485 }
486 }
487
488 #[tokio::test]
489 async fn test_new_session_has_system_message() {
490 let gateway = Arc::new(MockGateway::new(vec![]));
491 let broker = LlmBroker::new("test-model", gateway, None);
492 let session = ChatSession::new(broker);
493
494 assert_eq!(session.messages.len(), 1);
495 assert_eq!(session.messages[0].role(), MessageRole::System);
496 assert_eq!(session.messages[0].content(), Some("You are a helpful assistant."));
497 }
498
499 #[tokio::test]
500 async fn test_builder_custom_system_prompt() {
501 let gateway = Arc::new(MockGateway::new(vec![]));
502 let broker = LlmBroker::new("test-model", gateway, None);
503 let session = ChatSession::builder(broker).system_prompt("Custom system prompt").build();
504
505 assert_eq!(session.messages.len(), 1);
506 assert_eq!(session.messages[0].content(), Some("Custom system prompt"));
507 }
508
509 #[tokio::test]
510 async fn test_builder_custom_temperature() {
511 let gateway = Arc::new(MockGateway::new(vec![]));
512 let broker = LlmBroker::new("test-model", gateway, None);
513 let session = ChatSession::builder(broker).temperature(0.5).build();
514
515 assert_eq!(session.completion_config.temperature, Some(0.5));
516 }
517
518 #[tokio::test]
519 async fn test_builder_custom_max_context() {
520 let gateway = Arc::new(MockGateway::new(vec![]));
521 let broker = LlmBroker::new("test-model", gateway, None);
522 let session = ChatSession::builder(broker).max_context(16384).build();
523
524 assert_eq!(session.max_context, 16384);
525 }
526
527 #[tokio::test]
528 async fn test_send_adds_messages_to_history() {
529 let gateway = Arc::new(MockGateway::new(vec!["Hello, World!".to_string()]));
530 let broker = LlmBroker::new("test-model", gateway, None);
531 let mut session = ChatSession::new(broker);
532
533 let response = session.send("Hi").await.unwrap();
534
535 assert_eq!(response, "Hello, World!");
536 assert_eq!(session.messages.len(), 3);
538 assert_eq!(session.messages[1].role(), MessageRole::User);
539 assert_eq!(session.messages[1].content(), Some("Hi"));
540 assert_eq!(session.messages[2].role(), MessageRole::Assistant);
541 assert_eq!(session.messages[2].content(), Some("Hello, World!"));
542 }
543
544 #[tokio::test]
545 async fn test_send_multiple_turns() {
546 let gateway = Arc::new(MockGateway::new(vec![
547 "First response".to_string(),
548 "Second response".to_string(),
549 ]));
550 let broker = LlmBroker::new("test-model", gateway, None);
551 let mut session = ChatSession::new(broker);
552
553 session.send("First query").await.unwrap();
554 session.send("Second query").await.unwrap();
555
556 assert_eq!(session.messages.len(), 5);
558 assert_eq!(session.messages[3].content(), Some("Second query"));
559 assert_eq!(session.messages[4].content(), Some("Second response"));
560 }
561
562 #[tokio::test]
563 async fn test_insert_message_calculates_token_length() {
564 let gateway = Arc::new(MockGateway::new(vec![]));
565 let broker = LlmBroker::new("test-model", gateway, None);
566 let mut session = ChatSession::new(broker);
567
568 session.insert_message(LlmMessage::user("Hello"));
569
570 assert_eq!(session.messages.len(), 2);
571 assert!(session.messages[1].token_length > 0);
572 }
573
574 #[tokio::test]
575 async fn test_context_window_trimming() {
576 let gateway = Arc::new(MockGateway::new(vec![]));
577 let broker = LlmBroker::new("test-model", gateway, None);
578
579 let mut session = ChatSession::builder(broker).max_context(50).build();
581
582 for i in 0..10 {
584 session.insert_message(LlmMessage::user(format!(
585 "This is a longer message number {} with more content to increase token count",
586 i
587 )));
588 }
589
590 assert!(session.messages.len() < 11); assert_eq!(session.messages[0].role(), MessageRole::System);
595
596 assert!(session.total_tokens() <= 50);
598 }
599
600 #[tokio::test]
601 async fn test_context_window_preserves_system_prompt() {
602 let gateway = Arc::new(MockGateway::new(vec![]));
603 let broker = LlmBroker::new("test-model", gateway, None);
604
605 let mut session = ChatSession::builder(broker)
606 .system_prompt("Important system prompt")
607 .max_context(50)
608 .build();
609
610 for i in 0..20 {
612 session.insert_message(LlmMessage::user(format!("Message {}", i)));
613 }
614
615 assert_eq!(session.messages[0].role(), MessageRole::System);
617 assert_eq!(session.messages[0].content(), Some("Important system prompt"));
618 }
619
620 #[tokio::test]
621 async fn test_total_tokens() {
622 let gateway = Arc::new(MockGateway::new(vec![]));
623 let broker = LlmBroker::new("test-model", gateway, None);
624 let mut session = ChatSession::new(broker);
625
626 let initial_tokens = session.total_tokens();
627 assert!(initial_tokens > 0); session.insert_message(LlmMessage::user("Hello"));
630
631 assert!(session.total_tokens() > initial_tokens);
632 }
633
634 #[tokio::test]
635 async fn test_messages_accessor() {
636 let gateway = Arc::new(MockGateway::new(vec![]));
637 let broker = LlmBroker::new("test-model", gateway, None);
638 let mut session = ChatSession::new(broker);
639
640 session.insert_message(LlmMessage::user("Test"));
641
642 let messages = session.messages();
643 assert_eq!(messages.len(), 2);
644 assert_eq!(messages[0].role(), MessageRole::System);
645 assert_eq!(messages[1].role(), MessageRole::User);
646 }
647
648 #[tokio::test]
649 async fn test_builder_with_tools() {
650 let gateway = Arc::new(MockGateway::new(vec![]));
651 let broker = LlmBroker::new("test-model", gateway, None);
652
653 let tool: Box<dyn LlmTool> = Box::new(MockTool {
654 name: "test_tool".to_string(),
655 });
656
657 let session = ChatSession::builder(broker).tools(vec![tool]).build();
658
659 assert!(session.tools.is_some());
660 assert_eq!(session.tools.as_ref().unwrap().len(), 1);
661 }
662
663 #[tokio::test]
664 async fn test_sized_message_creation() {
665 let message = LlmMessage::user("Test content");
666 let sized = SizedLlmMessage::new(message, 5);
667
668 assert_eq!(sized.token_length, 5);
669 assert_eq!(sized.role(), MessageRole::User);
670 assert_eq!(sized.content(), Some("Test content"));
671 }
672
673 struct StreamingMockGateway {
675 stream_chunks: Vec<Vec<String>>,
676 call_count: Mutex<usize>,
677 }
678
679 impl StreamingMockGateway {
680 fn new(stream_chunks: Vec<Vec<String>>) -> Self {
681 Self {
682 stream_chunks,
683 call_count: Mutex::new(0),
684 }
685 }
686 }
687
688 #[async_trait::async_trait]
689 impl LlmGateway for StreamingMockGateway {
690 async fn complete(
691 &self,
692 _model: &str,
693 _messages: &[LlmMessage],
694 _tools: Option<&[Box<dyn LlmTool>]>,
695 _config: &CompletionConfig,
696 ) -> Result<LlmGatewayResponse> {
697 Ok(LlmGatewayResponse {
698 content: Some("default".to_string()),
699 object: None,
700 tool_calls: vec![],
701 thinking: None,
702 ..Default::default()
703 })
704 }
705
706 async fn complete_json(
707 &self,
708 _model: &str,
709 _messages: &[LlmMessage],
710 _schema: Value,
711 _config: &CompletionConfig,
712 ) -> Result<Value> {
713 Ok(json!({}))
714 }
715
716 async fn get_available_models(&self) -> Result<Vec<String>> {
717 Ok(vec!["test-model".to_string()])
718 }
719
720 async fn calculate_embeddings(
721 &self,
722 _text: &str,
723 _model: Option<&str>,
724 ) -> Result<Vec<f32>> {
725 Ok(vec![0.1, 0.2, 0.3])
726 }
727
728 fn complete_stream<'a>(
729 &'a self,
730 _model: &'a str,
731 _messages: &'a [LlmMessage],
732 _tools: Option<&'a [Box<dyn LlmTool>]>,
733 _config: &'a CompletionConfig,
734 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
735 let mut count = self.call_count.lock().unwrap();
736 let idx = *count;
737 *count += 1;
738
739 let chunks = if idx < self.stream_chunks.len() {
740 self.stream_chunks[idx].clone()
741 } else {
742 vec!["default".to_string()]
743 };
744
745 Box::pin(stream::iter(
746 chunks.into_iter().map(|c| Ok(StreamChunk::Content(c))).collect::<Vec<_>>(),
747 ))
748 }
749 }
750
751 #[tokio::test]
752 async fn test_send_stream_yields_content_chunks() {
753 let gateway = Arc::new(StreamingMockGateway::new(vec![vec![
754 "Hello".to_string(),
755 " world".to_string(),
756 ]]));
757 let broker = LlmBroker::new("test-model", gateway, None);
758 let mut session = ChatSession::new(broker);
759
760 let mut chunks = Vec::new();
761 let mut stream = session.send_stream("Hi");
762 while let Some(result) = stream.next().await {
763 chunks.push(result.unwrap());
764 }
765
766 assert_eq!(chunks, vec!["Hello", " world"]);
767 }
768
769 #[tokio::test]
770 async fn test_send_stream_grows_message_history() {
771 let gateway = Arc::new(StreamingMockGateway::new(vec![vec!["Response".to_string()]]));
772 let broker = LlmBroker::new("test-model", gateway, None);
773 let mut session = ChatSession::new(broker);
774
775 {
776 let mut stream = session.send_stream("Hi");
777 while stream.next().await.is_some() {}
778 }
779
780 assert_eq!(session.messages.len(), 3);
782 }
783
784 #[tokio::test]
785 async fn test_send_stream_records_assembled_response() {
786 let gateway = Arc::new(StreamingMockGateway::new(vec![vec![
787 "Hello".to_string(),
788 " world".to_string(),
789 ]]));
790 let broker = LlmBroker::new("test-model", gateway, None);
791 let mut session = ChatSession::new(broker);
792
793 {
794 let mut stream = session.send_stream("Hi");
795 while stream.next().await.is_some() {}
796 }
797
798 assert_eq!(session.messages[2].content(), Some("Hello world"));
799 assert_eq!(session.messages[2].role(), MessageRole::Assistant);
800 }
801
802 #[tokio::test]
803 async fn test_send_stream_records_user_message() {
804 let gateway = Arc::new(StreamingMockGateway::new(vec![vec!["Response".to_string()]]));
805 let broker = LlmBroker::new("test-model", gateway, None);
806 let mut session = ChatSession::new(broker);
807
808 {
809 let mut stream = session.send_stream("My question");
810 while stream.next().await.is_some() {}
811 }
812
813 assert_eq!(session.messages[1].role(), MessageRole::User);
814 assert_eq!(session.messages[1].content(), Some("My question"));
815 }
816
817 #[tokio::test]
818 async fn test_send_stream_respects_context_capacity() {
819 let gateway = Arc::new(StreamingMockGateway::new(vec![
820 vec!["This is a longer response to consume tokens in the context window".to_string()],
821 vec!["Another longer response that also consumes many tokens in context".to_string()],
822 ]));
823 let broker = LlmBroker::new("test-model", gateway, None);
824 let mut session = ChatSession::builder(broker).max_context(50).build();
825
826 {
827 let mut stream = session.send_stream("First longer query message with extra words");
828 while stream.next().await.is_some() {}
829 }
830 {
831 let mut stream = session.send_stream("Second longer query message with extra words");
832 while stream.next().await.is_some() {}
833 }
834
835 assert_eq!(session.messages[0].role(), MessageRole::System);
837 assert!(session.total_tokens() <= 50);
839 }
840
841 #[tokio::test]
842 async fn test_message_with_no_content_has_zero_tokens() {
843 let gateway = Arc::new(MockGateway::new(vec![]));
844 let broker = LlmBroker::new("test-model", gateway, None);
845 let mut session = ChatSession::new(broker);
846
847 let message = LlmMessage {
848 role: MessageRole::Assistant,
849 content: None,
850 tool_calls: None,
851 image_paths: None,
852 reasoning_content: None,
853 };
854
855 session.insert_message(message);
856
857 assert_eq!(session.messages.len(), 2);
859 assert_eq!(session.messages[1].token_length, 0);
860 }
861}