1use crate::error::Result;
2use crate::llm::models::{LlmGatewayResponse, LlmMessage};
3use crate::llm::stream_events::{StreamEventError, StreamEventStream};
4use crate::llm::tools::LlmTool;
5use async_trait::async_trait;
6use futures::stream::Stream;
7use serde::{Deserialize, Serialize};
8use serde_json::Value;
9use std::pin::Pin;
10
11#[derive(Debug, Clone)]
13pub enum ResponseFormat {
14 Text,
16 JsonObject { schema: Option<Value> },
18}
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
22#[serde(rename_all = "lowercase")]
23pub enum ReasoningEffort {
24 Disabled,
26 Enabled,
28 Low,
29 Medium,
30 High,
31 XHigh,
33}
34
35#[derive(Debug, Clone)]
51pub struct CompletionConfig {
52 pub temperature: Option<f32>,
54 pub num_ctx: Option<usize>,
56 pub max_tokens: Option<usize>,
59 pub top_p: Option<f32>,
60 pub top_k: Option<u32>,
61 pub response_format: Option<ResponseFormat>,
62 pub reasoning_effort: Option<ReasoningEffort>,
63 pub reasoning_profile: Option<crate::llm::reasoning::ReasoningProfile>,
65 pub preserve_reasoning: bool,
68 pub max_tool_iterations: usize,
70}
71
72impl CompletionConfig {
73 pub fn validate_reasoning(&self) -> Result<()> {
75 if let Some(profile) = &self.reasoning_profile {
76 profile.validate(self.reasoning_effort, self.preserve_reasoning)?;
77 }
78 Ok(())
79 }
80
81 pub fn with_unlimited_tool_iterations(mut self) -> Self {
84 self.max_tool_iterations = usize::MAX;
85 self
86 }
87
88 pub(crate) fn tool_iteration_limit_reached(&self, iteration: usize) -> bool {
89 self.max_tool_iterations != usize::MAX && iteration >= self.max_tool_iterations
90 }
91}
92
93impl Default for CompletionConfig {
94 fn default() -> Self {
95 Self {
96 temperature: None,
97 num_ctx: None,
98 max_tokens: None,
99 top_p: None,
100 top_k: None,
101 response_format: None,
102 reasoning_effort: None,
103 reasoning_profile: None,
104 preserve_reasoning: false,
105 max_tool_iterations: 10,
106 }
107 }
108}
109
110#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
112pub struct ModelContextWindow {
113 pub max_context_tokens: std::num::NonZeroUsize,
115 pub field: String,
117 pub model_num_ctx: Option<usize>,
119}
120
121impl ModelContextWindow {
122 pub(crate) fn advertised(value: &Value, field: String) -> Result<Self> {
123 let tokens = value
124 .as_u64()
125 .and_then(|n| usize::try_from(n).ok())
126 .and_then(std::num::NonZeroUsize::new)
127 .ok_or_else(|| {
128 crate::error::MojenticError::GatewayError(format!(
129 "Invalid positive integer at {field}"
130 ))
131 })?;
132 Ok(Self {
133 max_context_tokens: tokens,
134 field,
135 model_num_ctx: None,
136 })
137 }
138}
139
140#[async_trait]
142pub trait LlmGateway: Send + Sync {
143 fn stream_clock(&self) -> std::sync::Arc<dyn crate::llm::stream_timing::StreamClock> {
145 crate::llm::stream_timing::system_clock()
146 }
147
148 async fn get_model_context_window(&self, _model: &str) -> Result<Option<ModelContextWindow>> {
155 Ok(None)
156 }
157
158 async fn complete(
160 &self,
161 model: &str,
162 messages: &[LlmMessage],
163 tools: Option<&[Box<dyn LlmTool>]>,
164 config: &CompletionConfig,
165 ) -> Result<LlmGatewayResponse>;
166
167 async fn complete_json(
169 &self,
170 model: &str,
171 messages: &[LlmMessage],
172 schema: Value,
173 config: &CompletionConfig,
174 ) -> Result<Value>;
175
176 async fn complete_json_response(
183 &self,
184 model: &str,
185 messages: &[LlmMessage],
186 schema: Value,
187 config: &CompletionConfig,
188 ) -> Result<LlmGatewayResponse<Value>> {
189 let object = self.complete_json(model, messages, schema, config).await?;
190 Ok(LlmGatewayResponse {
191 object: Some(object),
192 ..Default::default()
193 })
194 }
195
196 async fn get_available_models(&self) -> Result<Vec<String>>;
198
199 async fn calculate_embeddings(&self, text: &str, model: Option<&str>) -> Result<Vec<f32>>;
201
202 fn complete_stream<'a>(
207 &'a self,
208 model: &'a str,
209 messages: &'a [LlmMessage],
210 tools: Option<&'a [Box<dyn LlmTool>]>,
211 config: &'a CompletionConfig,
212 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>;
213
214 fn complete_stream_events<'a>(
228 &'a self,
229 _model: &'a str,
230 _messages: &'a [LlmMessage],
231 _config: &'a CompletionConfig,
232 ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
233 Err(StreamEventError::StreamEventsUnsupported)
234 }
235}
236
237#[derive(Debug, Clone)]
239pub enum StreamChunk {
240 Content(String),
242 Thinking(String),
244 ToolCalls(Vec<crate::llm::models::LlmToolCall>),
246 Progress(StreamProgress),
248 Metrics(StreamMetrics),
250}
251
252#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
254pub struct StreamProgress {
255 pub provider: String,
256 pub frame_index: usize,
257 pub done: bool,
258 pub content_chars: usize,
259 pub thinking_chars: usize,
260 pub tool_call_count: usize,
261 pub accumulated_tool_call_count: usize,
262}
263
264#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
266pub struct StreamMetrics {
267 pub time_to_first_token_ms: Option<u64>,
269 pub first_token_kind: Option<String>,
271 pub request_duration_ms: Option<u64>,
273 pub provider_usage: Option<Value>,
275 pub provider: String,
276 pub total_duration_ns: Option<u64>,
277 pub load_duration_ns: Option<u64>,
278 pub prompt_eval_count: Option<u64>,
279 pub prompt_eval_duration_ns: Option<u64>,
280 pub eval_count: Option<u64>,
281 pub eval_duration_ns: Option<u64>,
282 pub tokens_per_second: Option<f64>,
283}
284
285#[cfg(test)]
286mod tests {
287 use super::*;
288
289 #[test]
290 fn test_completion_config_default() {
291 let config = CompletionConfig::default();
292
293 assert_eq!(config.temperature, None);
294 assert_eq!(config.num_ctx, None);
295 assert_eq!(config.max_tokens, None);
296 assert_eq!(config.top_p, None);
297 assert_eq!(config.top_k, None);
298 assert!(config.response_format.is_none());
299 assert!(config.reasoning_effort.is_none());
300 assert_eq!(config.max_tool_iterations, 10);
301 }
302
303 #[test]
304 fn test_completion_config_custom() {
305 let config = CompletionConfig {
306 temperature: Some(0.5),
307 num_ctx: Some(2048),
308 max_tokens: Some(1024),
309 top_p: Some(0.9),
310 top_k: Some(40),
311 response_format: Some(ResponseFormat::Text),
312 reasoning_effort: None,
313 reasoning_profile: None,
314 preserve_reasoning: false,
315 max_tool_iterations: 10,
316 };
317
318 assert_eq!(config.temperature, Some(0.5));
319 assert_eq!(config.num_ctx, Some(2048));
320 assert_eq!(config.max_tokens, Some(1024));
321 assert_eq!(config.top_p, Some(0.9));
322 assert_eq!(config.top_k, Some(40));
323 assert!(matches!(config.response_format, Some(ResponseFormat::Text)));
324 }
325
326 #[test]
327 fn test_completion_config_clone() {
328 let config1 = CompletionConfig {
329 temperature: Some(0.7),
330 num_ctx: Some(4096),
331 max_tokens: Some(2048),
332 top_p: Some(0.95),
333 top_k: Some(50),
334 response_format: Some(ResponseFormat::JsonObject { schema: None }),
335 reasoning_effort: None,
336 reasoning_profile: None,
337 preserve_reasoning: false,
338 max_tool_iterations: 10,
339 };
340
341 let config2 = config1.clone();
342
343 assert_eq!(config1.temperature, config2.temperature);
344 assert_eq!(config1.num_ctx, config2.num_ctx);
345 assert_eq!(config1.max_tokens, config2.max_tokens);
346 assert_eq!(config1.top_p, config2.top_p);
347 assert_eq!(config1.top_k, config2.top_k);
348 }
349
350 #[test]
351 fn test_response_format_text() {
352 let format = ResponseFormat::Text;
353 assert!(matches!(format, ResponseFormat::Text));
354 }
355
356 #[test]
357 fn test_response_format_json_no_schema() {
358 let format = ResponseFormat::JsonObject { schema: None };
359 assert!(matches!(format, ResponseFormat::JsonObject { schema: None }));
360 }
361
362 #[test]
363 fn test_response_format_json_with_schema() {
364 let schema = serde_json::json!({
365 "type": "object",
366 "properties": {
367 "name": {"type": "string"}
368 }
369 });
370 let format = ResponseFormat::JsonObject {
371 schema: Some(schema.clone()),
372 };
373
374 match format {
375 ResponseFormat::JsonObject { schema: Some(s) } => {
376 assert_eq!(s, schema);
377 }
378 _ => panic!("Expected JsonObject with schema"),
379 }
380 }
381
382 #[test]
383 fn test_completion_config_with_all_sampling_params() {
384 let config = CompletionConfig {
385 temperature: Some(0.8),
386 num_ctx: Some(8192),
387 max_tokens: Some(4096),
388 top_p: Some(0.92),
389 top_k: Some(60),
390 response_format: Some(ResponseFormat::JsonObject {
391 schema: Some(serde_json::json!({"type": "object"})),
392 }),
393 reasoning_effort: None,
394 reasoning_profile: None,
395 preserve_reasoning: false,
396 max_tool_iterations: 10,
397 };
398
399 assert_eq!(config.temperature, Some(0.8));
400 assert_eq!(config.top_p, Some(0.92));
401 assert_eq!(config.top_k, Some(60));
402 assert!(config.response_format.is_some());
403 }
404
405 #[test]
406 fn test_reasoning_effort_serialization() {
407 assert_eq!(serde_json::to_string(&ReasoningEffort::Disabled).unwrap(), "\"disabled\"");
408 assert_eq!(serde_json::to_string(&ReasoningEffort::Low).unwrap(), "\"low\"");
409 assert_eq!(serde_json::to_string(&ReasoningEffort::Medium).unwrap(), "\"medium\"");
410 assert_eq!(serde_json::to_string(&ReasoningEffort::High).unwrap(), "\"high\"");
411 }
412
413 #[test]
414 fn test_reasoning_effort_deserialization() {
415 assert_eq!(
416 serde_json::from_str::<ReasoningEffort>("\"disabled\"").unwrap(),
417 ReasoningEffort::Disabled
418 );
419 assert_eq!(
420 serde_json::from_str::<ReasoningEffort>("\"low\"").unwrap(),
421 ReasoningEffort::Low
422 );
423 assert_eq!(
424 serde_json::from_str::<ReasoningEffort>("\"medium\"").unwrap(),
425 ReasoningEffort::Medium
426 );
427 assert_eq!(
428 serde_json::from_str::<ReasoningEffort>("\"high\"").unwrap(),
429 ReasoningEffort::High
430 );
431 }
432
433 #[test]
434 fn test_completion_config_with_reasoning_effort() {
435 let config = CompletionConfig {
436 temperature: Some(1.0),
437 num_ctx: Some(32768),
438 max_tokens: Some(16384),
439 top_p: None,
440 top_k: None,
441 response_format: None,
442 reasoning_effort: Some(ReasoningEffort::High),
443 reasoning_profile: None,
444 preserve_reasoning: false,
445 max_tool_iterations: 10,
446 };
447
448 assert_eq!(config.reasoning_effort, Some(ReasoningEffort::High));
449 }
450}