Skip to main content

mojentic/llm/
gateway.rs

1use crate::error::Result;
2use crate::llm::models::{LlmGatewayResponse, LlmMessage};
3use crate::llm::stream_events::{StreamEventError, StreamEventStream};
4use crate::llm::tools::LlmTool;
5use async_trait::async_trait;
6use futures::stream::Stream;
7use serde::{Deserialize, Serialize};
8use serde_json::Value;
9use std::pin::Pin;
10
11/// Format specification for LLM responses
12#[derive(Debug, Clone)]
13pub enum ResponseFormat {
14    /// Plain text response
15    Text,
16    /// JSON object response with optional schema
17    JsonObject { schema: Option<Value> },
18}
19
20/// Reasoning effort level for models that support extended thinking
21#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
22#[serde(rename_all = "lowercase")]
23pub enum ReasoningEffort {
24    /// Disable provider-native extended thinking when the provider supports it.
25    Disabled,
26    /// Enable reasoning using the model default (for boolean-control models).
27    Enabled,
28    Low,
29    Medium,
30    High,
31    /// Extra-high reasoning effort; distinct from high.
32    XHigh,
33}
34
35/// Configuration for LLM completion.
36///
37/// Provider controls default to `None`: the provider applies its own settings.
38/// To restore the former 2.1 defaults explicitly:
39///
40/// ```
41/// use mojentic::llm::gateway::CompletionConfig;
42/// let config = CompletionConfig {
43///     temperature: Some(1.0),
44///     num_ctx: Some(32768),
45///     max_tokens: Some(16384),
46///     ..Default::default()
47/// };
48/// assert_eq!(config.max_tokens, Some(16384));
49/// ```
50#[derive(Debug, Clone)]
51pub struct CompletionConfig {
52    /// Sampling temperature. Unset delegates to provider settings.
53    pub temperature: Option<f32>,
54    /// Ollama context size. Other maintained providers omit this control.
55    pub num_ctx: Option<usize>,
56    /// Output limit: Ollama `options.num_predict`, OpenAI model-specific token
57    /// limit, or oMLX `max_tokens`. Unset sends no output limit.
58    pub max_tokens: Option<usize>,
59    pub top_p: Option<f32>,
60    pub top_k: Option<u32>,
61    pub response_format: Option<ResponseFormat>,
62    pub reasoning_effort: Option<ReasoningEffort>,
63    /// Explicit model capabilities. None delegates support/defaults to the provider.
64    pub reasoning_profile: Option<crate::llm::reasoning::ReasoningProfile>,
65    /// Retain completed assistant reasoning during broker-managed tool rounds.
66    /// This does not turn reasoning on, and does not retain cancelled partial calls.
67    pub preserve_reasoning: bool,
68    /// Maximum tool rounds; use `with_unlimited_tool_iterations` to disable the limit.
69    pub max_tool_iterations: usize,
70}
71
72impl CompletionConfig {
73    /// Validate explicit model capabilities before sending a provider request.
74    pub fn validate_reasoning(&self) -> Result<()> {
75        if let Some(profile) = &self.reasoning_profile {
76            profile.validate(self.reasoning_effort, self.preserve_reasoning)?;
77        }
78        Ok(())
79    }
80
81    /// Disable the tool-round limit while retaining the existing numeric configuration API.
82    /// `usize::MAX` is a symbolic unlimited value; the broker skips its limit check.
83    pub fn with_unlimited_tool_iterations(mut self) -> Self {
84        self.max_tool_iterations = usize::MAX;
85        self
86    }
87
88    pub(crate) fn tool_iteration_limit_reached(&self, iteration: usize) -> bool {
89        self.max_tool_iterations != usize::MAX && iteration >= self.max_tool_iterations
90    }
91}
92
93impl Default for CompletionConfig {
94    fn default() -> Self {
95        Self {
96            temperature: None,
97            num_ctx: None,
98            max_tokens: None,
99            top_p: None,
100            top_k: None,
101            response_format: None,
102            reasoning_effort: None,
103            reasoning_profile: None,
104            preserve_reasoning: false,
105            max_tool_iterations: 10,
106        }
107    }
108}
109
110/// Provider-advertised context capacity; independent of completion configuration.
111#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
112pub struct ModelContextWindow {
113    /// Positive provider maximum, in tokens.
114    pub max_context_tokens: std::num::NonZeroUsize,
115    /// Provider field path, or `registry` for OpenAI.
116    pub field: String,
117    /// Ollama model parameter used when the request omits `num_ctx`.
118    pub model_num_ctx: Option<usize>,
119}
120
121impl ModelContextWindow {
122    pub(crate) fn advertised(value: &Value, field: String) -> Result<Self> {
123        let tokens = value
124            .as_u64()
125            .and_then(|n| usize::try_from(n).ok())
126            .and_then(std::num::NonZeroUsize::new)
127            .ok_or_else(|| {
128                crate::error::MojenticError::GatewayError(format!(
129                    "Invalid positive integer at {field}"
130                ))
131            })?;
132        Ok(Self {
133            max_context_tokens: tokens,
134            field,
135            model_num_ctx: None,
136        })
137    }
138}
139
140/// Abstract interface for LLM providers
141#[async_trait]
142pub trait LlmGateway: Send + Sync {
143    /// Monotonic clock used by maintained streaming adapters.
144    fn stream_clock(&self) -> std::sync::Arc<dyn crate::llm::stream_timing::StreamClock> {
145        crate::llm::stream_timing::system_clock()
146    }
147
148    /// Query advertised context capacity without changing subsequent requests.
149    ///
150    /// Third-party gateways default to not advertised.
151    ///
152    /// # Errors
153    /// Returns the gateway's existing transport, status or malformed-body error.
154    async fn get_model_context_window(&self, _model: &str) -> Result<Option<ModelContextWindow>> {
155        Ok(None)
156    }
157
158    /// Complete an LLM request with text response
159    async fn complete(
160        &self,
161        model: &str,
162        messages: &[LlmMessage],
163        tools: Option<&[Box<dyn LlmTool>]>,
164        config: &CompletionConfig,
165    ) -> Result<LlmGatewayResponse>;
166
167    /// Complete an LLM request with structured JSON response
168    async fn complete_json(
169        &self,
170        model: &str,
171        messages: &[LlmMessage],
172        schema: Value,
173        config: &CompletionConfig,
174    ) -> Result<Value>;
175
176    /// Complete a structured JSON request and keep the provider's evidence.
177    ///
178    /// The response carries the parsed object, the raw content when the
179    /// gateway has it, and [`crate::llm::models::ResponseEvidence`]. The
180    /// default implementation wraps [`LlmGateway::complete_json`] and reports
181    /// no evidence; gateways that see provider metadata override it.
182    async fn complete_json_response(
183        &self,
184        model: &str,
185        messages: &[LlmMessage],
186        schema: Value,
187        config: &CompletionConfig,
188    ) -> Result<LlmGatewayResponse<Value>> {
189        let object = self.complete_json(model, messages, schema, config).await?;
190        Ok(LlmGatewayResponse {
191            object: Some(object),
192            ..Default::default()
193        })
194    }
195
196    /// Get list of available models
197    async fn get_available_models(&self) -> Result<Vec<String>>;
198
199    /// Calculate embeddings for text
200    async fn calculate_embeddings(&self, text: &str, model: Option<&str>) -> Result<Vec<f32>>;
201
202    /// Stream LLM responses chunk by chunk
203    ///
204    /// Returns a stream that yields either content chunks or tool calls.
205    /// Tool calls will be accumulated and yielded when complete.
206    fn complete_stream<'a>(
207        &'a self,
208        model: &'a str,
209        messages: &'a [LlmMessage],
210        tools: Option<&'a [Box<dyn LlmTool>]>,
211        config: &'a CompletionConfig,
212    ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>;
213
214    /// Stream one turn as [`crate::llm::StreamEvent`]s with terminal completion evidence.
215    ///
216    /// No tools are sent. The returned stream sends exactly one request when
217    /// first polled and ends with exactly one terminal event. Dropping it
218    /// cancels the request.
219    ///
220    /// # Errors
221    ///
222    /// Returns [`StreamEventError::StreamEventsUnsupported`] when the gateway
223    /// does not implement the events API (the default), or
224    /// [`StreamEventError::RequestFailed`] when the request cannot be built.
225    /// No request is sent in either case. The broker turns either into a
226    /// single terminal error event.
227    fn complete_stream_events<'a>(
228        &'a self,
229        _model: &'a str,
230        _messages: &'a [LlmMessage],
231        _config: &'a CompletionConfig,
232    ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
233        Err(StreamEventError::StreamEventsUnsupported)
234    }
235}
236
237/// Streaming response chunk
238#[derive(Debug, Clone)]
239pub enum StreamChunk {
240    /// Content text chunk
241    Content(String),
242    /// Reasoning/thinking text chunk that providers expose separately from assistant content.
243    Thinking(String),
244    /// Complete tool calls (accumulated from stream)
245    ToolCalls(Vec<crate::llm::models::LlmToolCall>),
246    /// Provider-neutral heartbeat/progress for a streamed provider frame.
247    Progress(StreamProgress),
248    /// Provider-neutral final generation metrics when the provider exposes them.
249    Metrics(StreamMetrics),
250}
251
252/// Provider-neutral progress for a streamed provider frame.
253#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
254pub struct StreamProgress {
255    pub provider: String,
256    pub frame_index: usize,
257    pub done: bool,
258    pub content_chars: usize,
259    pub thinking_chars: usize,
260    pub tool_call_count: usize,
261    pub accumulated_tool_call_count: usize,
262}
263
264/// Provider-neutral final generation metrics.
265#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
266pub struct StreamMetrics {
267    /// Client milliseconds from first dispatch to first semantic token.
268    pub time_to_first_token_ms: Option<u64>,
269    /// First semantic kind: thinking, content, or tool_call.
270    pub first_token_kind: Option<String>,
271    /// Client milliseconds from first dispatch to terminal success or failure.
272    pub request_duration_ms: Option<u64>,
273    /// Provider usage and durations in their original shape and units.
274    pub provider_usage: Option<Value>,
275    pub provider: String,
276    pub total_duration_ns: Option<u64>,
277    pub load_duration_ns: Option<u64>,
278    pub prompt_eval_count: Option<u64>,
279    pub prompt_eval_duration_ns: Option<u64>,
280    pub eval_count: Option<u64>,
281    pub eval_duration_ns: Option<u64>,
282    pub tokens_per_second: Option<f64>,
283}
284
285#[cfg(test)]
286mod tests {
287    use super::*;
288
289    #[test]
290    fn test_completion_config_default() {
291        let config = CompletionConfig::default();
292
293        assert_eq!(config.temperature, None);
294        assert_eq!(config.num_ctx, None);
295        assert_eq!(config.max_tokens, None);
296        assert_eq!(config.top_p, None);
297        assert_eq!(config.top_k, None);
298        assert!(config.response_format.is_none());
299        assert!(config.reasoning_effort.is_none());
300        assert_eq!(config.max_tool_iterations, 10);
301    }
302
303    #[test]
304    fn test_completion_config_custom() {
305        let config = CompletionConfig {
306            temperature: Some(0.5),
307            num_ctx: Some(2048),
308            max_tokens: Some(1024),
309            top_p: Some(0.9),
310            top_k: Some(40),
311            response_format: Some(ResponseFormat::Text),
312            reasoning_effort: None,
313            reasoning_profile: None,
314            preserve_reasoning: false,
315            max_tool_iterations: 10,
316        };
317
318        assert_eq!(config.temperature, Some(0.5));
319        assert_eq!(config.num_ctx, Some(2048));
320        assert_eq!(config.max_tokens, Some(1024));
321        assert_eq!(config.top_p, Some(0.9));
322        assert_eq!(config.top_k, Some(40));
323        assert!(matches!(config.response_format, Some(ResponseFormat::Text)));
324    }
325
326    #[test]
327    fn test_completion_config_clone() {
328        let config1 = CompletionConfig {
329            temperature: Some(0.7),
330            num_ctx: Some(4096),
331            max_tokens: Some(2048),
332            top_p: Some(0.95),
333            top_k: Some(50),
334            response_format: Some(ResponseFormat::JsonObject { schema: None }),
335            reasoning_effort: None,
336            reasoning_profile: None,
337            preserve_reasoning: false,
338            max_tool_iterations: 10,
339        };
340
341        let config2 = config1.clone();
342
343        assert_eq!(config1.temperature, config2.temperature);
344        assert_eq!(config1.num_ctx, config2.num_ctx);
345        assert_eq!(config1.max_tokens, config2.max_tokens);
346        assert_eq!(config1.top_p, config2.top_p);
347        assert_eq!(config1.top_k, config2.top_k);
348    }
349
350    #[test]
351    fn test_response_format_text() {
352        let format = ResponseFormat::Text;
353        assert!(matches!(format, ResponseFormat::Text));
354    }
355
356    #[test]
357    fn test_response_format_json_no_schema() {
358        let format = ResponseFormat::JsonObject { schema: None };
359        assert!(matches!(format, ResponseFormat::JsonObject { schema: None }));
360    }
361
362    #[test]
363    fn test_response_format_json_with_schema() {
364        let schema = serde_json::json!({
365            "type": "object",
366            "properties": {
367                "name": {"type": "string"}
368            }
369        });
370        let format = ResponseFormat::JsonObject {
371            schema: Some(schema.clone()),
372        };
373
374        match format {
375            ResponseFormat::JsonObject { schema: Some(s) } => {
376                assert_eq!(s, schema);
377            }
378            _ => panic!("Expected JsonObject with schema"),
379        }
380    }
381
382    #[test]
383    fn test_completion_config_with_all_sampling_params() {
384        let config = CompletionConfig {
385            temperature: Some(0.8),
386            num_ctx: Some(8192),
387            max_tokens: Some(4096),
388            top_p: Some(0.92),
389            top_k: Some(60),
390            response_format: Some(ResponseFormat::JsonObject {
391                schema: Some(serde_json::json!({"type": "object"})),
392            }),
393            reasoning_effort: None,
394            reasoning_profile: None,
395            preserve_reasoning: false,
396            max_tool_iterations: 10,
397        };
398
399        assert_eq!(config.temperature, Some(0.8));
400        assert_eq!(config.top_p, Some(0.92));
401        assert_eq!(config.top_k, Some(60));
402        assert!(config.response_format.is_some());
403    }
404
405    #[test]
406    fn test_reasoning_effort_serialization() {
407        assert_eq!(serde_json::to_string(&ReasoningEffort::Disabled).unwrap(), "\"disabled\"");
408        assert_eq!(serde_json::to_string(&ReasoningEffort::Low).unwrap(), "\"low\"");
409        assert_eq!(serde_json::to_string(&ReasoningEffort::Medium).unwrap(), "\"medium\"");
410        assert_eq!(serde_json::to_string(&ReasoningEffort::High).unwrap(), "\"high\"");
411    }
412
413    #[test]
414    fn test_reasoning_effort_deserialization() {
415        assert_eq!(
416            serde_json::from_str::<ReasoningEffort>("\"disabled\"").unwrap(),
417            ReasoningEffort::Disabled
418        );
419        assert_eq!(
420            serde_json::from_str::<ReasoningEffort>("\"low\"").unwrap(),
421            ReasoningEffort::Low
422        );
423        assert_eq!(
424            serde_json::from_str::<ReasoningEffort>("\"medium\"").unwrap(),
425            ReasoningEffort::Medium
426        );
427        assert_eq!(
428            serde_json::from_str::<ReasoningEffort>("\"high\"").unwrap(),
429            ReasoningEffort::High
430        );
431    }
432
433    #[test]
434    fn test_completion_config_with_reasoning_effort() {
435        let config = CompletionConfig {
436            temperature: Some(1.0),
437            num_ctx: Some(32768),
438            max_tokens: Some(16384),
439            top_p: None,
440            top_k: None,
441            response_format: None,
442            reasoning_effort: Some(ReasoningEffort::High),
443            reasoning_profile: None,
444            preserve_reasoning: false,
445            max_tool_iterations: 10,
446        };
447
448        assert_eq!(config.reasoning_effort, Some(ReasoningEffort::High));
449    }
450}