Skip to main content

mojentic/llm/gateways/
omlx.rs

1//! Gateway for [oMLX](https://github.com/jundot/omlx), an LLM server for Apple Silicon.
2//!
3//! oMLX speaks the OpenAI chat completions protocol, but this gateway does not
4//! treat it as OpenAI. It sends every setting as configured, with no per-model
5//! parameter adaptation and no OpenAI model registry. It maps the reasoning
6//! trace oMLX reports (`reasoning_content`) to the response's `thinking`, and
7//! it sends embeddings in one request with no client-side chunking.
8//!
9//! It reuses the OpenAI message adapter and the OpenAI stream parsers.
10
11use crate::error::{MojenticError, Result};
12use crate::llm::gateway::{
13    CompletionConfig, LlmGateway, ModelContextWindow, ReasoningEffort, ResponseFormat, StreamChunk,
14};
15use crate::llm::gateways::openai::{add_response_format, openai_response_evidence};
16use crate::llm::gateways::openai_legacy_stream::{
17    legacy_body_stream, read_legacy_line, LegacyLine, OpenAiLegacyParser,
18};
19use crate::llm::gateways::openai_messages_adapter::{
20    adapt_messages_to_openai_with_reasoning, convert_tool_calls,
21};
22use crate::llm::gateways::openai_stream_events::OpenAiEventParser;
23use crate::llm::models::{LlmGatewayResponse, LlmMessage, ResponseEvidence};
24use crate::llm::stream_events::{
25    drive_event_stream, FrameParser, StreamEvent, StreamEventError, StreamEventStream,
26};
27use crate::llm::stream_timing::{system_clock, StreamClock, StreamTiming};
28use crate::llm::tools::LlmTool;
29use async_trait::async_trait;
30use futures::stream::Stream;
31use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
32use reqwest::{Client, Method, RequestBuilder, Response};
33use serde_json::Value;
34use std::pin::Pin;
35use std::sync::Arc;
36use std::time::Duration;
37use tracing::{debug, info, warn};
38
39/// Response metadata key for the `Warning` header oMLX sends when it could
40/// not enforce a requested structured output format.
41///
42/// When oMLX cannot compile a grammar for `json_object` or `json_schema`, it
43/// falls back to instructions in the prompt and says so in a `Warning`
44/// header. The gateway records the header value here and logs a warning. It
45/// does not retry or fail: callers validate content as they would anyway.
46pub const RESPONSE_FORMAT_WARNING: &str = "response_format_warning";
47
48/// Host used when neither the configuration nor `OMLX_HOST` sets one.
49const DEFAULT_HOST: &str = "http://localhost:8000";
50
51/// Timeout used when `OMLX_TIMEOUT` is unset or unreadable: ten minutes.
52const DEFAULT_TIMEOUT: Duration = Duration::from_millis(600_000);
53
54/// The `model` oMLX reports on keep-alive stream frames.
55const KEEPALIVE_MODEL: &str = "keepalive";
56
57/// Characters left as they are in a model id path segment (RFC 3986 unreserved).
58const PATH_SEGMENT: &AsciiSet =
59    &NON_ALPHANUMERIC.remove(b'-').remove(b'.').remove(b'_').remove(b'~');
60
61/// Configuration for connecting to an oMLX server.
62///
63/// [`Default`] reads the environment, so explicit fields take precedence over
64/// the environment, which takes precedence over the built-in defaults:
65///
66/// | Field | Environment | Default |
67/// | ----- | ----------- | ------- |
68/// | `host` | `OMLX_HOST` | `http://localhost:8000` |
69/// | `api_key` | `OMLX_API_KEY` | none |
70/// | `timeout` | `OMLX_TIMEOUT`, in milliseconds | 10 minutes |
71///
72/// ```no_run
73/// use mojentic::llm::gateways::{OmlxConfig, OmlxGateway};
74///
75/// let gateway = OmlxGateway::with_config(OmlxConfig {
76///     host: "http://studio.local:8000".to_string(),
77///     ..Default::default()
78/// });
79/// ```
80#[derive(Clone, PartialEq, Eq)]
81pub struct OmlxConfig {
82    /// Server address without the `/v1` suffix. The gateway adds `/v1`.
83    pub host: String,
84    /// Sent as `Authorization: Bearer <key>`. No header is sent without one.
85    pub api_key: Option<String>,
86    /// Bounds non-streaming requests, including model load.
87    /// Streams are cancelled by dropping them; a whole-response timeout
88    /// would cut off long replies.
89    /// `None` means no timeout.
90    pub timeout: Option<Duration>,
91}
92
93/// Shows whether a key is set, never the key itself.
94impl std::fmt::Debug for OmlxConfig {
95    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
96        f.debug_struct("OmlxConfig")
97            .field("host", &self.host)
98            .field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
99            .field("timeout", &self.timeout)
100            .finish()
101    }
102}
103
104impl Default for OmlxConfig {
105    fn default() -> Self {
106        Self {
107            host: std::env::var("OMLX_HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string()),
108            api_key: std::env::var("OMLX_API_KEY").ok().filter(|key| !key.is_empty()),
109            timeout: Some(timeout_from_env(std::env::var("OMLX_TIMEOUT").ok())),
110        }
111    }
112}
113
114/// Read an `OMLX_TIMEOUT` value in milliseconds, falling back to the default.
115fn timeout_from_env(value: Option<String>) -> Duration {
116    let Some(value) = value else {
117        return DEFAULT_TIMEOUT;
118    };
119    match value.trim().parse::<u64>() {
120        Ok(millis) => Duration::from_millis(millis),
121        Err(_) => {
122            warn!(value = %value, "OMLX_TIMEOUT is not a number of milliseconds, using the default");
123            DEFAULT_TIMEOUT
124        }
125    }
126}
127
128/// Gateway for a local [oMLX](https://github.com/jundot/omlx) server.
129///
130/// Supports chat, structured output, tool calls, both streaming APIs,
131/// embeddings, and loading and unloading models.
132///
133/// # Thinking
134///
135/// oMLX reports a model's reasoning trace as `reasoning_content`. The gateway
136/// maps it to [`LlmGatewayResponse::thinking`] and, in the legacy streaming
137/// API, to [`StreamChunk::Thinking`]. The events API has no thinking event
138/// and yields none.
139///
140/// `reasoning_effort` of low, medium or high is sent unchanged to the model's
141/// chat template as `"low"`, `"medium"` or `"high"`. Its effect depends on the
142/// model. [`ReasoningEffort::Disabled`] sends `enable_thinking: false` and no
143/// `reasoning_effort`, which turns thinking off. Leaving it unset keeps the
144/// model's default; Qwen 3 models think by default.
145///
146/// # Truncation
147///
148/// When `max_tokens` ends generation during thinking, a non-streaming response
149/// carries the partial reasoning in `content` and no thinking, with a finish
150/// reason of `length`. A streaming response keeps it as thinking. The gateway
151/// maps the fields as oMLX reports them. When the finish reason in
152/// [`LlmGatewayResponse::evidence`] is not `stop`, `content` is not an answer.
153///
154/// # Example
155///
156/// ```no_run
157/// use mojentic::llm::gateways::OmlxGateway;
158/// use mojentic::llm::{LlmBroker, LlmMessage};
159/// use std::sync::Arc;
160///
161/// # async fn run() -> mojentic::Result<()> {
162/// let broker = LlmBroker::new("Qwen3.8-27B-MLX-8bit", Arc::new(OmlxGateway::new()), None);
163/// let reply = broker.generate(&[LlmMessage::user("Hello")], None, None, None).await?;
164/// # Ok(())
165/// # }
166/// ```
167pub struct OmlxGateway {
168    clock: Arc<dyn StreamClock>,
169    client: Client,
170    config: OmlxConfig,
171}
172
173impl OmlxGateway {
174    /// Select a monotonic measurement clock. The default uses std::time::Instant.
175    pub fn with_stream_clock(mut self, clock: Arc<dyn StreamClock>) -> Self {
176        self.clock = clock;
177        self
178    }
179
180    /// Opt into per-request recovery without changing legacy entrypoints.
181    pub fn with_recovery(
182        self,
183        policy: crate::llm::recovery::RecoveryPolicy,
184    ) -> Result<crate::llm::recovery::RecoveryGateway<Self>> {
185        crate::llm::recovery::RecoveryGateway::new(self, policy)
186    }
187
188    /// Create a gateway configured from the environment. See [`OmlxConfig`].
189    pub fn new() -> Self {
190        Self::with_config(OmlxConfig::default())
191    }
192
193    /// Create a gateway with explicit configuration.
194    pub fn with_config(config: OmlxConfig) -> Self {
195        Self {
196            client: Client::new(),
197            clock: system_clock(),
198            config,
199        }
200    }
201
202    /// Create a gateway for `host`, reading the key and timeout from the environment.
203    pub fn with_host(host: impl Into<String>) -> Self {
204        Self::with_config(OmlxConfig {
205            host: host.into(),
206            ..Default::default()
207        })
208    }
209
210    /// Load a model into memory ahead of its first request.
211    ///
212    /// Blocks until the model is in memory. A chat request loads its model on
213    /// its own, so this is only for warming up. oMLX downloads models only
214    /// through its admin dashboard; there is no pull operation.
215    ///
216    /// # Errors
217    ///
218    /// [`MojenticError::GatewayError`] with the HTTP status and response body
219    /// when oMLX rejects the request, or [`MojenticError::HttpError`] when the
220    /// request fails.
221    pub async fn load_model(&self, model: &str) -> Result<()> {
222        info!(model, "Loading oMLX model");
223        self.model_action(model, "load").await
224    }
225
226    /// Unload a model from memory.
227    ///
228    /// # Errors
229    ///
230    /// [`MojenticError::GatewayError`] with the HTTP status and response body
231    /// when oMLX rejects the request. Unloading a model that is not loaded is
232    /// a 400 `invalid_request_error`. [`MojenticError::HttpError`] when the
233    /// request fails.
234    pub async fn unload_model(&self, model: &str) -> Result<()> {
235        info!(model, "Unloading oMLX model");
236        self.model_action(model, "unload").await
237    }
238
239    async fn model_action(&self, model: &str, action: &str) -> Result<()> {
240        if model.trim().is_empty() {
241            return Err(MojenticError::InvalidArgument(
242                "oMLX model id must not be blank".to_string(),
243            ));
244        }
245        let model = utf8_percent_encode(model, PATH_SEGMENT);
246        let response = self
247            .request(Method::POST, &format!("/models/{model}/{action}"), false)
248            .send()
249            .await?;
250        success(response).await?;
251        Ok(())
252    }
253
254    /// A request to `/v1{path}` with authorization and timeout applied.
255    fn request(&self, method: Method, path: &str, streaming: bool) -> RequestBuilder {
256        let url = format!("{}/v1{}", self.config.host.trim_end_matches('/'), path);
257        let mut request = self.client.request(method, url);
258        if let Some(key) = self.config.api_key.as_ref().filter(|key| !key.trim().is_empty()) {
259            request = request.bearer_auth(key);
260        }
261        if !streaming {
262            if let Some(timeout) = self.config.timeout {
263                request = request.timeout(timeout);
264            }
265        }
266        request
267    }
268
269    fn chat_request(&self, body: &Value) -> RequestBuilder {
270        self.request(Method::POST, "/chat/completions", body["stream"] == true)
271            .json(body)
272    }
273
274    /// Send a non-streaming chat request and read the response.
275    async fn chat(&self, body: &Value, structured: bool) -> Result<ChatReply> {
276        let response = success(self.chat_request(body).send().await?).await?;
277        let warning = structured.then(|| format_warning(&response)).flatten();
278        let body: Value = response.json().await?;
279        let mut evidence = openai_response_evidence(&body);
280        if let Some(warning) = warning {
281            warn!(warning = %warning, "oMLX did not enforce the requested response format");
282            evidence
283                .metadata
284                .insert(RESPONSE_FORMAT_WARNING.to_string(), Value::String(warning));
285        }
286        Ok(ChatReply { body, evidence })
287    }
288}
289
290impl Default for OmlxGateway {
291    fn default() -> Self {
292        Self::new()
293    }
294}
295
296/// A chat completion body and the evidence read from it and its headers.
297struct ChatReply {
298    body: Value,
299    evidence: ResponseEvidence,
300}
301
302impl ChatReply {
303    fn message(&self) -> &Value {
304        &self.body["choices"][0]["message"]
305    }
306
307    fn text(&self, field: &str) -> Option<String> {
308        self.message()[field].as_str().map(String::from)
309    }
310}
311
312/// The response when its status is a success, otherwise a gateway error
313/// carrying the status and the body as oMLX sent it.
314async fn success(response: Response) -> Result<Response> {
315    let status = response.status();
316    if status.is_success() {
317        return Ok(response);
318    }
319    let body = response.text().await.unwrap_or_default();
320    Err(MojenticError::GatewayError(format!("oMLX API error: {status} - {body}")))
321}
322
323/// Every `Warning` header value, joined with `, `.
324fn format_warning(response: &Response) -> Option<String> {
325    format_warning_headers(response.headers())
326}
327
328pub(crate) fn format_warning_headers(headers: &reqwest::header::HeaderMap) -> Option<String> {
329    let values: Vec<_> = headers
330        .get_all(reqwest::header::WARNING)
331        .iter()
332        .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned())
333        .collect();
334    (!values.is_empty()).then(|| values.join(", "))
335}
336
337/// Whether the configuration asks for structured output.
338fn requests_structured_output(config: &CompletionConfig) -> bool {
339    matches!(config.response_format, Some(ResponseFormat::JsonObject { .. }))
340}
341
342/// Build a chat completion body from the configuration, unadapted.
343///
344/// `num_ctx` and `num_predict` are not sent: oMLX sets context length per model.
345fn chat_body(model: &str, messages: &[LlmMessage], config: &CompletionConfig) -> Result<Value> {
346    config.validate_reasoning()?;
347    let mut body = serde_json::json!({
348        "model": model,
349        "messages": adapt_messages_to_openai_with_reasoning(messages, true)?,
350    });
351    if let Some(temperature) = config.temperature {
352        body["temperature"] = serde_json::json!(temperature);
353    }
354    if let Some(max_tokens) = config.max_tokens {
355        body["max_tokens"] = serde_json::json!(max_tokens);
356    }
357    if config.preserve_reasoning {
358        body["chat_template_kwargs"] = serde_json::json!({"preserve_thinking": true});
359    }
360    if let Some(top_p) = config.top_p {
361        body["top_p"] = serde_json::json!(top_p);
362    }
363    if let Some(top_k) = config.top_k {
364        body["top_k"] = serde_json::json!(top_k);
365    }
366    match config.reasoning_effort {
367        // oMLX turns thinking off with its own flag, not an effort level.
368        Some(ReasoningEffort::Disabled) => body["enable_thinking"] = Value::Bool(false),
369        Some(ReasoningEffort::Enabled) => body["enable_thinking"] = Value::Bool(true),
370        Some(effort) => body["reasoning_effort"] = serde_json::to_value(effort)?,
371        None => {}
372    }
373    add_response_format(&mut body, config);
374    Ok(body)
375}
376
377fn tool_definitions(tools: &[Box<dyn LlmTool>]) -> Result<Value> {
378    let descriptors: Vec<_> = tools.iter().map(|tool| tool.descriptor()).collect();
379    Ok(serde_json::to_value(descriptors)?)
380}
381
382/// Whether a stream frame is one of oMLX's keep-alive frames.
383fn is_keepalive(frame: &Value) -> bool {
384    frame["model"] == KEEPALIVE_MODEL
385}
386
387/// Whether an SSE line is a `data:` line carrying a keep-alive frame.
388fn is_keepalive_line(line: &str) -> bool {
389    // Most lines are not keep-alives; skip the JSON parse for them.
390    if !line.contains(KEEPALIVE_MODEL) {
391        return false;
392    }
393    line.strip_prefix("data:")
394        .and_then(|data| serde_json::from_str::<Value>(data.trim_start()).ok())
395        .is_some_and(|frame| is_keepalive(&frame))
396}
397
398/// The OpenAI event parser, behind a filter that drops keep-alive frames so
399/// their `keepalive` model never enters the completion evidence.
400#[derive(Default)]
401struct OmlxEventParser {
402    inner: OpenAiEventParser,
403}
404
405impl FrameParser for OmlxEventParser {
406    fn parse_line(&mut self, line: &str) -> Vec<StreamEvent> {
407        if is_keepalive_line(line) {
408            return Vec::new();
409        }
410        self.inner.parse_line(line)
411    }
412
413    fn partial_evidence(&self) -> Option<ResponseEvidence> {
414        self.inner.partial_evidence()
415    }
416}
417
418/// Legacy stream chunks for one line: keep-alives dropped, reasoning as thinking.
419fn legacy_chunks(parser: &mut OpenAiLegacyParser, line: &str) -> Vec<StreamChunk> {
420    match read_legacy_line(line) {
421        Some(LegacyLine::Done) => parser.done(),
422        Some(LegacyLine::Frame(frame)) if is_keepalive(&frame) => Vec::new(),
423        Some(LegacyLine::Frame(frame)) => {
424            let reasoning = frame["choices"][0]["delta"]["reasoning_content"]
425                .as_str()
426                .filter(|text| !text.is_empty())
427                .map(|text| StreamChunk::Thinking(text.to_string()));
428            reasoning.into_iter().chain(parser.frame(&frame)).collect()
429        }
430        None => Vec::new(),
431    }
432}
433
434#[async_trait]
435impl LlmGateway for OmlxGateway {
436    fn stream_clock(&self) -> Arc<dyn StreamClock> {
437        self.clock.clone()
438    }
439
440    async fn get_model_context_window(&self, model: &str) -> Result<Option<ModelContextWindow>> {
441        let response = success(self.request(Method::GET, "/models", false).send().await?).await?;
442        let body: Value = response.json().await?;
443        let models = body["data"]
444            .as_array()
445            .ok_or_else(|| MojenticError::GatewayError("Invalid response format".into()))?;
446        let Some(entry) = models.iter().find(|entry| entry["id"].as_str() == Some(model)) else {
447            return Ok(None);
448        };
449        entry
450            .get("max_model_len")
451            .map(|value| ModelContextWindow::advertised(value, "max_model_len".into()))
452            .transpose()
453    }
454
455    async fn complete(
456        &self,
457        model: &str,
458        messages: &[LlmMessage],
459        tools: Option<&[Box<dyn LlmTool>]>,
460        config: &CompletionConfig,
461    ) -> Result<LlmGatewayResponse> {
462        info!("Delegating to oMLX for completion");
463        debug!("Model: {}, Message count: {}", model, messages.len());
464
465        let mut body = chat_body(model, messages, config)?;
466        if let Some(tools) = tools {
467            body["tools"] = tool_definitions(tools)?;
468        }
469
470        let reply = self.chat(&body, requests_structured_output(config)).await?;
471        let tool_calls =
472            reply.message()["tool_calls"].as_array().map(|calls| convert_tool_calls(calls));
473
474        Ok(LlmGatewayResponse {
475            content: reply.text("content"),
476            object: None,
477            tool_calls: tool_calls.unwrap_or_default(),
478            thinking: reply.text("reasoning_content"),
479            evidence: reply.evidence,
480        })
481    }
482
483    async fn complete_json(
484        &self,
485        model: &str,
486        messages: &[LlmMessage],
487        schema: Value,
488        config: &CompletionConfig,
489    ) -> Result<Value> {
490        self.complete_json_response(model, messages, schema, config)
491            .await?
492            .object
493            .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))
494    }
495
496    async fn complete_json_response(
497        &self,
498        model: &str,
499        messages: &[LlmMessage],
500        schema: Value,
501        config: &CompletionConfig,
502    ) -> Result<LlmGatewayResponse<Value>> {
503        info!("Requesting structured output from oMLX");
504
505        let config = CompletionConfig {
506            response_format: Some(ResponseFormat::JsonObject {
507                schema: Some(schema),
508            }),
509            ..config.clone()
510        };
511        let body = chat_body(model, messages, &config)?;
512
513        let reply = self.chat(&body, true).await?;
514        let content = reply
515            .text("content")
516            .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))?;
517        let object: Value = serde_json::from_str(&content)?;
518
519        Ok(LlmGatewayResponse {
520            content: Some(content),
521            object: Some(object),
522            tool_calls: Vec::new(),
523            thinking: reply.text("reasoning_content"),
524            evidence: reply.evidence,
525        })
526    }
527
528    async fn get_available_models(&self) -> Result<Vec<String>> {
529        debug!("Fetching available oMLX models");
530
531        let response = success(self.request(Method::GET, "/models", false).send().await?).await?;
532        let body: Value = response.json().await?;
533
534        let mut models: Vec<String> = body["data"]
535            .as_array()
536            .ok_or_else(|| MojenticError::GatewayError("Invalid response format".to_string()))?
537            .iter()
538            .filter_map(|model| model["id"].as_str().map(String::from))
539            .collect();
540        models.sort();
541        Ok(models)
542    }
543
544    /// Embed `text` in one request.
545    ///
546    /// oMLX has no standard embedding model, so `model` is required. There is
547    /// no client-side chunking; the server applies its model's input limit.
548    ///
549    /// # Errors
550    ///
551    /// [`MojenticError::InvalidArgument`] without sending a request when
552    /// `model` is missing or blank. [`MojenticError::GatewayError`] with the
553    /// status and body when oMLX rejects the request, for example a 400 for a
554    /// model that is not an embedding model.
555    async fn calculate_embeddings(&self, text: &str, model: Option<&str>) -> Result<Vec<f32>> {
556        let model = model.filter(|model| !model.trim().is_empty()).ok_or_else(|| {
557            MojenticError::InvalidArgument(
558                "oMLX embeddings need a model: oMLX has no default embedding model".to_string(),
559            )
560        })?;
561        debug!("Calculating embeddings with model: {}", model);
562
563        let body = serde_json::json!({"model": model, "input": text});
564        let response =
565            success(self.request(Method::POST, "/embeddings", false).json(&body).send().await?)
566                .await?;
567        let body: Value = response.json().await?;
568
569        let embedding = body["data"][0]["embedding"]
570            .as_array()
571            .ok_or_else(|| MojenticError::GatewayError("Invalid embeddings response".to_string()))?
572            .iter()
573            .filter_map(|value| value.as_f64().map(|number| number as f32))
574            .collect();
575        Ok(embedding)
576    }
577
578    fn complete_stream<'a>(
579        &'a self,
580        model: &'a str,
581        messages: &'a [LlmMessage],
582        tools: Option<&'a [Box<dyn LlmTool>]>,
583        config: &'a CompletionConfig,
584    ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
585        let timing = StreamTiming::new(self.stream_clock());
586        let measure = timing.clone();
587        let source: Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> =
588            Box::pin(async_stream::stream! {
589                info!("Starting oMLX streaming completion");
590                debug!("Model: {}, Message count: {}", model, messages.len());
591
592                let mut body = match chat_body(model, messages, config) {
593                    Ok(body) => body,
594                    Err(e) => {
595                        yield Err(e);
596                        return;
597                    }
598                };
599                body["stream"] = serde_json::json!(true);
600                if let Some(tools) = tools {
601                    match tool_definitions(tools) {
602                        Ok(definitions) => body["tools"] = definitions,
603                        Err(e) => {
604                            yield Err(e);
605                            return;
606                        }
607                    }
608                }
609
610                measure.dispatch();
611                let response = match self.chat_request(&body).send().await {
612                    Ok(response) => response,
613                    Err(e) => {
614                        yield Err(e.into());
615                        return;
616                    }
617                };
618                let response = match success(response).await {
619                    Ok(response) => response,
620                    Err(e) => {
621                        yield Err(e);
622                        return;
623                    }
624                };
625
626                let mut parser = OpenAiLegacyParser::default();
627                let chunks = legacy_body_stream(response.bytes_stream(), move |line: &str| {
628                    measure.observe_line(line);
629                    legacy_chunks(&mut parser, line)
630                });
631                for await chunk in chunks {
632                    yield chunk;
633                }
634            });
635        timing.legacy(source, "omlx")
636    }
637
638    fn complete_stream_events<'a>(
639        &'a self,
640        model: &'a str,
641        messages: &'a [LlmMessage],
642        config: &'a CompletionConfig,
643    ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
644        info!("Starting oMLX stream events");
645        let mut body =
646            chat_body(model, messages, config).map_err(StreamEventError::RequestFailed)?;
647        body["stream"] = serde_json::json!(true);
648        body["stream_options"] = serde_json::json!({"include_usage": true});
649
650        Ok(drive_event_stream(
651            self.chat_request(&body),
652            OmlxEventParser::default(),
653            self.stream_clock(),
654            "omlx",
655        ))
656    }
657}
658
659impl crate::llm::recovery::RecoveryProvider for OmlxGateway {
660    fn recovery_provider(&self) -> crate::llm::recovery::Provider {
661        crate::llm::recovery::Provider::Omlx
662    }
663    fn prepare_recovery(
664        &self,
665        model: &str,
666        messages: &[LlmMessage],
667        tools: Option<&[Box<dyn LlmTool>]>,
668        config: &CompletionConfig,
669        schema: Option<Value>,
670        stream: bool,
671    ) -> Result<reqwest::Request> {
672        let mut config = config.clone();
673        if let Some(schema) = schema {
674            config.response_format = Some(ResponseFormat::JsonObject {
675                schema: Some(schema),
676            });
677        }
678        let mut body = chat_body(model, messages, &config)?;
679        body["stream"] = Value::Bool(stream);
680        if let Some(tools) = tools {
681            body["tools"] = tool_definitions(tools)?;
682        }
683        let mut request = self.chat_request(&body).build()?;
684        *request.timeout_mut() = None;
685        Ok(request)
686    }
687}
688
689#[cfg(test)]
690mod tests {
691    use super::*;
692    use futures::stream::StreamExt;
693
694    #[tokio::test]
695    async fn blank_model_actions_fail_before_a_request() {
696        let gateway = OmlxGateway::with_host("http://127.0.0.1:1");
697        for model in ["", "  ", "\t"] {
698            assert!(matches!(
699                gateway.load_model(model).await,
700                Err(MojenticError::InvalidArgument(_))
701            ));
702            assert!(matches!(
703                gateway.unload_model(model).await,
704                Err(MojenticError::InvalidArgument(_))
705            ));
706        }
707    }
708
709    #[tokio::test]
710    async fn event_stream_is_not_cut_off_by_the_whole_response_timeout() {
711        use crate::llm::stream_events::testing::endless_stream_server;
712        let (url, closed) =
713            endless_stream_server("data: {\"choices\":[{\"delta\":{\"content\":\"first\"}}]}\n\n")
714                .await;
715        let gateway = OmlxGateway::with_config(OmlxConfig {
716            host: url,
717            api_key: None,
718            timeout: Some(Duration::from_millis(50)),
719        });
720        let messages = [LlmMessage::user("Hi")];
721        let config = CompletionConfig::default();
722        let mut stream = gateway.complete_stream_events("m", &messages, &config).unwrap();
723        assert!(matches!(stream.next().await, Some(crate::llm::StreamEvent::Content(_))));
724        tokio::time::sleep(Duration::from_millis(100)).await;
725        assert!(tokio::time::timeout(Duration::from_millis(20), stream.next()).await.is_err());
726        drop(stream);
727        tokio::time::timeout(Duration::from_secs(2), closed).await.unwrap().unwrap();
728    }
729
730    #[tokio::test]
731    async fn legacy_stream_outlives_the_timeout_and_cancels_on_drop() {
732        use crate::llm::stream_events::testing::endless_stream_server;
733        let (url, closed) =
734            endless_stream_server("data: {\"choices\":[{\"delta\":{\"content\":\"first\"}}]}\n\n")
735                .await;
736        let gateway = OmlxGateway::with_config(OmlxConfig {
737            host: url,
738            api_key: None,
739            timeout: Some(Duration::from_millis(50)),
740        });
741        let messages = [LlmMessage::user("Hi")];
742        let config = CompletionConfig::default();
743        let mut stream = gateway.complete_stream("m", &messages, None, &config);
744        assert!(matches!(stream.next().await, Some(Ok(StreamChunk::Content(_)))));
745        tokio::time::sleep(Duration::from_millis(100)).await;
746        assert!(tokio::time::timeout(Duration::from_millis(20), stream.next()).await.is_err());
747        drop(stream);
748        tokio::time::timeout(Duration::from_secs(2), closed).await.unwrap().unwrap();
749    }
750
751    #[tokio::test]
752    async fn legacy_stream_keeps_a_character_split_across_network_chunks() {
753        use crate::llm::stream_events::testing::{
754            split_body_server, split_inside_first_multibyte_char,
755        };
756        let body = concat!(
757            "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"\u{1f914} hmm\"}}]}\n\n",
758            "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"caf\u{e9}\"}}]}\n\n",
759            "data: [DONE]\n\n",
760        );
761        let url = split_body_server(split_inside_first_multibyte_char(body)).await;
762        let gateway = OmlxGateway::with_config(OmlxConfig {
763            host: url,
764            api_key: None,
765            timeout: None,
766        });
767        let messages = vec![LlmMessage::user("Hi")];
768        let config = CompletionConfig::default();
769
770        let items: Vec<_> = gateway.complete_stream("m", &messages, None, &config).collect().await;
771
772        assert!(
773            matches!(
774                items.as_slice(),
775                [Ok(StreamChunk::Thinking(thinking)), Ok(StreamChunk::Content(content)), Ok(StreamChunk::Metrics(_))]
776                    if thinking == "\u{1f914} hmm" && content == "caf\u{e9}"
777            ),
778            "{items:?}"
779        );
780    }
781
782    #[test]
783    fn debug_output_hides_the_api_key() {
784        let config = OmlxConfig {
785            host: "http://localhost:8000".into(),
786            api_key: Some("secret-key".into()),
787            timeout: None,
788        };
789
790        let shown = format!("{config:?}");
791
792        assert!(!shown.contains("secret-key"), "{shown}");
793        assert!(shown.contains("<redacted>"), "{shown}");
794    }
795
796    #[test]
797    fn timeout_env_is_milliseconds_with_a_ten_minute_default() {
798        assert_eq!(timeout_from_env(None), Duration::from_secs(600));
799        assert_eq!(timeout_from_env(Some("2500".into())), Duration::from_millis(2500));
800        assert_eq!(timeout_from_env(Some(" 1000 ".into())), Duration::from_secs(1));
801        assert_eq!(timeout_from_env(Some("ten".into())), Duration::from_secs(600));
802        assert_eq!(timeout_from_env(Some("-5".into())), Duration::from_secs(600));
803    }
804
805    #[test]
806    fn a_keepalive_line_has_exactly_the_keepalive_model() {
807        assert!(is_keepalive_line(r#"data: {"model":"keepalive","choices":[]}"#));
808        assert!(is_keepalive_line(r#"data:{"model":"keepalive","choices":[]}"#));
809        assert!(!is_keepalive_line(r#"data: {"model":"keepalive-7b","choices":[]}"#));
810        assert!(!is_keepalive_line(
811            r#"data: {"model":"m","choices":[{"delta":{"content":"keepalive"}}]}"#
812        ));
813        assert!(!is_keepalive_line(": keepalive"));
814        assert!(!is_keepalive_line("data: [DONE]"));
815    }
816
817    #[test]
818    fn the_event_parser_ignores_keepalive_evidence() {
819        let mut parser = OmlxEventParser::default();
820
821        let events = parser.parse_line(
822            r#"data: {"id":"c","created":0,"model":"keepalive","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}"#,
823        );
824
825        assert!(events.is_empty());
826        assert_eq!(parser.partial_evidence(), None);
827    }
828
829    #[test]
830    fn reasoning_comes_before_content_in_one_delta() {
831        let mut parser = OpenAiLegacyParser::default();
832
833        let chunks = legacy_chunks(
834            &mut parser,
835            r#"data: {"model":"m","choices":[{"delta":{"reasoning_content":"why","content":"what"}}]}"#,
836        );
837
838        assert!(matches!(
839            chunks.as_slice(),
840            [StreamChunk::Thinking(why), StreamChunk::Content(what)] if why == "why" && what == "what"
841        ));
842    }
843
844    #[test]
845    fn only_json_formats_request_structured_output() {
846        let with = |format| CompletionConfig {
847            response_format: format,
848            ..Default::default()
849        };
850
851        assert!(!requests_structured_output(&with(None)));
852        assert!(!requests_structured_output(&with(Some(ResponseFormat::Text))));
853        assert!(requests_structured_output(&with(Some(ResponseFormat::JsonObject {
854            schema: None
855        }))));
856    }
857}