1use crate::error::{MojenticError, Result};
12use crate::llm::gateway::{
13 CompletionConfig, LlmGateway, ModelContextWindow, ReasoningEffort, ResponseFormat, StreamChunk,
14};
15use crate::llm::gateways::openai::{add_response_format, openai_response_evidence};
16use crate::llm::gateways::openai_legacy_stream::{
17 legacy_body_stream, read_legacy_line, LegacyLine, OpenAiLegacyParser,
18};
19use crate::llm::gateways::openai_messages_adapter::{
20 adapt_messages_to_openai_with_reasoning, convert_tool_calls,
21};
22use crate::llm::gateways::openai_stream_events::OpenAiEventParser;
23use crate::llm::models::{LlmGatewayResponse, LlmMessage, ResponseEvidence};
24use crate::llm::stream_events::{
25 drive_event_stream, FrameParser, StreamEvent, StreamEventError, StreamEventStream,
26};
27use crate::llm::stream_timing::{system_clock, StreamClock, StreamTiming};
28use crate::llm::tools::LlmTool;
29use async_trait::async_trait;
30use futures::stream::Stream;
31use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
32use reqwest::{Client, Method, RequestBuilder, Response};
33use serde_json::Value;
34use std::pin::Pin;
35use std::sync::Arc;
36use std::time::Duration;
37use tracing::{debug, info, warn};
38
39pub const RESPONSE_FORMAT_WARNING: &str = "response_format_warning";
47
48const DEFAULT_HOST: &str = "http://localhost:8000";
50
51const DEFAULT_TIMEOUT: Duration = Duration::from_millis(600_000);
53
54const KEEPALIVE_MODEL: &str = "keepalive";
56
57const PATH_SEGMENT: &AsciiSet =
59 &NON_ALPHANUMERIC.remove(b'-').remove(b'.').remove(b'_').remove(b'~');
60
61#[derive(Clone, PartialEq, Eq)]
81pub struct OmlxConfig {
82 pub host: String,
84 pub api_key: Option<String>,
86 pub timeout: Option<Duration>,
91}
92
93impl std::fmt::Debug for OmlxConfig {
95 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
96 f.debug_struct("OmlxConfig")
97 .field("host", &self.host)
98 .field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
99 .field("timeout", &self.timeout)
100 .finish()
101 }
102}
103
104impl Default for OmlxConfig {
105 fn default() -> Self {
106 Self {
107 host: std::env::var("OMLX_HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string()),
108 api_key: std::env::var("OMLX_API_KEY").ok().filter(|key| !key.is_empty()),
109 timeout: Some(timeout_from_env(std::env::var("OMLX_TIMEOUT").ok())),
110 }
111 }
112}
113
114fn timeout_from_env(value: Option<String>) -> Duration {
116 let Some(value) = value else {
117 return DEFAULT_TIMEOUT;
118 };
119 match value.trim().parse::<u64>() {
120 Ok(millis) => Duration::from_millis(millis),
121 Err(_) => {
122 warn!(value = %value, "OMLX_TIMEOUT is not a number of milliseconds, using the default");
123 DEFAULT_TIMEOUT
124 }
125 }
126}
127
128pub struct OmlxGateway {
168 clock: Arc<dyn StreamClock>,
169 client: Client,
170 config: OmlxConfig,
171}
172
173impl OmlxGateway {
174 pub fn with_stream_clock(mut self, clock: Arc<dyn StreamClock>) -> Self {
176 self.clock = clock;
177 self
178 }
179
180 pub fn with_recovery(
182 self,
183 policy: crate::llm::recovery::RecoveryPolicy,
184 ) -> Result<crate::llm::recovery::RecoveryGateway<Self>> {
185 crate::llm::recovery::RecoveryGateway::new(self, policy)
186 }
187
188 pub fn new() -> Self {
190 Self::with_config(OmlxConfig::default())
191 }
192
193 pub fn with_config(config: OmlxConfig) -> Self {
195 Self {
196 client: Client::new(),
197 clock: system_clock(),
198 config,
199 }
200 }
201
202 pub fn with_host(host: impl Into<String>) -> Self {
204 Self::with_config(OmlxConfig {
205 host: host.into(),
206 ..Default::default()
207 })
208 }
209
210 pub async fn load_model(&self, model: &str) -> Result<()> {
222 info!(model, "Loading oMLX model");
223 self.model_action(model, "load").await
224 }
225
226 pub async fn unload_model(&self, model: &str) -> Result<()> {
235 info!(model, "Unloading oMLX model");
236 self.model_action(model, "unload").await
237 }
238
239 async fn model_action(&self, model: &str, action: &str) -> Result<()> {
240 if model.trim().is_empty() {
241 return Err(MojenticError::InvalidArgument(
242 "oMLX model id must not be blank".to_string(),
243 ));
244 }
245 let model = utf8_percent_encode(model, PATH_SEGMENT);
246 let response = self
247 .request(Method::POST, &format!("/models/{model}/{action}"), false)
248 .send()
249 .await?;
250 success(response).await?;
251 Ok(())
252 }
253
254 fn request(&self, method: Method, path: &str, streaming: bool) -> RequestBuilder {
256 let url = format!("{}/v1{}", self.config.host.trim_end_matches('/'), path);
257 let mut request = self.client.request(method, url);
258 if let Some(key) = self.config.api_key.as_ref().filter(|key| !key.trim().is_empty()) {
259 request = request.bearer_auth(key);
260 }
261 if !streaming {
262 if let Some(timeout) = self.config.timeout {
263 request = request.timeout(timeout);
264 }
265 }
266 request
267 }
268
269 fn chat_request(&self, body: &Value) -> RequestBuilder {
270 self.request(Method::POST, "/chat/completions", body["stream"] == true)
271 .json(body)
272 }
273
274 async fn chat(&self, body: &Value, structured: bool) -> Result<ChatReply> {
276 let response = success(self.chat_request(body).send().await?).await?;
277 let warning = structured.then(|| format_warning(&response)).flatten();
278 let body: Value = response.json().await?;
279 let mut evidence = openai_response_evidence(&body);
280 if let Some(warning) = warning {
281 warn!(warning = %warning, "oMLX did not enforce the requested response format");
282 evidence
283 .metadata
284 .insert(RESPONSE_FORMAT_WARNING.to_string(), Value::String(warning));
285 }
286 Ok(ChatReply { body, evidence })
287 }
288}
289
290impl Default for OmlxGateway {
291 fn default() -> Self {
292 Self::new()
293 }
294}
295
296struct ChatReply {
298 body: Value,
299 evidence: ResponseEvidence,
300}
301
302impl ChatReply {
303 fn message(&self) -> &Value {
304 &self.body["choices"][0]["message"]
305 }
306
307 fn text(&self, field: &str) -> Option<String> {
308 self.message()[field].as_str().map(String::from)
309 }
310}
311
312async fn success(response: Response) -> Result<Response> {
315 let status = response.status();
316 if status.is_success() {
317 return Ok(response);
318 }
319 let body = response.text().await.unwrap_or_default();
320 Err(MojenticError::GatewayError(format!("oMLX API error: {status} - {body}")))
321}
322
323fn format_warning(response: &Response) -> Option<String> {
325 format_warning_headers(response.headers())
326}
327
328pub(crate) fn format_warning_headers(headers: &reqwest::header::HeaderMap) -> Option<String> {
329 let values: Vec<_> = headers
330 .get_all(reqwest::header::WARNING)
331 .iter()
332 .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned())
333 .collect();
334 (!values.is_empty()).then(|| values.join(", "))
335}
336
337fn requests_structured_output(config: &CompletionConfig) -> bool {
339 matches!(config.response_format, Some(ResponseFormat::JsonObject { .. }))
340}
341
342fn chat_body(model: &str, messages: &[LlmMessage], config: &CompletionConfig) -> Result<Value> {
346 config.validate_reasoning()?;
347 let mut body = serde_json::json!({
348 "model": model,
349 "messages": adapt_messages_to_openai_with_reasoning(messages, true)?,
350 });
351 if let Some(temperature) = config.temperature {
352 body["temperature"] = serde_json::json!(temperature);
353 }
354 if let Some(max_tokens) = config.max_tokens {
355 body["max_tokens"] = serde_json::json!(max_tokens);
356 }
357 if config.preserve_reasoning {
358 body["chat_template_kwargs"] = serde_json::json!({"preserve_thinking": true});
359 }
360 if let Some(top_p) = config.top_p {
361 body["top_p"] = serde_json::json!(top_p);
362 }
363 if let Some(top_k) = config.top_k {
364 body["top_k"] = serde_json::json!(top_k);
365 }
366 match config.reasoning_effort {
367 Some(ReasoningEffort::Disabled) => body["enable_thinking"] = Value::Bool(false),
369 Some(ReasoningEffort::Enabled) => body["enable_thinking"] = Value::Bool(true),
370 Some(effort) => body["reasoning_effort"] = serde_json::to_value(effort)?,
371 None => {}
372 }
373 add_response_format(&mut body, config);
374 Ok(body)
375}
376
377fn tool_definitions(tools: &[Box<dyn LlmTool>]) -> Result<Value> {
378 let descriptors: Vec<_> = tools.iter().map(|tool| tool.descriptor()).collect();
379 Ok(serde_json::to_value(descriptors)?)
380}
381
382fn is_keepalive(frame: &Value) -> bool {
384 frame["model"] == KEEPALIVE_MODEL
385}
386
387fn is_keepalive_line(line: &str) -> bool {
389 if !line.contains(KEEPALIVE_MODEL) {
391 return false;
392 }
393 line.strip_prefix("data:")
394 .and_then(|data| serde_json::from_str::<Value>(data.trim_start()).ok())
395 .is_some_and(|frame| is_keepalive(&frame))
396}
397
398#[derive(Default)]
401struct OmlxEventParser {
402 inner: OpenAiEventParser,
403}
404
405impl FrameParser for OmlxEventParser {
406 fn parse_line(&mut self, line: &str) -> Vec<StreamEvent> {
407 if is_keepalive_line(line) {
408 return Vec::new();
409 }
410 self.inner.parse_line(line)
411 }
412
413 fn partial_evidence(&self) -> Option<ResponseEvidence> {
414 self.inner.partial_evidence()
415 }
416}
417
418fn legacy_chunks(parser: &mut OpenAiLegacyParser, line: &str) -> Vec<StreamChunk> {
420 match read_legacy_line(line) {
421 Some(LegacyLine::Done) => parser.done(),
422 Some(LegacyLine::Frame(frame)) if is_keepalive(&frame) => Vec::new(),
423 Some(LegacyLine::Frame(frame)) => {
424 let reasoning = frame["choices"][0]["delta"]["reasoning_content"]
425 .as_str()
426 .filter(|text| !text.is_empty())
427 .map(|text| StreamChunk::Thinking(text.to_string()));
428 reasoning.into_iter().chain(parser.frame(&frame)).collect()
429 }
430 None => Vec::new(),
431 }
432}
433
434#[async_trait]
435impl LlmGateway for OmlxGateway {
436 fn stream_clock(&self) -> Arc<dyn StreamClock> {
437 self.clock.clone()
438 }
439
440 async fn get_model_context_window(&self, model: &str) -> Result<Option<ModelContextWindow>> {
441 let response = success(self.request(Method::GET, "/models", false).send().await?).await?;
442 let body: Value = response.json().await?;
443 let models = body["data"]
444 .as_array()
445 .ok_or_else(|| MojenticError::GatewayError("Invalid response format".into()))?;
446 let Some(entry) = models.iter().find(|entry| entry["id"].as_str() == Some(model)) else {
447 return Ok(None);
448 };
449 entry
450 .get("max_model_len")
451 .map(|value| ModelContextWindow::advertised(value, "max_model_len".into()))
452 .transpose()
453 }
454
455 async fn complete(
456 &self,
457 model: &str,
458 messages: &[LlmMessage],
459 tools: Option<&[Box<dyn LlmTool>]>,
460 config: &CompletionConfig,
461 ) -> Result<LlmGatewayResponse> {
462 info!("Delegating to oMLX for completion");
463 debug!("Model: {}, Message count: {}", model, messages.len());
464
465 let mut body = chat_body(model, messages, config)?;
466 if let Some(tools) = tools {
467 body["tools"] = tool_definitions(tools)?;
468 }
469
470 let reply = self.chat(&body, requests_structured_output(config)).await?;
471 let tool_calls =
472 reply.message()["tool_calls"].as_array().map(|calls| convert_tool_calls(calls));
473
474 Ok(LlmGatewayResponse {
475 content: reply.text("content"),
476 object: None,
477 tool_calls: tool_calls.unwrap_or_default(),
478 thinking: reply.text("reasoning_content"),
479 evidence: reply.evidence,
480 })
481 }
482
483 async fn complete_json(
484 &self,
485 model: &str,
486 messages: &[LlmMessage],
487 schema: Value,
488 config: &CompletionConfig,
489 ) -> Result<Value> {
490 self.complete_json_response(model, messages, schema, config)
491 .await?
492 .object
493 .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))
494 }
495
496 async fn complete_json_response(
497 &self,
498 model: &str,
499 messages: &[LlmMessage],
500 schema: Value,
501 config: &CompletionConfig,
502 ) -> Result<LlmGatewayResponse<Value>> {
503 info!("Requesting structured output from oMLX");
504
505 let config = CompletionConfig {
506 response_format: Some(ResponseFormat::JsonObject {
507 schema: Some(schema),
508 }),
509 ..config.clone()
510 };
511 let body = chat_body(model, messages, &config)?;
512
513 let reply = self.chat(&body, true).await?;
514 let content = reply
515 .text("content")
516 .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))?;
517 let object: Value = serde_json::from_str(&content)?;
518
519 Ok(LlmGatewayResponse {
520 content: Some(content),
521 object: Some(object),
522 tool_calls: Vec::new(),
523 thinking: reply.text("reasoning_content"),
524 evidence: reply.evidence,
525 })
526 }
527
528 async fn get_available_models(&self) -> Result<Vec<String>> {
529 debug!("Fetching available oMLX models");
530
531 let response = success(self.request(Method::GET, "/models", false).send().await?).await?;
532 let body: Value = response.json().await?;
533
534 let mut models: Vec<String> = body["data"]
535 .as_array()
536 .ok_or_else(|| MojenticError::GatewayError("Invalid response format".to_string()))?
537 .iter()
538 .filter_map(|model| model["id"].as_str().map(String::from))
539 .collect();
540 models.sort();
541 Ok(models)
542 }
543
544 async fn calculate_embeddings(&self, text: &str, model: Option<&str>) -> Result<Vec<f32>> {
556 let model = model.filter(|model| !model.trim().is_empty()).ok_or_else(|| {
557 MojenticError::InvalidArgument(
558 "oMLX embeddings need a model: oMLX has no default embedding model".to_string(),
559 )
560 })?;
561 debug!("Calculating embeddings with model: {}", model);
562
563 let body = serde_json::json!({"model": model, "input": text});
564 let response =
565 success(self.request(Method::POST, "/embeddings", false).json(&body).send().await?)
566 .await?;
567 let body: Value = response.json().await?;
568
569 let embedding = body["data"][0]["embedding"]
570 .as_array()
571 .ok_or_else(|| MojenticError::GatewayError("Invalid embeddings response".to_string()))?
572 .iter()
573 .filter_map(|value| value.as_f64().map(|number| number as f32))
574 .collect();
575 Ok(embedding)
576 }
577
578 fn complete_stream<'a>(
579 &'a self,
580 model: &'a str,
581 messages: &'a [LlmMessage],
582 tools: Option<&'a [Box<dyn LlmTool>]>,
583 config: &'a CompletionConfig,
584 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
585 let timing = StreamTiming::new(self.stream_clock());
586 let measure = timing.clone();
587 let source: Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> =
588 Box::pin(async_stream::stream! {
589 info!("Starting oMLX streaming completion");
590 debug!("Model: {}, Message count: {}", model, messages.len());
591
592 let mut body = match chat_body(model, messages, config) {
593 Ok(body) => body,
594 Err(e) => {
595 yield Err(e);
596 return;
597 }
598 };
599 body["stream"] = serde_json::json!(true);
600 if let Some(tools) = tools {
601 match tool_definitions(tools) {
602 Ok(definitions) => body["tools"] = definitions,
603 Err(e) => {
604 yield Err(e);
605 return;
606 }
607 }
608 }
609
610 measure.dispatch();
611 let response = match self.chat_request(&body).send().await {
612 Ok(response) => response,
613 Err(e) => {
614 yield Err(e.into());
615 return;
616 }
617 };
618 let response = match success(response).await {
619 Ok(response) => response,
620 Err(e) => {
621 yield Err(e);
622 return;
623 }
624 };
625
626 let mut parser = OpenAiLegacyParser::default();
627 let chunks = legacy_body_stream(response.bytes_stream(), move |line: &str| {
628 measure.observe_line(line);
629 legacy_chunks(&mut parser, line)
630 });
631 for await chunk in chunks {
632 yield chunk;
633 }
634 });
635 timing.legacy(source, "omlx")
636 }
637
638 fn complete_stream_events<'a>(
639 &'a self,
640 model: &'a str,
641 messages: &'a [LlmMessage],
642 config: &'a CompletionConfig,
643 ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
644 info!("Starting oMLX stream events");
645 let mut body =
646 chat_body(model, messages, config).map_err(StreamEventError::RequestFailed)?;
647 body["stream"] = serde_json::json!(true);
648 body["stream_options"] = serde_json::json!({"include_usage": true});
649
650 Ok(drive_event_stream(
651 self.chat_request(&body),
652 OmlxEventParser::default(),
653 self.stream_clock(),
654 "omlx",
655 ))
656 }
657}
658
659impl crate::llm::recovery::RecoveryProvider for OmlxGateway {
660 fn recovery_provider(&self) -> crate::llm::recovery::Provider {
661 crate::llm::recovery::Provider::Omlx
662 }
663 fn prepare_recovery(
664 &self,
665 model: &str,
666 messages: &[LlmMessage],
667 tools: Option<&[Box<dyn LlmTool>]>,
668 config: &CompletionConfig,
669 schema: Option<Value>,
670 stream: bool,
671 ) -> Result<reqwest::Request> {
672 let mut config = config.clone();
673 if let Some(schema) = schema {
674 config.response_format = Some(ResponseFormat::JsonObject {
675 schema: Some(schema),
676 });
677 }
678 let mut body = chat_body(model, messages, &config)?;
679 body["stream"] = Value::Bool(stream);
680 if let Some(tools) = tools {
681 body["tools"] = tool_definitions(tools)?;
682 }
683 let mut request = self.chat_request(&body).build()?;
684 *request.timeout_mut() = None;
685 Ok(request)
686 }
687}
688
689#[cfg(test)]
690mod tests {
691 use super::*;
692 use futures::stream::StreamExt;
693
694 #[tokio::test]
695 async fn blank_model_actions_fail_before_a_request() {
696 let gateway = OmlxGateway::with_host("http://127.0.0.1:1");
697 for model in ["", " ", "\t"] {
698 assert!(matches!(
699 gateway.load_model(model).await,
700 Err(MojenticError::InvalidArgument(_))
701 ));
702 assert!(matches!(
703 gateway.unload_model(model).await,
704 Err(MojenticError::InvalidArgument(_))
705 ));
706 }
707 }
708
709 #[tokio::test]
710 async fn event_stream_is_not_cut_off_by_the_whole_response_timeout() {
711 use crate::llm::stream_events::testing::endless_stream_server;
712 let (url, closed) =
713 endless_stream_server("data: {\"choices\":[{\"delta\":{\"content\":\"first\"}}]}\n\n")
714 .await;
715 let gateway = OmlxGateway::with_config(OmlxConfig {
716 host: url,
717 api_key: None,
718 timeout: Some(Duration::from_millis(50)),
719 });
720 let messages = [LlmMessage::user("Hi")];
721 let config = CompletionConfig::default();
722 let mut stream = gateway.complete_stream_events("m", &messages, &config).unwrap();
723 assert!(matches!(stream.next().await, Some(crate::llm::StreamEvent::Content(_))));
724 tokio::time::sleep(Duration::from_millis(100)).await;
725 assert!(tokio::time::timeout(Duration::from_millis(20), stream.next()).await.is_err());
726 drop(stream);
727 tokio::time::timeout(Duration::from_secs(2), closed).await.unwrap().unwrap();
728 }
729
730 #[tokio::test]
731 async fn legacy_stream_outlives_the_timeout_and_cancels_on_drop() {
732 use crate::llm::stream_events::testing::endless_stream_server;
733 let (url, closed) =
734 endless_stream_server("data: {\"choices\":[{\"delta\":{\"content\":\"first\"}}]}\n\n")
735 .await;
736 let gateway = OmlxGateway::with_config(OmlxConfig {
737 host: url,
738 api_key: None,
739 timeout: Some(Duration::from_millis(50)),
740 });
741 let messages = [LlmMessage::user("Hi")];
742 let config = CompletionConfig::default();
743 let mut stream = gateway.complete_stream("m", &messages, None, &config);
744 assert!(matches!(stream.next().await, Some(Ok(StreamChunk::Content(_)))));
745 tokio::time::sleep(Duration::from_millis(100)).await;
746 assert!(tokio::time::timeout(Duration::from_millis(20), stream.next()).await.is_err());
747 drop(stream);
748 tokio::time::timeout(Duration::from_secs(2), closed).await.unwrap().unwrap();
749 }
750
751 #[tokio::test]
752 async fn legacy_stream_keeps_a_character_split_across_network_chunks() {
753 use crate::llm::stream_events::testing::{
754 split_body_server, split_inside_first_multibyte_char,
755 };
756 let body = concat!(
757 "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"\u{1f914} hmm\"}}]}\n\n",
758 "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"caf\u{e9}\"}}]}\n\n",
759 "data: [DONE]\n\n",
760 );
761 let url = split_body_server(split_inside_first_multibyte_char(body)).await;
762 let gateway = OmlxGateway::with_config(OmlxConfig {
763 host: url,
764 api_key: None,
765 timeout: None,
766 });
767 let messages = vec![LlmMessage::user("Hi")];
768 let config = CompletionConfig::default();
769
770 let items: Vec<_> = gateway.complete_stream("m", &messages, None, &config).collect().await;
771
772 assert!(
773 matches!(
774 items.as_slice(),
775 [Ok(StreamChunk::Thinking(thinking)), Ok(StreamChunk::Content(content)), Ok(StreamChunk::Metrics(_))]
776 if thinking == "\u{1f914} hmm" && content == "caf\u{e9}"
777 ),
778 "{items:?}"
779 );
780 }
781
782 #[test]
783 fn debug_output_hides_the_api_key() {
784 let config = OmlxConfig {
785 host: "http://localhost:8000".into(),
786 api_key: Some("secret-key".into()),
787 timeout: None,
788 };
789
790 let shown = format!("{config:?}");
791
792 assert!(!shown.contains("secret-key"), "{shown}");
793 assert!(shown.contains("<redacted>"), "{shown}");
794 }
795
796 #[test]
797 fn timeout_env_is_milliseconds_with_a_ten_minute_default() {
798 assert_eq!(timeout_from_env(None), Duration::from_secs(600));
799 assert_eq!(timeout_from_env(Some("2500".into())), Duration::from_millis(2500));
800 assert_eq!(timeout_from_env(Some(" 1000 ".into())), Duration::from_secs(1));
801 assert_eq!(timeout_from_env(Some("ten".into())), Duration::from_secs(600));
802 assert_eq!(timeout_from_env(Some("-5".into())), Duration::from_secs(600));
803 }
804
805 #[test]
806 fn a_keepalive_line_has_exactly_the_keepalive_model() {
807 assert!(is_keepalive_line(r#"data: {"model":"keepalive","choices":[]}"#));
808 assert!(is_keepalive_line(r#"data:{"model":"keepalive","choices":[]}"#));
809 assert!(!is_keepalive_line(r#"data: {"model":"keepalive-7b","choices":[]}"#));
810 assert!(!is_keepalive_line(
811 r#"data: {"model":"m","choices":[{"delta":{"content":"keepalive"}}]}"#
812 ));
813 assert!(!is_keepalive_line(": keepalive"));
814 assert!(!is_keepalive_line("data: [DONE]"));
815 }
816
817 #[test]
818 fn the_event_parser_ignores_keepalive_evidence() {
819 let mut parser = OmlxEventParser::default();
820
821 let events = parser.parse_line(
822 r#"data: {"id":"c","created":0,"model":"keepalive","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}"#,
823 );
824
825 assert!(events.is_empty());
826 assert_eq!(parser.partial_evidence(), None);
827 }
828
829 #[test]
830 fn reasoning_comes_before_content_in_one_delta() {
831 let mut parser = OpenAiLegacyParser::default();
832
833 let chunks = legacy_chunks(
834 &mut parser,
835 r#"data: {"model":"m","choices":[{"delta":{"reasoning_content":"why","content":"what"}}]}"#,
836 );
837
838 assert!(matches!(
839 chunks.as_slice(),
840 [StreamChunk::Thinking(why), StreamChunk::Content(what)] if why == "why" && what == "what"
841 ));
842 }
843
844 #[test]
845 fn only_json_formats_request_structured_output() {
846 let with = |format| CompletionConfig {
847 response_format: format,
848 ..Default::default()
849 };
850
851 assert!(!requests_structured_output(&with(None)));
852 assert!(!requests_structured_output(&with(Some(ResponseFormat::Text))));
853 assert!(requests_structured_output(&with(Some(ResponseFormat::JsonObject {
854 schema: None
855 }))));
856 }
857}