1use crate::error::{MojenticError, Result};
7use crate::llm::gateway::{
8 CompletionConfig, LlmGateway, ModelContextWindow, ResponseFormat, StreamChunk,
9};
10use crate::llm::gateways::openai_legacy_stream::{legacy_body_stream, OpenAiLegacyParser};
11use crate::llm::gateways::openai_messages_adapter::{adapt_messages_to_openai, convert_tool_calls};
12use crate::llm::gateways::openai_model_registry::{get_model_registry, ModelType};
13use crate::llm::gateways::openai_stream_events::OpenAiEventParser;
14use crate::llm::models::{LlmGatewayResponse, LlmMessage, ResponseEvidence};
15use crate::llm::stream_events::{drive_event_stream, StreamEventError, StreamEventStream};
16use crate::llm::stream_timing::{system_clock, StreamClock, StreamTiming};
17use crate::llm::tools::LlmTool;
18use async_trait::async_trait;
19use futures::stream::Stream;
20use reqwest::Client;
21use serde_json::Value;
22use std::collections::HashMap;
23use std::pin::Pin;
24use std::sync::Arc;
25use tracing::{debug, info, warn};
26
27#[derive(Debug, Clone)]
29pub struct OpenAIConfig {
30 pub api_key: String,
31 pub base_url: String,
32 pub timeout: Option<std::time::Duration>,
33}
34
35impl Default for OpenAIConfig {
36 fn default() -> Self {
37 Self {
38 api_key: std::env::var("OPENAI_API_KEY").unwrap_or_default(),
39 base_url: std::env::var("OPENAI_API_ENDPOINT")
40 .unwrap_or_else(|_| "https://api.openai.com/v1".to_string()),
41 timeout: None,
42 }
43 }
44}
45
46pub struct OpenAIGateway {
51 clock: Arc<dyn StreamClock>,
52 client: Client,
53 config: OpenAIConfig,
54}
55
56impl OpenAIGateway {
57 pub fn with_stream_clock(mut self, clock: Arc<dyn StreamClock>) -> Self {
59 self.clock = clock;
60 self
61 }
62
63 pub fn with_recovery(
65 self,
66 policy: crate::llm::recovery::RecoveryPolicy,
67 ) -> Result<crate::llm::recovery::RecoveryGateway<Self>> {
68 crate::llm::recovery::RecoveryGateway::new(self, policy)
69 }
70
71 pub fn new() -> Self {
73 Self::with_config(OpenAIConfig::default())
74 }
75
76 pub fn with_config(config: OpenAIConfig) -> Self {
78 let mut client_builder = Client::builder();
79
80 if let Some(timeout) = config.timeout {
81 client_builder = client_builder.timeout(timeout);
82 }
83
84 let client = client_builder.build().unwrap();
85
86 Self {
87 client,
88 config,
89 clock: system_clock(),
90 }
91 }
92
93 pub fn with_api_key(api_key: impl Into<String>) -> Self {
95 Self::with_config(OpenAIConfig {
96 api_key: api_key.into(),
97 ..Default::default()
98 })
99 }
100
101 pub fn with_api_key_and_base_url(
103 api_key: impl Into<String>,
104 base_url: impl Into<String>,
105 ) -> Self {
106 Self::with_config(OpenAIConfig {
107 api_key: api_key.into(),
108 base_url: base_url.into(),
109 ..Default::default()
110 })
111 }
112
113 fn adapt_parameters_for_model(
115 &self,
116 model: &str,
117 config: &CompletionConfig,
118 ) -> (HashMap<String, Value>, bool) {
119 let registry = get_model_registry();
120 let capabilities = registry.get_model_capabilities(model);
121
122 let mut params = HashMap::new();
123
124 debug!(
125 model = model,
126 model_type = ?capabilities.model_type,
127 supports_tools = capabilities.supports_tools,
128 supports_streaming = capabilities.supports_streaming,
129 "Adapting parameters for model"
130 );
131
132 if let Some(max_tokens) = config.max_tokens {
133 params.insert(
134 capabilities.get_token_limit_param().to_string(),
135 serde_json::json!(max_tokens),
136 );
137 }
138
139 if let Some(temperature) = config.temperature {
140 if capabilities.supports_temperature(temperature) {
141 params.insert("temperature".to_string(), serde_json::json!(temperature));
142 } else if let Some(allowed) = &capabilities.supported_temperatures {
143 if let [fixed] = allowed.as_slice() {
144 warn!(
145 model,
146 requested_temperature = temperature,
147 fixed_temperature = fixed,
148 "Model requires a fixed temperature"
149 );
150 params.insert("temperature".to_string(), serde_json::json!(fixed));
151 } else {
152 warn!(
153 model,
154 requested_temperature = temperature,
155 "Model does not support temperature parameter"
156 );
157 }
158 }
159 }
160
161 if let Some(top_p) = config.top_p {
163 params.insert("top_p".to_string(), serde_json::json!(top_p));
164 }
165
166 if let Some(effort) = config.reasoning_effort {
167 if capabilities.model_type != ModelType::Reasoning {
168 warn!(
169 model,
170 "reasoning_effort specified but model is not a reasoning model, ignoring"
171 );
172 } else if !matches!(
173 effort,
174 crate::llm::gateway::ReasoningEffort::Disabled
175 | crate::llm::gateway::ReasoningEffort::Enabled
176 ) {
177 params.insert("reasoning_effort".to_string(), serde_json::json!(effort));
178 }
179 }
180
181 (params, capabilities.supports_tools)
182 }
183
184 fn chat_body(
189 &self,
190 model: &str,
191 messages: &[LlmMessage],
192 config: &CompletionConfig,
193 ) -> Result<(Value, bool)> {
194 validate_reasoning_config(config)?;
195 let openai_messages = adapt_messages_to_openai(messages)?;
196 let (adapted_params, supports_tools) = self.adapt_parameters_for_model(model, config);
197
198 let mut body = serde_json::json!({
199 "model": model,
200 "messages": openai_messages,
201 });
202 for (key, value) in adapted_params {
203 body[key] = value;
204 }
205 add_response_format(&mut body, config);
206
207 Ok((body, supports_tools))
208 }
209
210 fn chat_request(&self, body: &Value) -> reqwest::RequestBuilder {
212 self.client
213 .post(format!("{}/chat/completions", self.config.base_url))
214 .header("Authorization", format!("Bearer {}", self.config.api_key))
215 .header("Content-Type", "application/json")
216 .json(body)
217 }
218
219 fn chunk_text(&self, text: &str, chunk_size: usize) -> Result<Vec<Vec<u32>>> {
221 let tokenizer = tiktoken_rs::cl100k_base()
222 .map_err(|error| MojenticError::GatewayError(format!("Tokenizer error: {error}")))?;
223 let tokens = tokenizer.encode_ordinary(text);
224 if tokens.is_empty() {
225 return Ok(vec![vec![]]);
226 }
227 Ok(tokens.chunks(chunk_size).map(<[u32]>::to_vec).collect())
228 }
229
230 fn weighted_average_embeddings(&self, embeddings: &[Vec<f32>], weights: &[f32]) -> Vec<f32> {
232 if embeddings.is_empty() {
233 return vec![];
234 }
235
236 let dimension = embeddings[0].len();
237 let total_weight: f32 = weights.iter().sum();
238
239 let average: Vec<f32> = (0..dimension)
241 .map(|dim_idx| {
242 embeddings
243 .iter()
244 .zip(weights.iter())
245 .map(|(embedding, &weight)| {
246 embedding.get(dim_idx).unwrap_or(&0.0) * (weight / total_weight)
247 })
248 .sum()
249 })
250 .collect();
251
252 let norm: f32 = average.iter().map(|x| x * x).sum::<f32>().sqrt();
254 if norm > 0.0 {
255 average.iter().map(|x| x / norm).collect()
256 } else {
257 average
258 }
259 }
260}
261
262impl Default for OpenAIGateway {
263 fn default() -> Self {
264 Self::new()
265 }
266}
267
268#[async_trait]
269impl LlmGateway for OpenAIGateway {
270 fn stream_clock(&self) -> Arc<dyn StreamClock> {
271 self.clock.clone()
272 }
273
274 async fn get_model_context_window(&self, model: &str) -> Result<Option<ModelContextWindow>> {
275 get_model_registry()
276 .registered_context_window(model)
277 .map(|tokens| {
278 ModelContextWindow::advertised(&serde_json::json!(tokens), "registry".into())
279 })
280 .transpose()
281 }
282
283 async fn complete(
284 &self,
285 model: &str,
286 messages: &[LlmMessage],
287 tools: Option<&[Box<dyn LlmTool>]>,
288 config: &CompletionConfig,
289 ) -> Result<LlmGatewayResponse> {
290 info!("Delegating to OpenAI for completion");
291 debug!("Model: {}, Message count: {}", model, messages.len());
292
293 let (mut body, supports_tools) = self.chat_body(model, messages, config)?;
294
295 if let Some(tools) = tools {
297 if supports_tools {
298 let tool_defs: Vec<_> = tools.iter().map(|t| t.descriptor()).collect();
299 body["tools"] = serde_json::to_value(tool_defs)?;
300 } else {
301 warn!(model = model, "Model does not support tools, ignoring tool configuration");
302 }
303 }
304
305 let response = self.chat_request(&body).send().await?;
307
308 if !response.status().is_success() {
309 let status = response.status();
310 let error_text = response.text().await.unwrap_or_default();
311 return Err(MojenticError::GatewayError(format!(
312 "OpenAI API error: {} - {}",
313 status, error_text
314 )));
315 }
316
317 let response_body: Value = response.json().await?;
318
319 let content = response_body["choices"][0]["message"]["content"].as_str().map(String::from);
321
322 let tool_calls =
324 if let Some(calls) = response_body["choices"][0]["message"]["tool_calls"].as_array() {
325 convert_tool_calls(calls)
326 } else {
327 vec![]
328 };
329
330 Ok(LlmGatewayResponse {
331 content,
332 object: None,
333 tool_calls,
334 thinking: None,
335 evidence: openai_response_evidence(&response_body),
336 })
337 }
338
339 async fn complete_json(
340 &self,
341 model: &str,
342 messages: &[LlmMessage],
343 schema: Value,
344 config: &CompletionConfig,
345 ) -> Result<Value> {
346 self.complete_json_response(model, messages, schema, config)
347 .await?
348 .object
349 .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))
350 }
351
352 async fn complete_json_response(
353 &self,
354 model: &str,
355 messages: &[LlmMessage],
356 schema: Value,
357 config: &CompletionConfig,
358 ) -> Result<LlmGatewayResponse<Value>> {
359 info!("Requesting structured output from OpenAI");
360
361 validate_reasoning_config(config)?;
362 let openai_messages = adapt_messages_to_openai(messages)?;
363 let (adapted_params, _) = self.adapt_parameters_for_model(model, config);
364
365 let mut body = serde_json::json!({
366 "model": model,
367 "messages": openai_messages,
368 "response_format": openai_response_format(&ResponseFormat::JsonObject {
369 schema: Some(schema)
370 }),
371 });
372
373 for (key, value) in adapted_params {
375 body[key] = value;
376 }
377
378 let response = self.chat_request(&body).send().await?;
379
380 if !response.status().is_success() {
381 let status = response.status();
382 let error_text = response.text().await.unwrap_or_default();
383 return Err(MojenticError::GatewayError(format!(
384 "OpenAI API error: {} - {}",
385 status, error_text
386 )));
387 }
388
389 let response_body: Value = response.json().await?;
390 let content = response_body["choices"][0]["message"]["content"]
391 .as_str()
392 .ok_or_else(|| MojenticError::GatewayError("No content in response".to_string()))?;
393
394 let object: Value = serde_json::from_str(content)?;
396
397 Ok(LlmGatewayResponse {
398 content: Some(content.to_string()),
399 object: Some(object),
400 evidence: openai_response_evidence(&response_body),
401 ..Default::default()
402 })
403 }
404
405 async fn get_available_models(&self) -> Result<Vec<String>> {
406 debug!("Fetching available OpenAI models");
407
408 let response = self
409 .client
410 .get(format!("{}/models", self.config.base_url))
411 .header("Authorization", format!("Bearer {}", self.config.api_key))
412 .send()
413 .await?;
414
415 if !response.status().is_success() {
416 return Err(MojenticError::GatewayError(format!(
417 "Failed to get models: {}",
418 response.status()
419 )));
420 }
421
422 let body: Value = response.json().await?;
423
424 let mut models = body["data"]
425 .as_array()
426 .ok_or_else(|| MojenticError::GatewayError("Invalid response format".to_string()))?
427 .iter()
428 .filter_map(|m| m["id"].as_str().map(String::from))
429 .collect::<Vec<_>>();
430
431 models.sort();
432 Ok(models)
433 }
434
435 async fn calculate_embeddings(&self, text: &str, model: Option<&str>) -> Result<Vec<f32>> {
436 let model = model.unwrap_or("text-embedding-3-large");
437 debug!("Calculating embeddings with model: {}", model);
438
439 let chunks = self.chunk_text(text, 8191)?;
441
442 if chunks.is_empty() {
443 return Ok(vec![]);
444 }
445
446 let mut all_embeddings = Vec::new();
447 let mut weights = Vec::new();
448
449 for chunk in &chunks {
450 let body = serde_json::json!({
451 "model": model,
452 "input": if chunks.len() == 1 { serde_json::json!(text) } else { serde_json::json!(chunk) }
453 });
454
455 let response = self
456 .client
457 .post(format!("{}/embeddings", self.config.base_url))
458 .header("Authorization", format!("Bearer {}", self.config.api_key))
459 .header("Content-Type", "application/json")
460 .json(&body)
461 .send()
462 .await?;
463
464 if !response.status().is_success() {
465 return Err(MojenticError::GatewayError(format!(
466 "Embeddings API error: {}",
467 response.status()
468 )));
469 }
470
471 let response_body: Value = response.json().await?;
472
473 let embedding: Vec<f32> = response_body["data"][0]["embedding"]
474 .as_array()
475 .ok_or_else(|| {
476 MojenticError::GatewayError("Invalid embeddings response".to_string())
477 })?
478 .iter()
479 .filter_map(|v| v.as_f64().map(|f| f as f32))
480 .collect();
481
482 weights.push(chunk.len() as f32);
483 all_embeddings.push(embedding);
484 }
485
486 if all_embeddings.len() == 1 {
488 return Ok(all_embeddings.remove(0));
489 }
490
491 Ok(self.weighted_average_embeddings(&all_embeddings, &weights))
493 }
494
495 fn complete_stream<'a>(
496 &'a self,
497 model: &'a str,
498 messages: &'a [LlmMessage],
499 tools: Option<&'a [Box<dyn LlmTool>]>,
500 config: &'a CompletionConfig,
501 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
502 let timing = StreamTiming::new(self.stream_clock());
503 let measure = timing.clone();
504 let source: Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> =
505 Box::pin(async_stream::stream! {
506 info!("Starting OpenAI streaming completion");
507 debug!("Model: {}, Message count: {}", model, messages.len());
508
509 let registry = get_model_registry();
511 let capabilities = registry.get_model_capabilities(model);
512 if !capabilities.supports_streaming {
513 yield Err(MojenticError::GatewayError(format!(
514 "Model {} does not support streaming",
515 model
516 )));
517 return;
518 }
519
520 let (mut body, supports_tools) = match self.chat_body(model, messages, config) {
521 Ok(built) => built,
522 Err(e) => {
523 yield Err(e);
524 return;
525 }
526 };
527 body["stream"] = serde_json::json!(true);
528
529 if let Some(tools) = tools {
531 if supports_tools {
532 let tool_defs: Vec<_> = tools.iter().map(|t| t.descriptor()).collect();
533 if let Ok(tools_value) = serde_json::to_value(tool_defs) {
534 body["tools"] = tools_value;
535 }
536 }
537 }
538
539 measure.dispatch();
541 let response = match self.chat_request(&body).send().await {
542 Ok(r) => r,
543 Err(e) => {
544 yield Err(e.into());
545 return;
546 }
547 };
548
549 if !response.status().is_success() {
550 yield Err(MojenticError::GatewayError(format!(
551 "OpenAI API error: {}",
552 response.status()
553 )));
554 return;
555 }
556
557 let mut parser = OpenAiLegacyParser::default();
558 let chunks = legacy_body_stream(response.bytes_stream(), move |line: &str| {
559 measure.observe_line(line);
560 parser.parse_line(line)
561 });
562 for await chunk in chunks {
563 yield chunk;
564 }
565 });
566 timing.legacy(source, "openai")
567 }
568
569 fn complete_stream_events<'a>(
570 &'a self,
571 model: &'a str,
572 messages: &'a [LlmMessage],
573 config: &'a CompletionConfig,
574 ) -> std::result::Result<StreamEventStream<'a>, StreamEventError> {
575 info!("Starting OpenAI stream events");
576 let (mut body, _) = self
577 .chat_body(model, messages, config)
578 .map_err(StreamEventError::RequestFailed)?;
579 body["stream"] = serde_json::json!(true);
580 body["stream_options"] = serde_json::json!({"include_usage": true});
581
582 Ok(drive_event_stream(
583 self.chat_request(&body),
584 OpenAiEventParser::default(),
585 self.stream_clock(),
586 "openai",
587 ))
588 }
589}
590
591const OPENAI_METADATA_FIELDS: [&str; 4] = ["id", "created", "system_fingerprint", "service_tier"];
593
594pub(crate) fn openai_response_evidence(body: &Value) -> ResponseEvidence {
596 ResponseEvidence {
597 usage: present(&body["usage"]),
598 provider_model: body["model"].as_str().map(String::from),
599 finish_reason: body["choices"][0]["finish_reason"].as_str().map(String::from),
600 metadata: openai_metadata(body),
601 }
602}
603
604pub(crate) fn openai_metadata(body: &Value) -> HashMap<String, Value> {
606 OPENAI_METADATA_FIELDS
607 .iter()
608 .filter_map(|field| present(&body[*field]).map(|value| (field.to_string(), value)))
609 .collect()
610}
611
612pub(crate) fn present(value: &Value) -> Option<Value> {
614 (!value.is_null()).then(|| value.clone())
615}
616
617pub(crate) fn openai_response_format(format: &ResponseFormat) -> Value {
619 match format {
620 ResponseFormat::Text => serde_json::json!({"type": "text"}),
621 ResponseFormat::JsonObject { schema: None } => serde_json::json!({"type": "json_object"}),
622 ResponseFormat::JsonObject {
623 schema: Some(schema),
624 } => serde_json::json!({
625 "type": "json_schema",
626 "json_schema": {"name": "response", "schema": schema}
627 }),
628 }
629}
630
631fn validate_reasoning_config(config: &CompletionConfig) -> Result<()> {
633 config.validate_reasoning()?;
634 if config.preserve_reasoning
635 || config.reasoning_effort == Some(crate::llm::gateway::ReasoningEffort::Enabled)
636 {
637 return Err(MojenticError::ConfigError(
638 "OpenAI gateway does not support native reasoning history or boolean reasoning enablement; use a supported named effort".into(),
639 ));
640 }
641 Ok(())
642}
643
644pub(crate) fn add_response_format(body: &mut Value, config: &CompletionConfig) {
645 if let Some(format) = &config.response_format {
646 body["response_format"] = openai_response_format(format);
647 }
648}
649
650impl crate::llm::recovery::RecoveryProvider for OpenAIGateway {
651 fn recovery_provider(&self) -> crate::llm::recovery::Provider {
652 crate::llm::recovery::Provider::OpenAi
653 }
654 fn prepare_recovery(
655 &self,
656 model: &str,
657 messages: &[LlmMessage],
658 tools: Option<&[Box<dyn LlmTool>]>,
659 config: &CompletionConfig,
660 schema: Option<Value>,
661 stream: bool,
662 ) -> Result<reqwest::Request> {
663 let mut config = config.clone();
664 if let Some(schema) = schema {
665 config.response_format = Some(ResponseFormat::JsonObject {
666 schema: Some(schema),
667 });
668 }
669 let (mut body, supports_tools) = self.chat_body(model, messages, &config)?;
670 body["stream"] = Value::Bool(stream);
671 if stream {
672 body["stream_options"] = serde_json::json!({"include_usage": true});
673 }
674 if let Some(tools) = tools {
675 if supports_tools {
676 body["tools"] =
677 serde_json::to_value(tools.iter().map(|t| t.descriptor()).collect::<Vec<_>>())?;
678 }
679 }
680 let mut request = self.chat_request(&body).build()?;
681 *request.timeout_mut() = None;
682 Ok(request)
683 }
684}
685
686#[cfg(test)]
687mod tests {
688 #[test]
689 fn unsupported_new_reasoning_controls_fail_instead_of_disappearing() {
690 use crate::llm::gateway::{CompletionConfig, ReasoningEffort};
691 assert!(super::validate_reasoning_config(&CompletionConfig::default()).is_ok());
692 assert!(super::validate_reasoning_config(&CompletionConfig {
693 reasoning_effort: Some(ReasoningEffort::Enabled),
694 ..Default::default()
695 })
696 .is_err());
697 assert!(super::validate_reasoning_config(&CompletionConfig {
698 preserve_reasoning: true,
699 ..Default::default()
700 })
701 .is_err());
702 }
703 use super::*;
704 use futures::stream::StreamExt;
705 use std::sync::Mutex;
706
707 static ENV_MUTEX: Mutex<()> = Mutex::new(());
708
709 #[test]
710 fn test_openai_config_default() {
711 let _guard = ENV_MUTEX.lock().unwrap();
712 std::env::remove_var("OPENAI_API_KEY");
713 std::env::remove_var("OPENAI_API_ENDPOINT");
714 let config = OpenAIConfig::default();
715 assert_eq!(config.api_key, "");
716 assert_eq!(config.base_url, "https://api.openai.com/v1");
717 assert!(config.timeout.is_none());
718 }
719
720 #[test]
721 fn test_openai_config_from_env() {
722 let _guard = ENV_MUTEX.lock().unwrap();
723 std::env::set_var("OPENAI_API_KEY", "test-key");
724 std::env::set_var("OPENAI_API_ENDPOINT", "https://custom.openai.com");
725 let config = OpenAIConfig::default();
726 assert_eq!(config.api_key, "test-key");
727 assert_eq!(config.base_url, "https://custom.openai.com");
728 std::env::remove_var("OPENAI_API_KEY");
729 std::env::remove_var("OPENAI_API_ENDPOINT");
730 }
731
732 #[test]
733 fn test_gateway_new() {
734 let gateway = OpenAIGateway::new();
735 assert_eq!(gateway.config.base_url, "https://api.openai.com/v1");
736 }
737
738 #[test]
739 fn test_gateway_with_api_key() {
740 let gateway = OpenAIGateway::with_api_key("my-api-key");
741 assert_eq!(gateway.config.api_key, "my-api-key");
742 }
743
744 #[test]
745 fn test_gateway_with_api_key_and_base_url() {
746 let gateway = OpenAIGateway::with_api_key_and_base_url("key", "https://custom.com");
747 assert_eq!(gateway.config.api_key, "key");
748 assert_eq!(gateway.config.base_url, "https://custom.com");
749 }
750
751 #[test]
752 fn test_gateway_default() {
753 let gateway = OpenAIGateway::default();
754 assert_eq!(gateway.config.base_url, "https://api.openai.com/v1");
755 }
756
757 #[test]
758 fn test_chunk_text_short() {
759 let gateway = OpenAIGateway::new();
760 let chunks = gateway.chunk_text("Hello world", 100).expect("tokens");
761 assert_eq!(chunks.len(), 1);
762 assert_eq!(chunks[0].len(), 2);
763 }
764
765 #[test]
766 fn test_chunk_text_long() {
767 let gateway = OpenAIGateway::new();
768 let long_text = "a".repeat(50000);
769 let chunks = gateway.chunk_text(&long_text, 100).expect("tokens");
770 assert!(chunks.len() > 1);
771 assert!(chunks.iter().all(|chunk| chunk.len() <= 100));
772 }
773
774 #[test]
775 fn test_weighted_average_embeddings_single() {
776 let gateway = OpenAIGateway::new();
777 let embeddings = vec![vec![1.0, 2.0, 3.0]];
778 let weights = vec![1.0];
779 let result = gateway.weighted_average_embeddings(&embeddings, &weights);
780
781 let norm = (1.0_f32 + 4.0 + 9.0).sqrt();
783 assert!((result[0] - 1.0 / norm).abs() < 0.001);
784 assert!((result[1] - 2.0 / norm).abs() < 0.001);
785 assert!((result[2] - 3.0 / norm).abs() < 0.001);
786 }
787
788 #[test]
789 fn test_weighted_average_embeddings_multiple() {
790 let gateway = OpenAIGateway::new();
791 let embeddings = vec![vec![1.0, 0.0], vec![0.0, 1.0]];
792 let weights = vec![1.0, 1.0];
793 let result = gateway.weighted_average_embeddings(&embeddings, &weights);
794
795 let expected = 1.0 / (2.0_f32).sqrt();
797 assert!((result[0] - expected).abs() < 0.001);
798 assert!((result[1] - expected).abs() < 0.001);
799 }
800
801 #[test]
802 fn test_weighted_average_embeddings_empty() {
803 let gateway = OpenAIGateway::new();
804 let embeddings: Vec<Vec<f32>> = vec![];
805 let weights: Vec<f32> = vec![];
806 let result = gateway.weighted_average_embeddings(&embeddings, &weights);
807 assert!(result.is_empty());
808 }
809
810 #[test]
811 fn test_adapt_parameters_chat_model() {
812 let gateway = OpenAIGateway::new();
813 let config = CompletionConfig {
814 temperature: Some(0.7),
815 max_tokens: Some(1000),
816 ..Default::default()
817 };
818
819 let (params, supports_tools) = gateway.adapt_parameters_for_model("gpt-4", &config);
820
821 assert!(params.contains_key("max_tokens"));
822 assert!(!params.contains_key("max_completion_tokens"));
823 assert!(supports_tools);
824 }
825
826 #[test]
827 fn test_adapt_parameters_reasoning_model() {
828 let gateway = OpenAIGateway::new();
829 let config = CompletionConfig {
830 temperature: Some(0.7),
831 max_tokens: Some(1000),
832 ..Default::default()
833 };
834
835 let (params, supports_tools) = gateway.adapt_parameters_for_model("o1", &config);
836
837 assert!(!params.contains_key("max_tokens"));
838 assert!(params.contains_key("max_completion_tokens"));
839 assert!(supports_tools); }
841
842 #[tokio::test]
843 async fn test_complete_success() {
844 let mut server = mockito::Server::new_async().await;
845 let mock = server
846 .mock("POST", "/chat/completions")
847 .with_status(200)
848 .with_body(r#"{"choices":[{"message":{"role":"assistant","content":"Hello!"}}]}"#)
849 .create();
850
851 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
852 let messages = vec![LlmMessage::user("Hi")];
853 let config = CompletionConfig::default();
854
855 let result = gateway.complete("gpt-4", &messages, None, &config).await;
856
857 mock.assert();
858 assert!(result.is_ok());
859 let response = result.unwrap();
860 assert_eq!(response.content, Some("Hello!".to_string()));
861 }
862
863 #[tokio::test]
864 async fn test_complete_with_tool_calls() {
865 let mut server = mockito::Server::new_async().await;
866 let mock = server
867 .mock("POST", "/chat/completions")
868 .with_status(200)
869 .with_body(r#"{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"location\": \"NYC\"}"}}]}}]}"#)
870 .create();
871
872 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
873 let messages = vec![LlmMessage::user("Weather?")];
874 let config = CompletionConfig::default();
875
876 let result = gateway.complete("gpt-4", &messages, None, &config).await;
877
878 mock.assert();
879 assert!(result.is_ok());
880 let response = result.unwrap();
881 assert_eq!(response.tool_calls.len(), 1);
882 assert_eq!(response.tool_calls[0].name, "get_weather");
883 }
884
885 #[tokio::test]
886 async fn test_complete_error() {
887 let mut server = mockito::Server::new_async().await;
888 let mock = server
889 .mock("POST", "/chat/completions")
890 .with_status(401)
891 .with_body("Unauthorized")
892 .create();
893
894 let gateway = OpenAIGateway::with_api_key_and_base_url("bad-key", server.url());
895 let messages = vec![LlmMessage::user("Hi")];
896 let config = CompletionConfig::default();
897
898 let result = gateway.complete("gpt-4", &messages, None, &config).await;
899
900 mock.assert();
901 assert!(result.is_err());
902 }
903
904 #[tokio::test]
905 async fn test_complete_json() {
906 let mut server = mockito::Server::new_async().await;
907 let mock = server
908 .mock("POST", "/chat/completions")
909 .with_status(200)
910 .with_body(
911 r#"{"choices":[{"message":{"content":"{\"name\":\"test\",\"value\":42}"}}]}"#,
912 )
913 .create();
914
915 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
916 let messages = vec![LlmMessage::user("Generate JSON")];
917 let schema = serde_json::json!({"type": "object"});
918 let config = CompletionConfig::default();
919
920 let result = gateway.complete_json("gpt-4", &messages, schema, &config).await;
921
922 mock.assert();
923 assert!(result.is_ok());
924 let json = result.unwrap();
925 assert_eq!(json["name"], "test");
926 assert_eq!(json["value"], 42);
927 }
928
929 #[tokio::test]
930 async fn test_get_available_models() {
931 let mut server = mockito::Server::new_async().await;
932 let mock = server
933 .mock("GET", "/models")
934 .with_status(200)
935 .with_body(r#"{"data":[{"id":"gpt-4"},{"id":"gpt-3.5-turbo"}]}"#)
936 .create();
937
938 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
939 let result = gateway.get_available_models().await;
940
941 mock.assert();
942 assert!(result.is_ok());
943 let models = result.unwrap();
944 assert_eq!(models.len(), 2);
945 assert_eq!(models[0], "gpt-3.5-turbo");
947 assert_eq!(models[1], "gpt-4");
948 }
949
950 #[tokio::test]
951 async fn test_calculate_embeddings() {
952 let mut server = mockito::Server::new_async().await;
953 let mock = server
954 .mock("POST", "/embeddings")
955 .with_status(200)
956 .with_body(r#"{"data":[{"embedding":[0.1,0.2,0.3,0.4]}]}"#)
957 .create();
958
959 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
960 let result = gateway.calculate_embeddings("test text", None).await;
961
962 mock.assert();
963 assert!(result.is_ok());
964 let embeddings = result.unwrap();
965 assert_eq!(embeddings, vec![0.1, 0.2, 0.3, 0.4]);
966 }
967
968 #[tokio::test]
969 async fn test_calculate_embeddings_weights_by_token_count() {
970 let text = " hello".repeat(8291);
971 let tokenizer = tiktoken_rs::cl100k_base().expect("tokenizer");
972 assert_eq!(tokenizer.encode_ordinary(&text).len(), 8291);
973 let mut server = mockito::Server::new_async().await;
974 let calls = std::sync::atomic::AtomicUsize::new(0);
975 let chunk_lengths = std::sync::Arc::new(Mutex::new(Vec::new()));
976 let captured_lengths = chunk_lengths.clone();
977 let mock = server
978 .mock("POST", "/embeddings")
979 .with_status(200)
980 .with_body_from_request(move |request| {
981 let body: Value = serde_json::from_slice(request.body().expect("request body"))
982 .expect("JSON body");
983 captured_lengths
984 .lock()
985 .expect("lengths")
986 .push(body["input"].as_array().expect("token input").len());
987 let embedding = if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
988 vec![1.0, 0.0]
989 } else {
990 vec![0.0, 1.0]
991 };
992 serde_json::json!({"data": [{"embedding": embedding}]}).to_string().into_bytes()
993 })
994 .expect(2)
995 .create();
996 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
997 let result = gateway.calculate_embeddings(&text, None).await.expect("embeddings");
998 mock.assert();
999 assert_eq!(*chunk_lengths.lock().expect("lengths"), vec![8191, 100]);
1000 let norm = (8191.0_f32.powi(2) + 100.0_f32.powi(2)).sqrt();
1001 assert!((result[0] - 8191.0 / norm).abs() < 1e-6, "{result:?}");
1002 assert!((result[1] - 100.0 / norm).abs() < 1e-6, "{result:?}");
1003 assert!((result.iter().map(|v| v * v).sum::<f32>().sqrt() - 1.0).abs() < 1e-6);
1004 }
1005
1006 async fn streamed_request_body(config: CompletionConfig) -> Value {
1008 let captured = std::sync::Arc::new(Mutex::new(None::<Value>));
1009 let sink = captured.clone();
1010 let mut server = mockito::Server::new_async().await;
1011 let mock = server
1012 .mock("POST", "/chat/completions")
1013 .match_request(move |request| {
1014 let body = serde_json::from_slice(request.body().expect("request body"))
1015 .expect("JSON request body");
1016 *sink.lock().unwrap() = Some(body);
1017 true
1018 })
1019 .with_status(200)
1020 .with_body("data: [DONE]\n\n")
1021 .create_async()
1022 .await;
1023
1024 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1025 let messages = vec![LlmMessage::user("Hi")];
1026 let mut stream = gateway.complete_stream("gpt-4o", &messages, None, &config);
1027 while stream.next().await.is_some() {}
1028
1029 mock.assert_async().await;
1030 let body = captured.lock().unwrap().take();
1031 body.expect("streaming request body was captured")
1032 }
1033
1034 async fn legacy_stream_items(body: &str) -> Vec<Result<StreamChunk>> {
1036 let mut server = mockito::Server::new_async().await;
1037 let _mock = server
1038 .mock("POST", "/chat/completions")
1039 .with_status(200)
1040 .with_header("content-type", "text/event-stream")
1041 .with_body(body)
1042 .create_async()
1043 .await;
1044
1045 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1046 let messages = vec![LlmMessage::user("Hi")];
1047 let config = CompletionConfig::default();
1048 gateway
1049 .complete_stream("gpt-4o", &messages, None, &config)
1050 .filter(|item| futures::future::ready(!matches!(item, Ok(StreamChunk::Metrics(_)))))
1051 .collect()
1052 .await
1053 }
1054
1055 #[tokio::test]
1056 async fn legacy_stream_yields_non_empty_content_and_skips_unreadable_frames() {
1057 let items = legacy_stream_items(concat!(
1058 ": keep-alive\n\n",
1059 "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"\"}}]}\n\n",
1060 "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"}}]}\n\n",
1061 "data: {not json\n\n",
1062 "data: {\"choices\":[]}\n\n",
1063 "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n",
1064 "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
1065 "data: [DONE]\n\n",
1066 ))
1067 .await;
1068
1069 let texts: Vec<_> = items
1070 .iter()
1071 .map(|item| match item {
1072 Ok(StreamChunk::Content(text)) => text.clone(),
1073 other => panic!("expected only content, got {other:?}"),
1074 })
1075 .collect();
1076 assert_eq!(texts, ["Hel", "lo"]);
1077 }
1078
1079 #[tokio::test]
1080 async fn legacy_stream_accumulates_tool_call_fragments_until_the_finish_reason() {
1081 let items = legacy_stream_items(concat!(
1082 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]}}]}\n\n",
1083 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"location\\\":\"}}]}}]}\n\n",
1084 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\" \\\"NYC\\\"}\"}}]}}]}\n\n",
1085 "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",
1086 "data: [DONE]\n\n",
1087 ))
1088 .await;
1089
1090 let [Ok(StreamChunk::ToolCalls(calls))] = items.as_slice() else {
1091 panic!("expected one tool-calls chunk, got {items:?}");
1092 };
1093 assert_eq!(calls.len(), 1);
1094 assert_eq!(calls[0].id.as_deref(), Some("call_1"));
1095 assert_eq!(calls[0].name, "get_weather");
1096 assert_eq!(calls[0].arguments["location"], "NYC");
1097 }
1098
1099 #[tokio::test]
1100 async fn legacy_stream_yields_pending_tool_calls_at_done() {
1101 let items = legacy_stream_items(concat!(
1102 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"b\",\"arguments\":\"{}\"}}]}}]}\n\n",
1103 "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{}\"}}]}}]}\n\n",
1104 "data: [DONE]\n\n",
1105 ))
1106 .await;
1107
1108 let [Ok(StreamChunk::ToolCalls(calls))] = items.as_slice() else {
1109 panic!("expected one tool-calls chunk, got {items:?}");
1110 };
1111 let names: Vec<_> = calls.iter().map(|call| call.name.as_str()).collect();
1112 assert_eq!(names, ["a", "b"]);
1113 }
1114
1115 #[tokio::test]
1116 async fn legacy_stream_keeps_a_character_split_across_network_chunks() {
1117 use crate::llm::stream_events::testing::{
1118 split_body_server, split_inside_first_multibyte_char,
1119 };
1120 let body = concat!(
1121 "data: {\"choices\":[{\"delta\":{\"content\":\"Caf\u{e9} \u{1f30a}\"}}]}\n\n",
1122 "data: [DONE]\n\n",
1123 );
1124 let url = split_body_server(split_inside_first_multibyte_char(body)).await;
1125
1126 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", url);
1127 let messages = vec![LlmMessage::user("Hi")];
1128 let config = CompletionConfig::default();
1129 let items: Vec<_> = gateway
1130 .complete_stream("gpt-4o", &messages, None, &config)
1131 .filter(|item| futures::future::ready(!matches!(item, Ok(StreamChunk::Metrics(_)))))
1132 .collect()
1133 .await;
1134
1135 assert!(
1136 matches!(items.as_slice(), [Ok(StreamChunk::Content(text))] if text == "Caf\u{e9} \u{1f30a}"),
1137 "{items:?}"
1138 );
1139 }
1140
1141 #[tokio::test]
1142 async fn legacy_stream_reports_http_error_status() {
1143 let mut server = mockito::Server::new_async().await;
1144 let _mock = server
1145 .mock("POST", "/chat/completions")
1146 .with_status(500)
1147 .with_body("boom")
1148 .create_async()
1149 .await;
1150
1151 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1152 let messages = vec![LlmMessage::user("Hi")];
1153 let config = CompletionConfig::default();
1154 let items: Vec<_> = gateway
1155 .complete_stream("gpt-4o", &messages, None, &config)
1156 .filter(|item| futures::future::ready(!matches!(item, Ok(StreamChunk::Metrics(_)))))
1157 .collect()
1158 .await;
1159
1160 assert!(matches!(
1161 items.as_slice(),
1162 [Err(MojenticError::GatewayError(message))] if message == "OpenAI API error: 500 Internal Server Error"
1163 ));
1164 }
1165
1166 fn config_with_format(format: Option<ResponseFormat>) -> CompletionConfig {
1167 CompletionConfig {
1168 response_format: format,
1169 ..Default::default()
1170 }
1171 }
1172
1173 #[tokio::test]
1174 async fn streaming_request_without_format_leaves_body_unchanged() {
1175 let body = streamed_request_body(config_with_format(None)).await;
1176
1177 assert_eq!(body["stream"], true);
1178 assert!(body.get("response_format").is_none());
1179 }
1180
1181 #[tokio::test]
1182 async fn streaming_request_forwards_text_format() {
1183 let body = streamed_request_body(config_with_format(Some(ResponseFormat::Text))).await;
1184
1185 assert_eq!(body["response_format"], serde_json::json!({"type": "text"}));
1186 }
1187
1188 #[tokio::test]
1189 async fn streaming_request_forwards_json_object_format() {
1190 let body = streamed_request_body(config_with_format(Some(ResponseFormat::JsonObject {
1191 schema: None,
1192 })))
1193 .await;
1194
1195 assert_eq!(body["response_format"], serde_json::json!({"type": "json_object"}));
1196 }
1197
1198 #[tokio::test]
1199 async fn streaming_request_forwards_json_schema_format() {
1200 let schema =
1201 serde_json::json!({"type": "object", "properties": {"n": {"type": "integer"}}});
1202 let body = streamed_request_body(config_with_format(Some(ResponseFormat::JsonObject {
1203 schema: Some(schema.clone()),
1204 })))
1205 .await;
1206
1207 assert_eq!(
1208 body["response_format"],
1209 serde_json::json!({
1210 "type": "json_schema",
1211 "json_schema": {"name": "response", "schema": schema}
1212 })
1213 );
1214 }
1215
1216 #[tokio::test]
1217 async fn non_streaming_request_forwards_configured_format() {
1218 let mut server = mockito::Server::new_async().await;
1219 let mock = server
1220 .mock("POST", "/chat/completions")
1221 .match_body(mockito::Matcher::PartialJson(serde_json::json!({
1222 "response_format": {"type": "json_object"}
1223 })))
1224 .with_status(200)
1225 .with_body(r#"{"choices":[{"message":{"role":"assistant","content":"{}"}}]}"#)
1226 .create_async()
1227 .await;
1228
1229 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1230 let messages = vec![LlmMessage::user("Hi")];
1231 let config = config_with_format(Some(ResponseFormat::JsonObject { schema: None }));
1232
1233 gateway.complete("gpt-4o", &messages, None, &config).await.unwrap();
1234
1235 mock.assert_async().await;
1236 }
1237
1238 #[tokio::test]
1239 async fn complete_carries_reported_evidence_unchanged() {
1240 let mut server = mockito::Server::new_async().await;
1241 let mock = server
1242 .mock("POST", "/chat/completions")
1243 .with_status(200)
1244 .with_body(
1245 r#"{"id":"chatcmpl-9","object":"chat.completion","created":1727000000,
1246 "model":"gpt-4o-2024-08-06","system_fingerprint":"fp_1",
1247 "choices":[{"index":0,"message":{"role":"assistant","content":"Hi"},"finish_reason":"length"}],
1248 "usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5,
1249 "completion_tokens_details":{"reasoning_tokens":0}}}"#,
1250 )
1251 .create_async()
1252 .await;
1253
1254 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1255 let messages = vec![LlmMessage::user("Hi")];
1256
1257 let response = gateway
1258 .complete("gpt-4o", &messages, None, &CompletionConfig::default())
1259 .await
1260 .unwrap();
1261
1262 mock.assert_async().await;
1263 let evidence = response.evidence;
1264 assert_eq!(evidence.provider_model.as_deref(), Some("gpt-4o-2024-08-06"));
1265 assert_eq!(evidence.finish_reason.as_deref(), Some("length"));
1266 assert_eq!(
1267 evidence.usage,
1268 Some(serde_json::json!({"prompt_tokens":4,"completion_tokens":1,"total_tokens":5,
1269 "completion_tokens_details":{"reasoning_tokens":0}}))
1270 );
1271 assert_eq!(evidence.metadata["id"], "chatcmpl-9");
1272 assert_eq!(evidence.metadata["system_fingerprint"], "fp_1");
1273 assert_eq!(evidence.metadata["created"], 1727000000);
1274 }
1275
1276 #[tokio::test]
1277 async fn complete_leaves_unreported_usage_unknown() {
1278 let mut server = mockito::Server::new_async().await;
1279 let mock = server
1280 .mock("POST", "/chat/completions")
1281 .with_status(200)
1282 .with_body(r#"{"choices":[{"message":{"role":"assistant","content":"Hello!"}}]}"#)
1283 .create_async()
1284 .await;
1285
1286 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1287 let messages = vec![LlmMessage::user("Hi")];
1288
1289 let response = gateway
1290 .complete("gpt-4o", &messages, None, &CompletionConfig::default())
1291 .await
1292 .unwrap();
1293
1294 mock.assert_async().await;
1295 assert_eq!(response.evidence, crate::llm::models::ResponseEvidence::default());
1296 }
1297
1298 #[tokio::test]
1299 async fn complete_json_response_carries_object_and_evidence() {
1300 let mut server = mockito::Server::new_async().await;
1301 let mock = server
1302 .mock("POST", "/chat/completions")
1303 .with_status(200)
1304 .with_body(
1305 r#"{"model":"gpt-4o-mini","choices":[{"message":{"content":"{\"n\":1}"},"finish_reason":"stop"}],
1306 "usage":{"prompt_tokens":2,"completion_tokens":3}}"#,
1307 )
1308 .create_async()
1309 .await;
1310
1311 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1312 let messages = vec![LlmMessage::user("JSON")];
1313
1314 let response = gateway
1315 .complete_json_response(
1316 "gpt-4o",
1317 &messages,
1318 serde_json::json!({"type": "object"}),
1319 &CompletionConfig::default(),
1320 )
1321 .await
1322 .unwrap();
1323
1324 mock.assert_async().await;
1325 assert_eq!(response.object, Some(serde_json::json!({"n": 1})));
1326 assert_eq!(response.content.as_deref(), Some(r#"{"n":1}"#));
1327 assert_eq!(response.evidence.provider_model.as_deref(), Some("gpt-4o-mini"));
1328 assert_eq!(response.evidence.finish_reason.as_deref(), Some("stop"));
1329 assert_eq!(
1330 response.evidence.usage,
1331 Some(serde_json::json!({"prompt_tokens":2,"completion_tokens":3}))
1332 );
1333 }
1334
1335 mod stream_events {
1336 use super::*;
1337 use crate::llm::stream_events::testing::endless_stream_server;
1338 use crate::llm::{StreamEvent, StreamEventError};
1339
1340 async fn collect_events(
1341 server: &mockito::Server,
1342 config: &CompletionConfig,
1343 ) -> Vec<StreamEvent> {
1344 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1345 let messages = vec![LlmMessage::user("Hi")];
1346 let stream = gateway
1347 .complete_stream_events("gpt-4o", &messages, config)
1348 .expect("OpenAI supports stream events");
1349 let events: Vec<_> = stream.collect().await;
1350 let metrics: Vec<_> = events
1351 .iter()
1352 .filter_map(|e| {
1353 if let StreamEvent::Metrics(m) = e {
1354 Some(m)
1355 } else {
1356 None
1357 }
1358 })
1359 .collect();
1360 assert_eq!(metrics.len(), 1);
1361 assert!(metrics[0].request_duration_ms.is_some());
1362 events
1365 .into_iter()
1366 .filter(|event| !matches!(event, StreamEvent::Metrics(_)))
1367 .collect()
1368 }
1369
1370 #[tokio::test]
1371 async fn request_streams_without_tools_and_asks_for_usage() {
1372 let captured = std::sync::Arc::new(Mutex::new(None::<Value>));
1373 let sink = captured.clone();
1374 let mut server = mockito::Server::new_async().await;
1375 let mock = server
1376 .mock("POST", "/chat/completions")
1377 .match_header("authorization", "Bearer test-key")
1378 .match_request(move |request| {
1379 *sink.lock().unwrap() =
1380 serde_json::from_slice(request.body().expect("request body")).ok();
1381 true
1382 })
1383 .with_status(200)
1384 .with_body("data: [DONE]\n\n")
1385 .create_async()
1386 .await;
1387 let config = CompletionConfig {
1388 response_format: Some(ResponseFormat::JsonObject { schema: None }),
1389 ..Default::default()
1390 };
1391
1392 collect_events(&server, &config).await;
1393
1394 mock.assert_async().await;
1395 let body = captured.lock().unwrap().take().expect("captured body");
1396 assert_eq!(body["model"], "gpt-4o");
1397 assert_eq!(body["stream"], true);
1398 assert_eq!(body["stream_options"], serde_json::json!({"include_usage": true}));
1399 assert_eq!(body["response_format"], serde_json::json!({"type": "json_object"}));
1400 assert!(body.get("tools").is_none());
1401 }
1402
1403 #[tokio::test]
1404 async fn completes_over_http_with_usage() {
1405 let mut server = mockito::Server::new_async().await;
1406 let mock = server
1407 .mock("POST", "/chat/completions")
1408 .with_status(200)
1409 .with_header("content-type", "text/event-stream")
1410 .with_body(concat!(
1411 "data: {\"model\":\"gpt-4o-x\",\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n",
1412 "data: {\"model\":\"gpt-4o-x\",\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
1413 "data: {\"model\":\"gpt-4o-x\",\"choices\":[],\"usage\":{\"total_tokens\":3}}\n\n",
1414 "data: [DONE]\n\n",
1415 ))
1416 .create_async()
1417 .await;
1418
1419 let events = collect_events(&server, &CompletionConfig::default()).await;
1420
1421 mock.assert_async().await;
1422 assert!(matches!(
1423 events.as_slice(),
1424 [StreamEvent::Content(text), StreamEvent::Completed(evidence)]
1425 if text == "Hi"
1426 && evidence.usage == Some(serde_json::json!({"total_tokens": 3}))
1427 && evidence.provider_model.as_deref() == Some("gpt-4o-x")
1428 ));
1429 }
1430
1431 #[tokio::test]
1432 async fn end_of_stream_without_done_is_incomplete_stream() {
1433 let mut server = mockito::Server::new_async().await;
1434 let _mock = server
1435 .mock("POST", "/chat/completions")
1436 .with_status(200)
1437 .with_body(concat!(
1438 "data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n",
1439 "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
1440 ))
1441 .create_async()
1442 .await;
1443
1444 let events = collect_events(&server, &CompletionConfig::default()).await;
1445
1446 assert!(matches!(
1447 events.as_slice(),
1448 [
1449 StreamEvent::Content(_),
1450 StreamEvent::Error(StreamEventError::IncompleteStream(_))
1451 ]
1452 ));
1453 }
1454
1455 #[tokio::test]
1456 async fn http_error_status_is_reported_with_body() {
1457 let mut server = mockito::Server::new_async().await;
1458 let _mock = server
1459 .mock("POST", "/chat/completions")
1460 .with_status(429)
1461 .with_body(r#"{"error":{"message":"slow down"}}"#)
1462 .create_async()
1463 .await;
1464
1465 let events = collect_events(&server, &CompletionConfig::default()).await;
1466
1467 assert!(matches!(
1468 events.as_slice(),
1469 [StreamEvent::Error(StreamEventError::ProviderError { status: Some(429), error })]
1470 if error["message"] == "slow down"
1471 ));
1472 }
1473
1474 #[tokio::test]
1475 async fn dropping_the_stream_cancels_the_request() {
1476 let (url, closed) =
1477 endless_stream_server("data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n")
1478 .await;
1479 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", url);
1480 let messages = vec![LlmMessage::user("Hi")];
1481 let config = CompletionConfig::default();
1482
1483 let mut stream =
1484 gateway.complete_stream_events("gpt-4o", &messages, &config).expect("supported");
1485 assert!(matches!(stream.next().await, Some(StreamEvent::Content(_))));
1486 drop(stream);
1487
1488 tokio::time::timeout(std::time::Duration::from_secs(5), closed)
1489 .await
1490 .expect("the request was not cancelled")
1491 .expect("server reported the disconnect");
1492 }
1493 }
1494
1495 #[tokio::test]
1496 async fn test_calculate_embeddings_custom_model() {
1497 let mut server = mockito::Server::new_async().await;
1498 let mock = server
1499 .mock("POST", "/embeddings")
1500 .match_body(mockito::Matcher::JsonString(
1501 r#"{"model":"text-embedding-3-small","input":"test"}"#.to_string(),
1502 ))
1503 .with_status(200)
1504 .with_body(r#"{"data":[{"embedding":[0.5,0.6]}]}"#)
1505 .create();
1506
1507 let gateway = OpenAIGateway::with_api_key_and_base_url("test-key", server.url());
1508 let result = gateway.calculate_embeddings("test", Some("text-embedding-3-small")).await;
1509
1510 mock.assert();
1511 assert!(result.is_ok());
1512 }
1513}