1use crate::llm::tools::{LlmTool, ToolCallExecution, ToolRunCtx, ToolRunner};
17use crate::realtime::codec::{decode_base64_pcm16, encode_base64_pcm16};
18use crate::realtime::config::{
19 defaults, InterruptOutputPolicy, RealtimeAudioFormat, RealtimeModality, RealtimeToolChoice,
20 RealtimeVoiceConfig, SemanticVadConfig, ServerVadConfig, TurnDetectionMode,
21};
22use crate::realtime::events::{InterruptReason, RealtimeEvent, SessionCloseReason, TokenUsage};
23use crate::realtime::gateway::RealtimeGatewaySession;
24use crate::tracer::TracerSystem;
25use serde_json::{json, Value};
26use std::collections::HashMap;
27use std::sync::Arc;
28use tokio::sync::mpsc;
29use tokio_util::sync::CancellationToken;
30use uuid::Uuid;
31
32pub struct RealtimeSession {
34 events_rx: mpsc::UnboundedReceiver<RealtimeEvent>,
35 commands_tx: mpsc::UnboundedSender<Command>,
36}
37
38#[derive(Debug)]
39enum Command {
40 SendText(String),
41 SendAudio(Vec<i16>),
42 CommitAudio,
43 Interrupt,
44 UpdateInstructions(String),
45 Close,
46}
47
48#[derive(Debug, Default)]
49struct PendingCall {
50 name: String,
51 args_buffer: String,
52 done: bool,
53}
54
55#[derive(Debug)]
56struct TurnState {
57 turn_id: String,
58 calls: HashMap<String, PendingCall>,
59 cancelled: bool,
60 cancel_token: CancellationToken,
61}
62
63impl RealtimeSession {
64 pub async fn next_event(&mut self) -> Option<RealtimeEvent> {
66 self.events_rx.recv().await
67 }
68
69 pub fn send_text(&self, text: impl Into<String>) -> Result<(), &'static str> {
70 self.commands_tx
71 .send(Command::SendText(text.into()))
72 .map_err(|_| "session closed")
73 }
74
75 pub fn send_audio_frame(&self, samples: Vec<i16>) -> Result<(), &'static str> {
76 self.commands_tx.send(Command::SendAudio(samples)).map_err(|_| "session closed")
77 }
78
79 pub fn commit_audio(&self) -> Result<(), &'static str> {
80 self.commands_tx.send(Command::CommitAudio).map_err(|_| "session closed")
81 }
82
83 pub fn interrupt(&self) -> Result<(), &'static str> {
84 self.commands_tx.send(Command::Interrupt).map_err(|_| "session closed")
85 }
86
87 pub fn update_instructions(&self, instructions: impl Into<String>) -> Result<(), &'static str> {
88 self.commands_tx
89 .send(Command::UpdateInstructions(instructions.into()))
90 .map_err(|_| "session closed")
91 }
92
93 pub fn close(&self) -> Result<(), &'static str> {
94 self.commands_tx.send(Command::Close).map_err(|_| "session closed")
95 }
96
97 pub async fn run(
101 gateway: Box<dyn RealtimeGatewaySession>,
102 config: RealtimeVoiceConfig,
103 tool_runner: Arc<dyn ToolRunner>,
104 tracer: Option<Arc<TracerSystem>>,
105 correlation_id: String,
106 ) -> Self {
107 let (events_tx, events_rx) = mpsc::unbounded_channel();
108 let (commands_tx, commands_rx) = mpsc::unbounded_channel();
109
110 let driver = SessionDriver {
111 gateway,
112 config,
113 tool_runner,
114 tracer,
115 correlation_id,
116 events_tx,
117 current_turn: None,
118 current_response_id: None,
119 };
120
121 tokio::spawn(driver.run(commands_rx));
122
123 Self {
124 events_rx,
125 commands_tx,
126 }
127 }
128}
129
130struct SessionDriver {
133 gateway: Box<dyn RealtimeGatewaySession>,
134 config: RealtimeVoiceConfig,
135 tool_runner: Arc<dyn ToolRunner>,
136 tracer: Option<Arc<TracerSystem>>,
137 correlation_id: String,
138 events_tx: mpsc::UnboundedSender<RealtimeEvent>,
139 current_turn: Option<TurnState>,
140 current_response_id: Option<String>,
141}
142
143impl SessionDriver {
144 async fn run(mut self, mut commands_rx: mpsc::UnboundedReceiver<Command>) {
145 let initial = build_session_update(&self.config);
146 let _ = self.gateway.send_event(&initial).await;
147 let _ = self.events_tx.send(RealtimeEvent::SessionOpened {
148 session_id: self.gateway.session_id().to_string(),
149 });
150
151 loop {
152 tokio::select! {
153 cmd = commands_rx.recv() => {
154 match cmd {
155 Some(Command::SendText(text)) => self.send_text(text).await,
156 Some(Command::SendAudio(samples)) => self.send_audio(samples).await,
157 Some(Command::CommitAudio) => self.commit_audio().await,
158 Some(Command::Interrupt) => self.cancel_turn(InterruptReason::Manual).await,
159 Some(Command::UpdateInstructions(text)) => self.update_instructions(text).await,
160 Some(Command::Close) | None => {
161 let _ = self.gateway.close().await;
162 let _ = self.events_tx.send(RealtimeEvent::SessionClosed { reason: SessionCloseReason::Client });
163 return;
164 }
165 }
166 }
167 event = self.gateway.next_event() => {
168 match event {
169 Some(raw) => self.handle_server_event(raw).await,
170 None => {
171 let _ = self.events_tx.send(RealtimeEvent::SessionClosed { reason: SessionCloseReason::Server });
172 return;
173 }
174 }
175 }
176 }
177 }
178 }
179
180 async fn send_text(&mut self, text: String) {
181 let _ = self
182 .gateway
183 .send_event(&json!({
184 "type": "conversation.item.create",
185 "item": {
186 "type": "message",
187 "role": "user",
188 "content": [{ "type": "input_text", "text": text }]
189 }
190 }))
191 .await;
192 let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
193 }
194
195 async fn send_audio(&mut self, samples: Vec<i16>) {
196 let _ = self
197 .gateway
198 .send_event(&json!({
199 "type": "input_audio_buffer.append",
200 "audio": encode_base64_pcm16(&samples)
201 }))
202 .await;
203 }
204
205 async fn commit_audio(&mut self) {
206 let _ = self.gateway.send_event(&json!({ "type": "input_audio_buffer.commit" })).await;
207 let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
208 }
209
210 async fn update_instructions(&mut self, instructions: String) {
211 self.config.instructions = Some(instructions.clone());
212 let _ = self
213 .gateway
214 .send_event(&json!({
215 "type": "session.update",
216 "session": { "instructions": instructions }
217 }))
218 .await;
219 }
220
221 async fn cancel_turn(&mut self, reason: InterruptReason) {
222 let Some(turn) = self.current_turn.as_mut() else {
223 return;
224 };
225 if turn.cancelled {
226 return;
227 }
228 turn.cancelled = true;
229 turn.cancel_token.cancel();
230 let _ = self.events_tx.send(RealtimeEvent::Interrupted {
231 turn_id: turn.turn_id.clone(),
232 reason,
233 });
234 if self.current_response_id.is_some() {
235 let _ = self.gateway.send_event(&json!({ "type": "response.cancel" })).await;
236 }
237 }
238
239 async fn handle_server_event(&mut self, raw: Value) {
240 let Some(event_type) = raw.get("type").and_then(|v| v.as_str()) else {
241 return;
242 };
243
244 match event_type {
245 "session.created" => {}
246 "session.updated" => {
247 let _ = self.events_tx.send(RealtimeEvent::SessionUpdated {
248 config: HashMap::new(),
249 });
250 }
251 "input_audio_buffer.speech_started" => {
252 let at_ms = raw.get("audio_start_ms").and_then(|v| v.as_u64()).unwrap_or(0);
253 let _ = self.events_tx.send(RealtimeEvent::UserSpeechStarted { at_ms });
254 if self.current_turn.is_some() {
255 self.cancel_turn(InterruptReason::BargeIn).await;
256 }
257 }
258 "input_audio_buffer.speech_stopped" => {
259 let at_ms = raw.get("audio_end_ms").and_then(|v| v.as_u64()).unwrap_or(0);
260 let _ = self.events_tx.send(RealtimeEvent::UserSpeechStopped { at_ms });
261 }
262 "conversation.item.input_audio_transcription.delta" => {
263 self.emit_user_transcript_delta(&raw);
264 }
265 "conversation.item.input_audio_transcription.completed" => {
266 self.emit_user_transcript(&raw);
267 }
268 "response.created" => self.start_response(&raw),
269 "response.output_item.added" => self.add_output_item(&raw),
270 "response.function_call_arguments.delta" => self.fn_args_delta(&raw),
271 "response.function_call_arguments.done" => self.fn_args_done(&raw),
272 "response.text.delta" | "response.output_text.delta" => self.text_delta(&raw),
273 "response.text.done" | "response.output_text.done" => self.text_done(&raw),
274 "response.audio_transcript.delta" | "response.output_audio_transcript.delta" => {
275 self.transcript_delta(&raw)
276 }
277 "response.audio_transcript.done" | "response.output_audio_transcript.done" => {
278 self.transcript_done(&raw)
279 }
280 "response.audio.delta" | "response.output_audio.delta" => self.audio_delta(&raw),
281 "response.done" => self.response_done(&raw).await,
282 "rate_limits.updated" => {
283 let _ = self.events_tx.send(RealtimeEvent::RateLimited {
284 reset_ms: 0,
285 details: HashMap::new(),
286 });
287 }
288 "error" => self.emit_error(&raw),
289 _ => {}
290 }
291 }
292
293 fn emit_user_transcript_delta(&self, raw: &Value) {
294 if let (Some(id), Some(delta)) = (
295 raw.get("item_id").and_then(|v| v.as_str()),
296 raw.get("delta").and_then(|v| v.as_str()),
297 ) {
298 let _ = self.events_tx.send(RealtimeEvent::UserTranscriptDelta {
299 item_id: id.to_string(),
300 delta: delta.to_string(),
301 });
302 }
303 }
304
305 fn emit_user_transcript(&self, raw: &Value) {
306 if let (Some(id), Some(text)) = (
307 raw.get("item_id").and_then(|v| v.as_str()),
308 raw.get("transcript").and_then(|v| v.as_str()),
309 ) {
310 let _ = self.events_tx.send(RealtimeEvent::UserTranscript {
311 item_id: id.to_string(),
312 text: text.to_string(),
313 });
314 }
315 }
316
317 fn start_response(&mut self, raw: &Value) {
318 let Some(id) = raw.pointer("/response/id").and_then(|v| v.as_str()) else {
319 return;
320 };
321 let turn_id = id.to_string();
322 self.current_response_id = Some(turn_id.clone());
323 self.current_turn = Some(TurnState {
324 turn_id: turn_id.clone(),
325 calls: HashMap::new(),
326 cancelled: false,
327 cancel_token: CancellationToken::new(),
328 });
329 let _ = self.events_tx.send(RealtimeEvent::AssistantTurnStarted { turn_id });
330 }
331
332 fn add_output_item(&mut self, raw: &Value) {
333 let Some(turn) = self.current_turn.as_mut() else {
334 return;
335 };
336 let Some(item) = raw.get("item") else { return };
337 if item.get("type").and_then(|v| v.as_str()) != Some("function_call") {
338 return;
339 }
340 let (Some(call_id), Some(name)) = (
341 item.get("call_id").and_then(|v| v.as_str()),
342 item.get("name").and_then(|v| v.as_str()),
343 ) else {
344 return;
345 };
346 turn.calls.insert(
347 call_id.to_string(),
348 PendingCall {
349 name: name.to_string(),
350 args_buffer: String::new(),
351 done: false,
352 },
353 );
354 let _ = self.events_tx.send(RealtimeEvent::ToolCallStarted {
355 turn_id: turn.turn_id.clone(),
356 call_id: call_id.to_string(),
357 name: name.to_string(),
358 });
359 }
360
361 fn fn_args_delta(&mut self, raw: &Value) {
362 let Some(turn) = self.current_turn.as_mut() else {
363 return;
364 };
365 let (Some(call_id), Some(delta)) = (
366 raw.get("call_id").and_then(|v| v.as_str()),
367 raw.get("delta").and_then(|v| v.as_str()),
368 ) else {
369 return;
370 };
371 if let Some(call) = turn.calls.get_mut(call_id) {
372 call.args_buffer.push_str(delta);
373 }
374 let _ = self.events_tx.send(RealtimeEvent::ToolCallArgsDelta {
375 call_id: call_id.to_string(),
376 delta: delta.to_string(),
377 });
378 }
379
380 fn fn_args_done(&mut self, raw: &Value) {
381 let Some(turn) = self.current_turn.as_mut() else {
382 return;
383 };
384 let Some(call_id) = raw.get("call_id").and_then(|v| v.as_str()) else {
385 return;
386 };
387 if let Some(call) = turn.calls.get_mut(call_id) {
388 if let Some(args) = raw.get("arguments").and_then(|v| v.as_str()) {
389 call.args_buffer = args.to_string();
390 }
391 call.done = true;
392 }
393 }
394
395 fn text_delta(&self, raw: &Value) {
396 let Some(turn) = self.current_turn.as_ref() else {
397 return;
398 };
399 if let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) {
400 let _ = self.events_tx.send(RealtimeEvent::AssistantTextDelta {
401 turn_id: turn.turn_id.clone(),
402 delta: delta.to_string(),
403 });
404 }
405 }
406
407 fn text_done(&self, raw: &Value) {
408 let Some(turn) = self.current_turn.as_ref() else {
409 return;
410 };
411 if let Some(text) = raw.get("text").and_then(|v| v.as_str()) {
412 let _ = self.events_tx.send(RealtimeEvent::AssistantText {
413 turn_id: turn.turn_id.clone(),
414 text: text.to_string(),
415 });
416 }
417 }
418
419 fn transcript_delta(&self, raw: &Value) {
420 let Some(turn) = self.current_turn.as_ref() else {
421 return;
422 };
423 if let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) {
424 let _ = self.events_tx.send(RealtimeEvent::AssistantTranscriptDelta {
425 turn_id: turn.turn_id.clone(),
426 delta: delta.to_string(),
427 });
428 }
429 }
430
431 fn transcript_done(&self, raw: &Value) {
432 let Some(turn) = self.current_turn.as_ref() else {
433 return;
434 };
435 if let Some(text) = raw.get("transcript").and_then(|v| v.as_str()) {
436 let _ = self.events_tx.send(RealtimeEvent::AssistantTranscript {
437 turn_id: turn.turn_id.clone(),
438 text: text.to_string(),
439 });
440 }
441 }
442
443 fn audio_delta(&self, raw: &Value) {
444 let Some(turn) = self.current_turn.as_ref() else {
445 return;
446 };
447 let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) else {
448 return;
449 };
450 if let Ok(pcm) = decode_base64_pcm16(delta) {
451 let _ = self.events_tx.send(RealtimeEvent::AssistantAudioDelta {
452 turn_id: turn.turn_id.clone(),
453 pcm,
454 });
455 }
456 }
457
458 async fn response_done(&mut self, raw: &Value) {
459 let Some(turn) = self.current_turn.take() else {
460 return;
461 };
462 let Some(response_id) = raw.pointer("/response/id").and_then(|v| v.as_str()) else {
463 return;
464 };
465 if turn.turn_id != response_id {
466 return;
467 }
468
469 let _ = self.events_tx.send(RealtimeEvent::AssistantTurnCompleted {
470 turn_id: turn.turn_id.clone(),
471 usage: token_usage_from(raw),
472 });
473
474 if turn.calls.is_empty() {
475 self.current_response_id = None;
476 return;
477 }
478
479 self.run_tool_batch(turn).await;
480 self.current_response_id = None;
481 }
482
483 fn emit_error(&self, raw: &Value) {
484 let message = raw
485 .pointer("/error/message")
486 .and_then(|v| v.as_str())
487 .unwrap_or("unknown realtime error");
488 if message.to_lowercase().contains("no active response") {
489 return;
490 }
491 let _ = self.events_tx.send(RealtimeEvent::Error {
492 error: message.to_string(),
493 recoverable: true,
494 });
495 }
496
497 async fn run_tool_batch(&mut self, turn: TurnState) {
498 let mut executions = Vec::new();
499 for (call_id, call) in &turn.calls {
500 if !call.done && call.args_buffer.is_empty() {
501 continue;
502 }
503 let args: HashMap<String, Value> = if call.args_buffer.is_empty() {
504 HashMap::new()
505 } else {
506 match serde_json::from_str::<HashMap<String, Value>>(&call.args_buffer) {
507 Ok(a) => a,
508 Err(err) => {
509 let _ = self.events_tx.send(RealtimeEvent::ToolCallFailed {
510 call_id: call_id.clone(),
511 name: call.name.clone(),
512 error: err.to_string(),
513 });
514 continue;
515 }
516 }
517 };
518 executions.push(ToolCallExecution {
519 id: call_id.clone(),
520 name: call.name.clone(),
521 args,
522 });
523 }
524
525 if executions.is_empty() {
526 return;
527 }
528
529 let tools_vec: Vec<Box<dyn LlmTool>> = self
530 .config
531 .tools
532 .as_ref()
533 .map(|ts| ts.iter().map(|t| t.clone_box()).collect())
534 .unwrap_or_default();
535
536 for exec in &executions {
537 let _ = self.events_tx.send(RealtimeEvent::ToolCallDispatched {
538 call_id: exec.id.clone(),
539 name: exec.name.clone(),
540 args: exec.args.clone(),
541 });
542 }
543
544 let ctx = ToolRunCtx {
545 cancel: turn.cancel_token.clone(),
546 correlation_id: Some(self.correlation_id.clone()),
547 source: Some("RealtimeVoiceBroker".to_string()),
548 };
549
550 let outcomes = self.tool_runner.run_batch(&executions, &tools_vec, &ctx).await;
551
552 for outcome in &outcomes {
553 if outcome.ok {
554 let _ = self.events_tx.send(RealtimeEvent::ToolCallCompleted {
555 call_id: outcome.id.clone(),
556 name: outcome.name.clone(),
557 result: outcome.result.clone().unwrap_or(Value::Null),
558 });
559 } else {
560 let _ = self.events_tx.send(RealtimeEvent::ToolCallFailed {
561 call_id: outcome.id.clone(),
562 name: outcome.name.clone(),
563 error: outcome.error.clone().unwrap_or_default(),
564 });
565 }
566 }
567
568 if let Some(tracer) = &self.tracer {
569 for (exec, outcome) in executions.iter().zip(outcomes.iter()) {
570 tracer.record_tool_call(
571 &outcome.name,
572 exec.args.clone(),
573 outcome.result.clone().unwrap_or(Value::Null),
574 Some("RealtimeVoiceBroker".to_string()),
575 Some(outcome.duration_ms as f64),
576 "RealtimeVoiceBroker",
577 &self.correlation_id,
578 );
579 }
580 let ok = outcomes.iter().filter(|o| o.ok).count();
581 let fail = outcomes.len() - ok;
582 tracer.record_tool_batch(
583 Uuid::new_v4().to_string(),
584 executions.iter().map(|e| e.name.clone()).collect(),
585 ok,
586 fail,
587 outcomes.iter().map(|o| o.duration_ms).sum::<u64>() as f64,
588 Some("RealtimeVoiceBroker".to_string()),
589 "RealtimeVoiceBroker",
590 &self.correlation_id,
591 );
592 }
593
594 let policy = self.config.on_interrupt.unwrap_or(defaults().on_interrupt);
595 let to_submit: Vec<_> = if turn.cancelled {
596 match policy {
597 InterruptOutputPolicy::Drop => Vec::new(),
598 InterruptOutputPolicy::Submit => outcomes.iter().collect(),
599 InterruptOutputPolicy::SubmitCompletedOnly => {
600 outcomes.iter().filter(|o| o.ok).collect()
601 }
602 }
603 } else {
604 outcomes.iter().collect()
605 };
606
607 let mut submitted_ids = Vec::new();
608 for outcome in to_submit {
609 let output = if outcome.ok {
610 serde_json::to_string(outcome.result.as_ref().unwrap_or(&Value::Null))
611 .unwrap_or_default()
612 } else {
613 json!({ "error": outcome.error.clone().unwrap_or_default() }).to_string()
614 };
615 let _ = self
616 .gateway
617 .send_event(&json!({
618 "type": "conversation.item.create",
619 "item": {
620 "type": "function_call_output",
621 "call_id": outcome.id,
622 "output": output
623 }
624 }))
625 .await;
626 submitted_ids.push(outcome.id.clone());
627 }
628
629 if !submitted_ids.is_empty() {
630 let _ = self.events_tx.send(RealtimeEvent::ToolBatchSubmitted {
631 turn_id: turn.turn_id.clone(),
632 call_ids: submitted_ids,
633 });
634 let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
635 }
636 }
637}
638
639fn token_usage_from(raw: &Value) -> Option<TokenUsage> {
640 let usage = raw.pointer("/response/usage")?;
641 Some(TokenUsage {
642 prompt_tokens: usage.get("input_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
643 completion_tokens: usage.get("output_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
644 total_tokens: usage.get("total_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
645 extras: None,
646 })
647}
648
649pub fn build_session_update(config: &RealtimeVoiceConfig) -> Value {
652 let modalities = config.modalities.clone().unwrap_or_else(|| defaults().modalities);
653
654 let output_modalities: Vec<&str> = if modalities.contains(&RealtimeModality::Audio) {
655 vec!["audio"]
656 } else {
657 vec!["text"]
658 };
659
660 let mut audio_input = serde_json::Map::new();
661 audio_input.insert(
662 "format".to_string(),
663 encode_audio_format(config.input_audio_format.unwrap_or(defaults().input_audio_format)),
664 );
665 audio_input.insert(
666 "turn_detection".to_string(),
667 encode_turn_detection(
668 config.turn_detection.as_ref().unwrap_or(&TurnDetectionMode::ServerVad),
669 ),
670 );
671
672 if config.disable_input_audio_transcription {
673 audio_input.insert("transcription".to_string(), Value::Null);
674 } else if let Some(t) = config.input_audio_transcription.as_ref() {
675 audio_input.insert("transcription".to_string(), json!({ "model": t.model }));
676 }
677
678 let mut audio_output = serde_json::Map::new();
679 audio_output.insert(
680 "format".to_string(),
681 encode_audio_format(config.output_audio_format.unwrap_or(defaults().output_audio_format)),
682 );
683 if let Some(voice) = &config.voice {
684 audio_output.insert("voice".to_string(), Value::String(voice.clone()));
685 }
686
687 let mut session = serde_json::Map::new();
688 session.insert("type".to_string(), Value::String("realtime".to_string()));
689 session.insert(
690 "output_modalities".to_string(),
691 Value::Array(output_modalities.iter().map(|s| Value::String(s.to_string())).collect()),
692 );
693 session.insert("audio".to_string(), json!({ "input": audio_input, "output": audio_output }));
694 session.insert(
695 "tool_choice".to_string(),
696 encode_tool_choice(config.tool_choice.as_ref().unwrap_or(&RealtimeToolChoice::Auto)),
697 );
698 if let Some(instr) = &config.instructions {
699 session.insert("instructions".to_string(), Value::String(instr.clone()));
700 }
701 if let Some(max) = config.max_response_output_tokens {
702 session.insert("max_output_tokens".to_string(), Value::from(max));
703 }
704 if let Some(tools) = &config.tools {
705 let tool_list: Vec<Value> = tools
706 .iter()
707 .map(|t| {
708 let d = t.descriptor();
709 json!({
710 "type": "function",
711 "name": d.function.name,
712 "description": d.function.description,
713 "parameters": d.function.parameters
714 })
715 })
716 .collect();
717 session.insert("tools".to_string(), Value::Array(tool_list));
718 }
719 if let Some(extras) = &config.provider_extras {
720 for (k, v) in extras {
721 session.insert(k.clone(), v.clone());
722 }
723 }
724
725 json!({ "type": "session.update", "session": session })
726}
727
728fn encode_audio_format(fmt: RealtimeAudioFormat) -> Value {
729 match fmt {
730 RealtimeAudioFormat::Pcm16 => json!({ "type": "audio/pcm", "rate": 24000 }),
731 RealtimeAudioFormat::G711Ulaw => json!({ "type": "audio/pcmu" }),
732 RealtimeAudioFormat::G711Alaw => json!({ "type": "audio/pcma" }),
733 }
734}
735
736fn encode_turn_detection(mode: &TurnDetectionMode) -> Value {
737 match mode {
738 TurnDetectionMode::None => Value::Null,
739 TurnDetectionMode::ServerVad => json!({ "type": "server_vad" }),
740 TurnDetectionMode::SemanticVad => json!({ "type": "semantic_vad" }),
741 TurnDetectionMode::ServerVadCustom(vad) => encode_server_vad(vad),
742 TurnDetectionMode::SemanticVadCustom(vad) => encode_semantic_vad(vad),
743 }
744}
745
746fn encode_server_vad(vad: &ServerVadConfig) -> Value {
747 let mut map = serde_json::Map::new();
748 map.insert("type".to_string(), Value::String("server_vad".to_string()));
749 if let Some(t) = vad.threshold {
750 map.insert("threshold".to_string(), Value::from(t));
751 }
752 if let Some(p) = vad.prefix_padding_ms {
753 map.insert("prefix_padding_ms".to_string(), Value::from(p));
754 }
755 if let Some(s) = vad.silence_duration_ms {
756 map.insert("silence_duration_ms".to_string(), Value::from(s));
757 }
758 if let Some(c) = vad.create_response {
759 map.insert("create_response".to_string(), Value::from(c));
760 }
761 if let Some(i) = vad.interrupt_response {
762 map.insert("interrupt_response".to_string(), Value::from(i));
763 }
764 if let Some(t) = vad.idle_timeout_ms {
765 map.insert("idle_timeout_ms".to_string(), Value::from(t));
766 }
767 Value::Object(map)
768}
769
770fn encode_semantic_vad(vad: &SemanticVadConfig) -> Value {
771 let mut map = serde_json::Map::new();
772 map.insert("type".to_string(), Value::String("semantic_vad".to_string()));
773 if let Some(e) = &vad.eagerness {
774 map.insert("eagerness".to_string(), Value::String(e.clone()));
775 }
776 Value::Object(map)
777}
778
779fn encode_tool_choice(choice: &RealtimeToolChoice) -> Value {
780 match choice {
781 RealtimeToolChoice::Auto => Value::String("auto".to_string()),
782 RealtimeToolChoice::None => Value::String("none".to_string()),
783 RealtimeToolChoice::Required => Value::String("required".to_string()),
784 RealtimeToolChoice::Function(name) => json!({ "type": "function", "name": name }),
785 }
786}
787
788#[cfg(test)]
789mod tests {
790 use super::*;
791
792 #[test]
793 fn build_session_update_text_only() {
794 let cfg = RealtimeVoiceConfig {
795 modalities: Some(vec![RealtimeModality::Text]),
796 instructions: Some("be brief".to_string()),
797 ..Default::default()
798 };
799
800 let payload = build_session_update(&cfg);
801
802 assert_eq!(payload["type"], "session.update");
803 assert_eq!(payload["session"]["type"], "realtime");
804 assert_eq!(payload["session"]["output_modalities"], json!(["text"]));
805 assert_eq!(payload["session"]["instructions"], "be brief");
806 }
807
808 #[test]
809 fn build_session_update_includes_voice() {
810 let cfg = RealtimeVoiceConfig {
811 voice: Some("verse".to_string()),
812 ..Default::default()
813 };
814
815 let payload = build_session_update(&cfg);
816
817 assert_eq!(payload["session"]["audio"]["output"]["voice"], "verse");
818 }
819}