Skip to main content

mojentic/realtime/
session.rs

1//! Realtime session — driver that normalises raw OpenAI events into
2//! vendor-neutral [`RealtimeEvent`] values and dispatches tool batches.
3//!
4//! Compared to the Python and TypeScript ports, the Rust session is
5//! consumed via two channels:
6//!
7//! - `events_rx`: vendor-neutral [`RealtimeEvent`] stream
8//! - `audio_rx`: decoded PCM frames (also surfaced as
9//!   `AssistantAudioDelta` events for completeness)
10//!
11//! Callers spawn the session via [`RealtimeSession::run`], which takes
12//! the gateway session and returns the channels; the run task pumps the
13//! gateway until close. This mirrors the broker → session orchestration
14//! in the other ports while staying idiomatic Rust async.
15
16use crate::llm::tools::{LlmTool, ToolCallExecution, ToolRunCtx, ToolRunner};
17use crate::realtime::codec::{decode_base64_pcm16, encode_base64_pcm16};
18use crate::realtime::config::{
19    defaults, InterruptOutputPolicy, RealtimeAudioFormat, RealtimeModality, RealtimeToolChoice,
20    RealtimeVoiceConfig, SemanticVadConfig, ServerVadConfig, TurnDetectionMode,
21};
22use crate::realtime::events::{InterruptReason, RealtimeEvent, SessionCloseReason, TokenUsage};
23use crate::realtime::gateway::RealtimeGatewaySession;
24use crate::tracer::TracerSystem;
25use serde_json::{json, Value};
26use std::collections::HashMap;
27use std::sync::Arc;
28use tokio::sync::mpsc;
29use tokio_util::sync::CancellationToken;
30use uuid::Uuid;
31
32/// Outbound handle returned by [`RealtimeSession::run`].
33pub struct RealtimeSession {
34    events_rx: mpsc::UnboundedReceiver<RealtimeEvent>,
35    commands_tx: mpsc::UnboundedSender<Command>,
36}
37
38#[derive(Debug)]
39enum Command {
40    SendText(String),
41    SendAudio(Vec<i16>),
42    CommitAudio,
43    Interrupt,
44    UpdateInstructions(String),
45    Close,
46}
47
48#[derive(Debug, Default)]
49struct PendingCall {
50    name: String,
51    args_buffer: String,
52    done: bool,
53}
54
55#[derive(Debug)]
56struct TurnState {
57    turn_id: String,
58    calls: HashMap<String, PendingCall>,
59    cancelled: bool,
60    cancel_token: CancellationToken,
61}
62
63impl RealtimeSession {
64    /// Receive the next vendor-neutral event from the session.
65    pub async fn next_event(&mut self) -> Option<RealtimeEvent> {
66        self.events_rx.recv().await
67    }
68
69    pub fn send_text(&self, text: impl Into<String>) -> Result<(), &'static str> {
70        self.commands_tx
71            .send(Command::SendText(text.into()))
72            .map_err(|_| "session closed")
73    }
74
75    pub fn send_audio_frame(&self, samples: Vec<i16>) -> Result<(), &'static str> {
76        self.commands_tx.send(Command::SendAudio(samples)).map_err(|_| "session closed")
77    }
78
79    pub fn commit_audio(&self) -> Result<(), &'static str> {
80        self.commands_tx.send(Command::CommitAudio).map_err(|_| "session closed")
81    }
82
83    pub fn interrupt(&self) -> Result<(), &'static str> {
84        self.commands_tx.send(Command::Interrupt).map_err(|_| "session closed")
85    }
86
87    pub fn update_instructions(&self, instructions: impl Into<String>) -> Result<(), &'static str> {
88        self.commands_tx
89            .send(Command::UpdateInstructions(instructions.into()))
90            .map_err(|_| "session closed")
91    }
92
93    pub fn close(&self) -> Result<(), &'static str> {
94        self.commands_tx.send(Command::Close).map_err(|_| "session closed")
95    }
96
97    /// Spawn a session driver that orchestrates the gateway. Returns
98    /// the session handle; the driver task lives until the gateway
99    /// session closes or [`RealtimeSession::close`] is called.
100    pub async fn run(
101        gateway: Box<dyn RealtimeGatewaySession>,
102        config: RealtimeVoiceConfig,
103        tool_runner: Arc<dyn ToolRunner>,
104        tracer: Option<Arc<TracerSystem>>,
105        correlation_id: String,
106    ) -> Self {
107        let (events_tx, events_rx) = mpsc::unbounded_channel();
108        let (commands_tx, commands_rx) = mpsc::unbounded_channel();
109
110        let driver = SessionDriver {
111            gateway,
112            config,
113            tool_runner,
114            tracer,
115            correlation_id,
116            events_tx,
117            current_turn: None,
118            current_response_id: None,
119        };
120
121        tokio::spawn(driver.run(commands_rx));
122
123        Self {
124            events_rx,
125            commands_tx,
126        }
127    }
128}
129
130/// Internal driver — owns the gateway session and pumps both inbound
131/// gateway events and outbound commands.
132struct SessionDriver {
133    gateway: Box<dyn RealtimeGatewaySession>,
134    config: RealtimeVoiceConfig,
135    tool_runner: Arc<dyn ToolRunner>,
136    tracer: Option<Arc<TracerSystem>>,
137    correlation_id: String,
138    events_tx: mpsc::UnboundedSender<RealtimeEvent>,
139    current_turn: Option<TurnState>,
140    current_response_id: Option<String>,
141}
142
143impl SessionDriver {
144    async fn run(mut self, mut commands_rx: mpsc::UnboundedReceiver<Command>) {
145        let initial = build_session_update(&self.config);
146        let _ = self.gateway.send_event(&initial).await;
147        let _ = self.events_tx.send(RealtimeEvent::SessionOpened {
148            session_id: self.gateway.session_id().to_string(),
149        });
150
151        loop {
152            tokio::select! {
153                cmd = commands_rx.recv() => {
154                    match cmd {
155                        Some(Command::SendText(text)) => self.send_text(text).await,
156                        Some(Command::SendAudio(samples)) => self.send_audio(samples).await,
157                        Some(Command::CommitAudio) => self.commit_audio().await,
158                        Some(Command::Interrupt) => self.cancel_turn(InterruptReason::Manual).await,
159                        Some(Command::UpdateInstructions(text)) => self.update_instructions(text).await,
160                        Some(Command::Close) | None => {
161                            let _ = self.gateway.close().await;
162                            let _ = self.events_tx.send(RealtimeEvent::SessionClosed { reason: SessionCloseReason::Client });
163                            return;
164                        }
165                    }
166                }
167                event = self.gateway.next_event() => {
168                    match event {
169                        Some(raw) => self.handle_server_event(raw).await,
170                        None => {
171                            let _ = self.events_tx.send(RealtimeEvent::SessionClosed { reason: SessionCloseReason::Server });
172                            return;
173                        }
174                    }
175                }
176            }
177        }
178    }
179
180    async fn send_text(&mut self, text: String) {
181        let _ = self
182            .gateway
183            .send_event(&json!({
184                "type": "conversation.item.create",
185                "item": {
186                    "type": "message",
187                    "role": "user",
188                    "content": [{ "type": "input_text", "text": text }]
189                }
190            }))
191            .await;
192        let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
193    }
194
195    async fn send_audio(&mut self, samples: Vec<i16>) {
196        let _ = self
197            .gateway
198            .send_event(&json!({
199                "type": "input_audio_buffer.append",
200                "audio": encode_base64_pcm16(&samples)
201            }))
202            .await;
203    }
204
205    async fn commit_audio(&mut self) {
206        let _ = self.gateway.send_event(&json!({ "type": "input_audio_buffer.commit" })).await;
207        let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
208    }
209
210    async fn update_instructions(&mut self, instructions: String) {
211        self.config.instructions = Some(instructions.clone());
212        let _ = self
213            .gateway
214            .send_event(&json!({
215                "type": "session.update",
216                "session": { "instructions": instructions }
217            }))
218            .await;
219    }
220
221    async fn cancel_turn(&mut self, reason: InterruptReason) {
222        let Some(turn) = self.current_turn.as_mut() else {
223            return;
224        };
225        if turn.cancelled {
226            return;
227        }
228        turn.cancelled = true;
229        turn.cancel_token.cancel();
230        let _ = self.events_tx.send(RealtimeEvent::Interrupted {
231            turn_id: turn.turn_id.clone(),
232            reason,
233        });
234        if self.current_response_id.is_some() {
235            let _ = self.gateway.send_event(&json!({ "type": "response.cancel" })).await;
236        }
237    }
238
239    async fn handle_server_event(&mut self, raw: Value) {
240        let Some(event_type) = raw.get("type").and_then(|v| v.as_str()) else {
241            return;
242        };
243
244        match event_type {
245            "session.created" => {}
246            "session.updated" => {
247                let _ = self.events_tx.send(RealtimeEvent::SessionUpdated {
248                    config: HashMap::new(),
249                });
250            }
251            "input_audio_buffer.speech_started" => {
252                let at_ms = raw.get("audio_start_ms").and_then(|v| v.as_u64()).unwrap_or(0);
253                let _ = self.events_tx.send(RealtimeEvent::UserSpeechStarted { at_ms });
254                if self.current_turn.is_some() {
255                    self.cancel_turn(InterruptReason::BargeIn).await;
256                }
257            }
258            "input_audio_buffer.speech_stopped" => {
259                let at_ms = raw.get("audio_end_ms").and_then(|v| v.as_u64()).unwrap_or(0);
260                let _ = self.events_tx.send(RealtimeEvent::UserSpeechStopped { at_ms });
261            }
262            "conversation.item.input_audio_transcription.delta" => {
263                self.emit_user_transcript_delta(&raw);
264            }
265            "conversation.item.input_audio_transcription.completed" => {
266                self.emit_user_transcript(&raw);
267            }
268            "response.created" => self.start_response(&raw),
269            "response.output_item.added" => self.add_output_item(&raw),
270            "response.function_call_arguments.delta" => self.fn_args_delta(&raw),
271            "response.function_call_arguments.done" => self.fn_args_done(&raw),
272            "response.text.delta" | "response.output_text.delta" => self.text_delta(&raw),
273            "response.text.done" | "response.output_text.done" => self.text_done(&raw),
274            "response.audio_transcript.delta" | "response.output_audio_transcript.delta" => {
275                self.transcript_delta(&raw)
276            }
277            "response.audio_transcript.done" | "response.output_audio_transcript.done" => {
278                self.transcript_done(&raw)
279            }
280            "response.audio.delta" | "response.output_audio.delta" => self.audio_delta(&raw),
281            "response.done" => self.response_done(&raw).await,
282            "rate_limits.updated" => {
283                let _ = self.events_tx.send(RealtimeEvent::RateLimited {
284                    reset_ms: 0,
285                    details: HashMap::new(),
286                });
287            }
288            "error" => self.emit_error(&raw),
289            _ => {}
290        }
291    }
292
293    fn emit_user_transcript_delta(&self, raw: &Value) {
294        if let (Some(id), Some(delta)) = (
295            raw.get("item_id").and_then(|v| v.as_str()),
296            raw.get("delta").and_then(|v| v.as_str()),
297        ) {
298            let _ = self.events_tx.send(RealtimeEvent::UserTranscriptDelta {
299                item_id: id.to_string(),
300                delta: delta.to_string(),
301            });
302        }
303    }
304
305    fn emit_user_transcript(&self, raw: &Value) {
306        if let (Some(id), Some(text)) = (
307            raw.get("item_id").and_then(|v| v.as_str()),
308            raw.get("transcript").and_then(|v| v.as_str()),
309        ) {
310            let _ = self.events_tx.send(RealtimeEvent::UserTranscript {
311                item_id: id.to_string(),
312                text: text.to_string(),
313            });
314        }
315    }
316
317    fn start_response(&mut self, raw: &Value) {
318        let Some(id) = raw.pointer("/response/id").and_then(|v| v.as_str()) else {
319            return;
320        };
321        let turn_id = id.to_string();
322        self.current_response_id = Some(turn_id.clone());
323        self.current_turn = Some(TurnState {
324            turn_id: turn_id.clone(),
325            calls: HashMap::new(),
326            cancelled: false,
327            cancel_token: CancellationToken::new(),
328        });
329        let _ = self.events_tx.send(RealtimeEvent::AssistantTurnStarted { turn_id });
330    }
331
332    fn add_output_item(&mut self, raw: &Value) {
333        let Some(turn) = self.current_turn.as_mut() else {
334            return;
335        };
336        let Some(item) = raw.get("item") else { return };
337        if item.get("type").and_then(|v| v.as_str()) != Some("function_call") {
338            return;
339        }
340        let (Some(call_id), Some(name)) = (
341            item.get("call_id").and_then(|v| v.as_str()),
342            item.get("name").and_then(|v| v.as_str()),
343        ) else {
344            return;
345        };
346        turn.calls.insert(
347            call_id.to_string(),
348            PendingCall {
349                name: name.to_string(),
350                args_buffer: String::new(),
351                done: false,
352            },
353        );
354        let _ = self.events_tx.send(RealtimeEvent::ToolCallStarted {
355            turn_id: turn.turn_id.clone(),
356            call_id: call_id.to_string(),
357            name: name.to_string(),
358        });
359    }
360
361    fn fn_args_delta(&mut self, raw: &Value) {
362        let Some(turn) = self.current_turn.as_mut() else {
363            return;
364        };
365        let (Some(call_id), Some(delta)) = (
366            raw.get("call_id").and_then(|v| v.as_str()),
367            raw.get("delta").and_then(|v| v.as_str()),
368        ) else {
369            return;
370        };
371        if let Some(call) = turn.calls.get_mut(call_id) {
372            call.args_buffer.push_str(delta);
373        }
374        let _ = self.events_tx.send(RealtimeEvent::ToolCallArgsDelta {
375            call_id: call_id.to_string(),
376            delta: delta.to_string(),
377        });
378    }
379
380    fn fn_args_done(&mut self, raw: &Value) {
381        let Some(turn) = self.current_turn.as_mut() else {
382            return;
383        };
384        let Some(call_id) = raw.get("call_id").and_then(|v| v.as_str()) else {
385            return;
386        };
387        if let Some(call) = turn.calls.get_mut(call_id) {
388            if let Some(args) = raw.get("arguments").and_then(|v| v.as_str()) {
389                call.args_buffer = args.to_string();
390            }
391            call.done = true;
392        }
393    }
394
395    fn text_delta(&self, raw: &Value) {
396        let Some(turn) = self.current_turn.as_ref() else {
397            return;
398        };
399        if let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) {
400            let _ = self.events_tx.send(RealtimeEvent::AssistantTextDelta {
401                turn_id: turn.turn_id.clone(),
402                delta: delta.to_string(),
403            });
404        }
405    }
406
407    fn text_done(&self, raw: &Value) {
408        let Some(turn) = self.current_turn.as_ref() else {
409            return;
410        };
411        if let Some(text) = raw.get("text").and_then(|v| v.as_str()) {
412            let _ = self.events_tx.send(RealtimeEvent::AssistantText {
413                turn_id: turn.turn_id.clone(),
414                text: text.to_string(),
415            });
416        }
417    }
418
419    fn transcript_delta(&self, raw: &Value) {
420        let Some(turn) = self.current_turn.as_ref() else {
421            return;
422        };
423        if let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) {
424            let _ = self.events_tx.send(RealtimeEvent::AssistantTranscriptDelta {
425                turn_id: turn.turn_id.clone(),
426                delta: delta.to_string(),
427            });
428        }
429    }
430
431    fn transcript_done(&self, raw: &Value) {
432        let Some(turn) = self.current_turn.as_ref() else {
433            return;
434        };
435        if let Some(text) = raw.get("transcript").and_then(|v| v.as_str()) {
436            let _ = self.events_tx.send(RealtimeEvent::AssistantTranscript {
437                turn_id: turn.turn_id.clone(),
438                text: text.to_string(),
439            });
440        }
441    }
442
443    fn audio_delta(&self, raw: &Value) {
444        let Some(turn) = self.current_turn.as_ref() else {
445            return;
446        };
447        let Some(delta) = raw.get("delta").and_then(|v| v.as_str()) else {
448            return;
449        };
450        if let Ok(pcm) = decode_base64_pcm16(delta) {
451            let _ = self.events_tx.send(RealtimeEvent::AssistantAudioDelta {
452                turn_id: turn.turn_id.clone(),
453                pcm,
454            });
455        }
456    }
457
458    async fn response_done(&mut self, raw: &Value) {
459        let Some(turn) = self.current_turn.take() else {
460            return;
461        };
462        let Some(response_id) = raw.pointer("/response/id").and_then(|v| v.as_str()) else {
463            return;
464        };
465        if turn.turn_id != response_id {
466            return;
467        }
468
469        let _ = self.events_tx.send(RealtimeEvent::AssistantTurnCompleted {
470            turn_id: turn.turn_id.clone(),
471            usage: token_usage_from(raw),
472        });
473
474        if turn.calls.is_empty() {
475            self.current_response_id = None;
476            return;
477        }
478
479        self.run_tool_batch(turn).await;
480        self.current_response_id = None;
481    }
482
483    fn emit_error(&self, raw: &Value) {
484        let message = raw
485            .pointer("/error/message")
486            .and_then(|v| v.as_str())
487            .unwrap_or("unknown realtime error");
488        if message.to_lowercase().contains("no active response") {
489            return;
490        }
491        let _ = self.events_tx.send(RealtimeEvent::Error {
492            error: message.to_string(),
493            recoverable: true,
494        });
495    }
496
497    async fn run_tool_batch(&mut self, turn: TurnState) {
498        let mut executions = Vec::new();
499        for (call_id, call) in &turn.calls {
500            if !call.done && call.args_buffer.is_empty() {
501                continue;
502            }
503            let args: HashMap<String, Value> = if call.args_buffer.is_empty() {
504                HashMap::new()
505            } else {
506                match serde_json::from_str::<HashMap<String, Value>>(&call.args_buffer) {
507                    Ok(a) => a,
508                    Err(err) => {
509                        let _ = self.events_tx.send(RealtimeEvent::ToolCallFailed {
510                            call_id: call_id.clone(),
511                            name: call.name.clone(),
512                            error: err.to_string(),
513                        });
514                        continue;
515                    }
516                }
517            };
518            executions.push(ToolCallExecution {
519                id: call_id.clone(),
520                name: call.name.clone(),
521                args,
522            });
523        }
524
525        if executions.is_empty() {
526            return;
527        }
528
529        let tools_vec: Vec<Box<dyn LlmTool>> = self
530            .config
531            .tools
532            .as_ref()
533            .map(|ts| ts.iter().map(|t| t.clone_box()).collect())
534            .unwrap_or_default();
535
536        for exec in &executions {
537            let _ = self.events_tx.send(RealtimeEvent::ToolCallDispatched {
538                call_id: exec.id.clone(),
539                name: exec.name.clone(),
540                args: exec.args.clone(),
541            });
542        }
543
544        let ctx = ToolRunCtx {
545            cancel: turn.cancel_token.clone(),
546            correlation_id: Some(self.correlation_id.clone()),
547            source: Some("RealtimeVoiceBroker".to_string()),
548        };
549
550        let outcomes = self.tool_runner.run_batch(&executions, &tools_vec, &ctx).await;
551
552        for outcome in &outcomes {
553            if outcome.ok {
554                let _ = self.events_tx.send(RealtimeEvent::ToolCallCompleted {
555                    call_id: outcome.id.clone(),
556                    name: outcome.name.clone(),
557                    result: outcome.result.clone().unwrap_or(Value::Null),
558                });
559            } else {
560                let _ = self.events_tx.send(RealtimeEvent::ToolCallFailed {
561                    call_id: outcome.id.clone(),
562                    name: outcome.name.clone(),
563                    error: outcome.error.clone().unwrap_or_default(),
564                });
565            }
566        }
567
568        if let Some(tracer) = &self.tracer {
569            for (exec, outcome) in executions.iter().zip(outcomes.iter()) {
570                tracer.record_tool_call(
571                    &outcome.name,
572                    exec.args.clone(),
573                    outcome.result.clone().unwrap_or(Value::Null),
574                    Some("RealtimeVoiceBroker".to_string()),
575                    Some(outcome.duration_ms as f64),
576                    "RealtimeVoiceBroker",
577                    &self.correlation_id,
578                );
579            }
580            let ok = outcomes.iter().filter(|o| o.ok).count();
581            let fail = outcomes.len() - ok;
582            tracer.record_tool_batch(
583                Uuid::new_v4().to_string(),
584                executions.iter().map(|e| e.name.clone()).collect(),
585                ok,
586                fail,
587                outcomes.iter().map(|o| o.duration_ms).sum::<u64>() as f64,
588                Some("RealtimeVoiceBroker".to_string()),
589                "RealtimeVoiceBroker",
590                &self.correlation_id,
591            );
592        }
593
594        let policy = self.config.on_interrupt.unwrap_or(defaults().on_interrupt);
595        let to_submit: Vec<_> = if turn.cancelled {
596            match policy {
597                InterruptOutputPolicy::Drop => Vec::new(),
598                InterruptOutputPolicy::Submit => outcomes.iter().collect(),
599                InterruptOutputPolicy::SubmitCompletedOnly => {
600                    outcomes.iter().filter(|o| o.ok).collect()
601                }
602            }
603        } else {
604            outcomes.iter().collect()
605        };
606
607        let mut submitted_ids = Vec::new();
608        for outcome in to_submit {
609            let output = if outcome.ok {
610                serde_json::to_string(outcome.result.as_ref().unwrap_or(&Value::Null))
611                    .unwrap_or_default()
612            } else {
613                json!({ "error": outcome.error.clone().unwrap_or_default() }).to_string()
614            };
615            let _ = self
616                .gateway
617                .send_event(&json!({
618                    "type": "conversation.item.create",
619                    "item": {
620                        "type": "function_call_output",
621                        "call_id": outcome.id,
622                        "output": output
623                    }
624                }))
625                .await;
626            submitted_ids.push(outcome.id.clone());
627        }
628
629        if !submitted_ids.is_empty() {
630            let _ = self.events_tx.send(RealtimeEvent::ToolBatchSubmitted {
631                turn_id: turn.turn_id.clone(),
632                call_ids: submitted_ids,
633            });
634            let _ = self.gateway.send_event(&json!({ "type": "response.create" })).await;
635        }
636    }
637}
638
639fn token_usage_from(raw: &Value) -> Option<TokenUsage> {
640    let usage = raw.pointer("/response/usage")?;
641    Some(TokenUsage {
642        prompt_tokens: usage.get("input_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
643        completion_tokens: usage.get("output_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
644        total_tokens: usage.get("total_tokens").and_then(|v| v.as_u64()).map(|n| n as u32),
645        extras: None,
646    })
647}
648
649/// Build the vendor-specific ``session.update`` payload from a
650/// vendor-neutral config. Matches the OpenAI Realtime GA shape.
651pub fn build_session_update(config: &RealtimeVoiceConfig) -> Value {
652    let modalities = config.modalities.clone().unwrap_or_else(|| defaults().modalities);
653
654    let output_modalities: Vec<&str> = if modalities.contains(&RealtimeModality::Audio) {
655        vec!["audio"]
656    } else {
657        vec!["text"]
658    };
659
660    let mut audio_input = serde_json::Map::new();
661    audio_input.insert(
662        "format".to_string(),
663        encode_audio_format(config.input_audio_format.unwrap_or(defaults().input_audio_format)),
664    );
665    audio_input.insert(
666        "turn_detection".to_string(),
667        encode_turn_detection(
668            config.turn_detection.as_ref().unwrap_or(&TurnDetectionMode::ServerVad),
669        ),
670    );
671
672    if config.disable_input_audio_transcription {
673        audio_input.insert("transcription".to_string(), Value::Null);
674    } else if let Some(t) = config.input_audio_transcription.as_ref() {
675        audio_input.insert("transcription".to_string(), json!({ "model": t.model }));
676    }
677
678    let mut audio_output = serde_json::Map::new();
679    audio_output.insert(
680        "format".to_string(),
681        encode_audio_format(config.output_audio_format.unwrap_or(defaults().output_audio_format)),
682    );
683    if let Some(voice) = &config.voice {
684        audio_output.insert("voice".to_string(), Value::String(voice.clone()));
685    }
686
687    let mut session = serde_json::Map::new();
688    session.insert("type".to_string(), Value::String("realtime".to_string()));
689    session.insert(
690        "output_modalities".to_string(),
691        Value::Array(output_modalities.iter().map(|s| Value::String(s.to_string())).collect()),
692    );
693    session.insert("audio".to_string(), json!({ "input": audio_input, "output": audio_output }));
694    session.insert(
695        "tool_choice".to_string(),
696        encode_tool_choice(config.tool_choice.as_ref().unwrap_or(&RealtimeToolChoice::Auto)),
697    );
698    if let Some(instr) = &config.instructions {
699        session.insert("instructions".to_string(), Value::String(instr.clone()));
700    }
701    if let Some(max) = config.max_response_output_tokens {
702        session.insert("max_output_tokens".to_string(), Value::from(max));
703    }
704    if let Some(tools) = &config.tools {
705        let tool_list: Vec<Value> = tools
706            .iter()
707            .map(|t| {
708                let d = t.descriptor();
709                json!({
710                    "type": "function",
711                    "name": d.function.name,
712                    "description": d.function.description,
713                    "parameters": d.function.parameters
714                })
715            })
716            .collect();
717        session.insert("tools".to_string(), Value::Array(tool_list));
718    }
719    if let Some(extras) = &config.provider_extras {
720        for (k, v) in extras {
721            session.insert(k.clone(), v.clone());
722        }
723    }
724
725    json!({ "type": "session.update", "session": session })
726}
727
728fn encode_audio_format(fmt: RealtimeAudioFormat) -> Value {
729    match fmt {
730        RealtimeAudioFormat::Pcm16 => json!({ "type": "audio/pcm", "rate": 24000 }),
731        RealtimeAudioFormat::G711Ulaw => json!({ "type": "audio/pcmu" }),
732        RealtimeAudioFormat::G711Alaw => json!({ "type": "audio/pcma" }),
733    }
734}
735
736fn encode_turn_detection(mode: &TurnDetectionMode) -> Value {
737    match mode {
738        TurnDetectionMode::None => Value::Null,
739        TurnDetectionMode::ServerVad => json!({ "type": "server_vad" }),
740        TurnDetectionMode::SemanticVad => json!({ "type": "semantic_vad" }),
741        TurnDetectionMode::ServerVadCustom(vad) => encode_server_vad(vad),
742        TurnDetectionMode::SemanticVadCustom(vad) => encode_semantic_vad(vad),
743    }
744}
745
746fn encode_server_vad(vad: &ServerVadConfig) -> Value {
747    let mut map = serde_json::Map::new();
748    map.insert("type".to_string(), Value::String("server_vad".to_string()));
749    if let Some(t) = vad.threshold {
750        map.insert("threshold".to_string(), Value::from(t));
751    }
752    if let Some(p) = vad.prefix_padding_ms {
753        map.insert("prefix_padding_ms".to_string(), Value::from(p));
754    }
755    if let Some(s) = vad.silence_duration_ms {
756        map.insert("silence_duration_ms".to_string(), Value::from(s));
757    }
758    if let Some(c) = vad.create_response {
759        map.insert("create_response".to_string(), Value::from(c));
760    }
761    if let Some(i) = vad.interrupt_response {
762        map.insert("interrupt_response".to_string(), Value::from(i));
763    }
764    if let Some(t) = vad.idle_timeout_ms {
765        map.insert("idle_timeout_ms".to_string(), Value::from(t));
766    }
767    Value::Object(map)
768}
769
770fn encode_semantic_vad(vad: &SemanticVadConfig) -> Value {
771    let mut map = serde_json::Map::new();
772    map.insert("type".to_string(), Value::String("semantic_vad".to_string()));
773    if let Some(e) = &vad.eagerness {
774        map.insert("eagerness".to_string(), Value::String(e.clone()));
775    }
776    Value::Object(map)
777}
778
779fn encode_tool_choice(choice: &RealtimeToolChoice) -> Value {
780    match choice {
781        RealtimeToolChoice::Auto => Value::String("auto".to_string()),
782        RealtimeToolChoice::None => Value::String("none".to_string()),
783        RealtimeToolChoice::Required => Value::String("required".to_string()),
784        RealtimeToolChoice::Function(name) => json!({ "type": "function", "name": name }),
785    }
786}
787
788#[cfg(test)]
789mod tests {
790    use super::*;
791
792    #[test]
793    fn build_session_update_text_only() {
794        let cfg = RealtimeVoiceConfig {
795            modalities: Some(vec![RealtimeModality::Text]),
796            instructions: Some("be brief".to_string()),
797            ..Default::default()
798        };
799
800        let payload = build_session_update(&cfg);
801
802        assert_eq!(payload["type"], "session.update");
803        assert_eq!(payload["session"]["type"], "realtime");
804        assert_eq!(payload["session"]["output_modalities"], json!(["text"]));
805        assert_eq!(payload["session"]["instructions"], "be brief");
806    }
807
808    #[test]
809    fn build_session_update_includes_voice() {
810        let cfg = RealtimeVoiceConfig {
811            voice: Some("verse".to_string()),
812            ..Default::default()
813        };
814
815        let payload = build_session_update(&cfg);
816
817        assert_eq!(payload["session"]["audio"]["output"]["voice"], "verse");
818    }
819}