mojentic/llm/
stream_events.rs1use crate::error::MojenticError;
12use crate::llm::models::ResponseEvidence;
13use futures::stream::{Stream, StreamExt};
14use serde_json::Value;
15use std::pin::Pin;
16use thiserror::Error;
17
18pub type StreamEventStream<'a> = Pin<Box<dyn Stream<Item = StreamEvent> + Send + 'a>>;
20
21#[derive(Debug)]
23#[non_exhaustive]
24pub enum StreamEvent {
25 Content(String),
27 Metrics(crate::llm::gateway::StreamMetrics),
29 Completed(ResponseEvidence),
34 Error(StreamEventError),
36}
37
38impl StreamEvent {
39 pub fn is_terminal(&self) -> bool {
41 matches!(self, StreamEvent::Completed(_) | StreamEvent::Error(_))
42 }
43}
44
45#[derive(Debug, Error)]
52#[non_exhaustive]
53pub enum StreamEventError {
54 #[error("incomplete completion (finish reason: {})", .0.finish_reason.as_deref().unwrap_or("unknown"))]
59 IncompleteCompletion(Box<ResponseEvidence>),
60
61 #[error("stream ended without a terminal marker")]
65 IncompleteStream(Option<Box<ResponseEvidence>>),
66
67 #[error("provider error{}: {error}", .status.map(|code| format!(" (HTTP {code})")).unwrap_or_default())]
70 ProviderError {
71 status: Option<u16>,
73 error: Value,
76 },
77
78 #[error("unexpected tool calls in stream")]
81 UnexpectedToolCalls,
82
83 #[error("invalid stream event: {0}")]
85 InvalidStreamEvent(String),
86
87 #[error("stream events are not supported by this gateway")]
89 StreamEventsUnsupported,
90
91 #[error("stream request failed: {0}")]
93 RequestFailed(#[source] MojenticError),
94}
95
96impl StreamEventError {
97 pub(crate) fn provider_frame(error: Value) -> Self {
99 StreamEventError::ProviderError {
100 status: None,
101 error,
102 }
103 }
104
105 fn provider_status(status: u16, body: &str) -> Self {
107 let error = match serde_json::from_str::<Value>(body) {
108 Ok(Value::Object(mut fields)) if fields.contains_key("error") => {
109 fields.remove("error").unwrap_or(Value::Null)
110 }
111 Ok(value) => value,
112 Err(_) => Value::String(body.to_string()),
113 };
114 StreamEventError::ProviderError {
115 status: Some(status),
116 error,
117 }
118 }
119}
120
121pub(crate) trait FrameParser {
126 fn parse_line(&mut self, line: &str) -> Vec<StreamEvent>;
127
128 fn partial_evidence(&self) -> Option<ResponseEvidence>;
130}
131
132pub(crate) fn drive_event_stream<'a, P>(
137 request: reqwest::RequestBuilder,
138 mut parser: P,
139 clock: std::sync::Arc<dyn crate::llm::stream_timing::StreamClock>,
140 provider: &'static str,
141) -> StreamEventStream<'a>
142where
143 P: FrameParser + Send + 'a,
144{
145 let timing = crate::llm::stream_timing::StreamTiming::new(clock);
146 let measure = timing.clone();
147 let mut source: StreamEventStream<'a> = Box::pin(async_stream::stream! {
148 measure.dispatch();
149 let response = match request.send().await {
150 Ok(response) => response,
151 Err(error) => {
152 yield StreamEvent::Error(StreamEventError::RequestFailed(error.into()));
153 return;
154 }
155 };
156
157 let status = response.status();
158 if !status.is_success() {
159 let body = response.text().await.unwrap_or_default();
160 yield StreamEvent::Error(StreamEventError::provider_status(status.as_u16(), &body));
161 return;
162 }
163
164 let mut bytes = response.bytes_stream();
165 let mut lines = LineBuffer::default();
166 while let Some(chunk) = bytes.next().await {
167 let chunk = match chunk {
168 Ok(chunk) => chunk,
169 Err(error) => {
170 yield StreamEvent::Error(StreamEventError::RequestFailed(error.into()));
171 return;
172 }
173 };
174 for line in lines.push(&chunk) {
175 if let Ok(text) = std::str::from_utf8(&line) { measure.observe_line(text); }
176 let events = match std::str::from_utf8(&line) {
177 Ok(text) => parser.parse_line(text.trim_end_matches('\r')),
178 Err(_) => vec![invalid_event("frame is not valid UTF-8")],
179 };
180 for event in events {
181 let terminal = event.is_terminal();
182 yield event;
183 if terminal {
184 return;
185 }
186 }
187 }
188 }
189
190 yield StreamEvent::Error(StreamEventError::IncompleteStream(
191 parser.partial_evidence().map(Box::new),
192 ));
193 });
194 Box::pin(async_stream::stream! {
195 while let Some(mut event) = source.next().await {
196 if event.is_terminal() {
197 let metrics = timing.finish(crate::llm::gateway::StreamMetrics { provider: provider.into(), ..Default::default() });
198 let evidence = match &mut event {
199 StreamEvent::Completed(evidence) => Some(evidence),
200 StreamEvent::Error(StreamEventError::IncompleteCompletion(evidence)) => Some(evidence.as_mut()),
201 StreamEvent::Error(StreamEventError::IncompleteStream(Some(evidence))) => Some(evidence.as_mut()),
202 _ => None,
203 };
204 if let Some(evidence) = evidence {
205 crate::llm::stream_timing::retain_timing(evidence, &metrics);
206 }
207 yield StreamEvent::Metrics(metrics);
208 }
209 yield event;
210 }
211 })
212}
213
214pub(crate) fn invalid_event(reason: impl Into<String>) -> StreamEvent {
216 StreamEvent::Error(StreamEventError::InvalidStreamEvent(reason.into()))
217}
218
219#[derive(Default)]
226pub(crate) struct LineBuffer {
227 pending: Vec<u8>,
228}
229
230impl LineBuffer {
231 pub(crate) fn has_pending(&self) -> bool {
232 !self.pending.is_empty()
233 }
234
235 pub(crate) fn push(&mut self, chunk: &[u8]) -> Vec<Vec<u8>> {
236 self.pending.extend_from_slice(chunk);
237 let Some(last_newline) = self.pending.iter().rposition(|byte| *byte == b'\n') else {
238 return Vec::new();
239 };
240 let rest = self.pending.split_off(last_newline + 1);
241 let complete = std::mem::replace(&mut self.pending, rest);
242 complete[..complete.len() - 1]
243 .split(|byte| *byte == b'\n')
244 .map(<[u8]>::to_vec)
245 .collect()
246 }
247}
248
249#[cfg(test)]
250pub(crate) mod testing {
251 use tokio::io::{AsyncReadExt, AsyncWriteExt};
254 use tokio::net::TcpListener;
255 use tokio::sync::oneshot;
256
257 pub(crate) async fn endless_stream_server(
262 first_frame: &'static str,
263 ) -> (String, oneshot::Receiver<()>) {
264 let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
265 let address = listener.local_addr().expect("test server address");
266 let (closed_tx, closed_rx) = oneshot::channel();
267
268 tokio::spawn(async move {
269 let (mut socket, _) = listener.accept().await.expect("accept client");
270 read_request(&mut socket).await;
271 let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n";
272 let chunk = format!("{:x}\r\n{}\r\n", first_frame.len(), first_frame);
273 socket.write_all(head.as_bytes()).await.expect("write head");
274 socket.write_all(chunk.as_bytes()).await.expect("write first frame");
275
276 let mut buffer = [0u8; 64];
278 while let Ok(read) = socket.read(&mut buffer).await {
279 if read == 0 {
280 break;
281 }
282 }
283 let _ = closed_tx.send(());
284 });
285
286 (format!("http://{address}"), closed_rx)
287 }
288
289 pub(crate) async fn split_body_server(parts: Vec<Vec<u8>>) -> String {
294 let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
295 let address = listener.local_addr().expect("test server address");
296
297 tokio::spawn(async move {
298 let (mut socket, _) = listener.accept().await.expect("accept client");
299 read_request(&mut socket).await;
300 let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n";
301 socket.write_all(head.as_bytes()).await.expect("write head");
302 for part in parts {
303 let mut chunk = format!("{:x}\r\n", part.len()).into_bytes();
304 chunk.extend_from_slice(&part);
305 chunk.extend_from_slice(b"\r\n");
306 socket.write_all(&chunk).await.expect("write part");
307 socket.flush().await.expect("flush part");
308 tokio::time::sleep(std::time::Duration::from_millis(30)).await;
309 }
310 socket.write_all(b"0\r\n\r\n").await.expect("write end");
311 socket.flush().await.expect("flush end");
312 });
313
314 format!("http://{address}")
315 }
316
317 pub(crate) fn split_inside_first_multibyte_char(body: &str) -> Vec<Vec<u8>> {
319 let (index, _) = body
320 .char_indices()
321 .find(|(_, c)| c.len_utf8() > 1)
322 .expect("body has a multi-byte character");
323 let bytes = body.as_bytes();
324 vec![bytes[..index + 1].to_vec(), bytes[index + 1..].to_vec()]
325 }
326
327 async fn read_request(socket: &mut tokio::net::TcpStream) {
328 let mut request = Vec::new();
329 let mut buffer = [0u8; 4096];
330 loop {
331 let read = socket.read(&mut buffer).await.expect("read request");
332 request.extend_from_slice(&buffer[..read]);
333 let text = String::from_utf8_lossy(&request);
334 if let Some(head_end) = text.find("\r\n\r\n") {
335 let length = text[..head_end]
336 .lines()
337 .find_map(|line| {
338 let (name, value) = line.split_once(':')?;
339 name.eq_ignore_ascii_case("content-length")
340 .then(|| value.trim().parse::<usize>().ok())?
341 })
342 .unwrap_or(0);
343 if request.len() >= head_end + 4 + length {
344 return;
345 }
346 }
347 if read == 0 {
348 return;
349 }
350 }
351 }
352}
353
354#[cfg(test)]
355mod tests {
356 use super::*;
357
358 #[test]
359 fn line_buffer_returns_only_complete_lines() {
360 let mut lines = LineBuffer::default();
361
362 assert!(lines.push(b"data: {\"a\"").is_empty());
363 assert_eq!(lines.push(b":1}\n\ndata: [DO"), vec![b"data: {\"a\":1}".to_vec(), vec![]]);
364 assert_eq!(lines.push(b"NE]\n"), vec![b"data: [DONE]".to_vec()]);
365 }
366
367 #[test]
368 fn line_buffer_keeps_utf8_sequences_split_across_chunks() {
369 let mut lines = LineBuffer::default();
370 let text = "héllo\n".as_bytes();
371
372 assert!(lines.push(&text[..2]).is_empty());
373 let complete = lines.push(&text[2..]);
374
375 assert_eq!(complete, vec!["héllo".as_bytes().to_vec()]);
376 }
377
378 #[test]
379 fn only_content_is_non_terminal() {
380 assert!(!StreamEvent::Content("x".into()).is_terminal());
381 assert!(StreamEvent::Completed(ResponseEvidence::default()).is_terminal());
382 assert!(StreamEvent::Error(StreamEventError::IncompleteStream(None)).is_terminal());
383 }
384
385 #[test]
386 fn http_provider_error_keeps_status_and_error_field() {
387 let error = StreamEventError::provider_status(429, r#"{"error":{"message":"slow"}}"#);
388
389 assert!(matches!(
390 &error,
391 StreamEventError::ProviderError { status: Some(429), error } if error["message"] == "slow"
392 ));
393 assert!(error.to_string().starts_with("provider error (HTTP 429)"));
394 }
395
396 #[test]
397 fn http_provider_error_keeps_a_non_json_body_as_text() {
398 let error = StreamEventError::provider_status(502, "Bad Gateway");
399
400 assert!(matches!(
401 error,
402 StreamEventError::ProviderError { status: Some(502), error } if error == "Bad Gateway"
403 ));
404 }
405
406 #[test]
407 fn incomplete_completion_names_its_finish_reason() {
408 let error = StreamEventError::IncompleteCompletion(Box::new(ResponseEvidence {
409 finish_reason: Some("length".into()),
410 ..Default::default()
411 }));
412
413 assert_eq!(error.to_string(), "incomplete completion (finish reason: length)");
414 }
415}