Skip to main content

mojentic/llm/
stream_events.rs

1//! Single-turn streaming with terminal completion evidence.
2//!
3//! [`crate::llm::LlmBroker::generate_stream_events`] yields [`StreamEvent`]s for
4//! one model turn. Every stream ends with exactly one terminal event:
5//! [`StreamEvent::Completed`] when the provider reported a clean stop, or
6//! [`StreamEvent::Error`] otherwise. Nothing follows the terminal event.
7//!
8//! Content yielded before an error is evidence of what the provider sent, not
9//! a result. Truncated or unfinished output is always an error.
10
11use crate::error::MojenticError;
12use crate::llm::models::ResponseEvidence;
13use futures::stream::{Stream, StreamExt};
14use serde_json::Value;
15use std::pin::Pin;
16use thiserror::Error;
17
18/// A boxed stream of [`StreamEvent`]s for one model turn.
19pub type StreamEventStream<'a> = Pin<Box<dyn Stream<Item = StreamEvent> + Send + 'a>>;
20
21/// One event in a single-turn event stream.
22#[derive(Debug)]
23#[non_exhaustive]
24pub enum StreamEvent {
25    /// Visible assistant content, in the order the provider sent it.
26    Content(String),
27    /// Client timing emitted immediately before terminal evidence, including errors.
28    Metrics(crate::llm::gateway::StreamMetrics),
29    /// Terminal success: the provider finished with `stop` and closed the stream.
30    ///
31    /// The evidence holds the finish reason, usage and provider model when
32    /// reported, plus other provider metadata such as Ollama durations.
33    Completed(ResponseEvidence),
34    /// Terminal failure. Content yielded before it is not a complete result.
35    Error(StreamEventError),
36}
37
38impl StreamEvent {
39    /// Whether this event ends the stream.
40    pub fn is_terminal(&self) -> bool {
41        matches!(self, StreamEvent::Completed(_) | StreamEvent::Error(_))
42    }
43}
44
45/// Why a single-turn event stream ended without a completed response.
46///
47/// The variants follow the cross-port reason vocabulary:
48/// `incomplete_completion`, `incomplete_stream`, `provider_error`,
49/// `unexpected_tool_calls`, `invalid_stream_event`,
50/// `stream_events_unsupported` and `request_failed`.
51#[derive(Debug, Error)]
52#[non_exhaustive]
53pub enum StreamEventError {
54    /// The provider finished for a reason other than `stop`, for example
55    /// `length`. Carries the finish reason, usage, provider model and metadata.
56    ///
57    /// Ollama servers too old to send `done_reason` always end here.
58    #[error("incomplete completion (finish reason: {})", .0.finish_reason.as_deref().unwrap_or("unknown"))]
59    IncompleteCompletion(Box<ResponseEvidence>),
60
61    /// The stream ended without the provider's terminal marker. Carries the
62    /// evidence that arrived before the end (such as model and usage), or
63    /// `None` when none did.
64    #[error("stream ended without a terminal marker")]
65    IncompleteStream(Option<Box<ResponseEvidence>>),
66
67    /// The provider reported an error, either as an error frame in the stream
68    /// or as a non-success HTTP status.
69    #[error("provider error{}: {error}", .status.map(|code| format!(" (HTTP {code})")).unwrap_or_default())]
70    ProviderError {
71        /// HTTP status, when the provider rejected the request with one.
72        status: Option<u16>,
73        /// The provider's error value: the frame's `error` field, or the
74        /// response body (its `error` field when it has one).
75        error: Value,
76    },
77
78    /// The provider streamed a native tool call. The events API supplies no
79    /// tools and executes none.
80    #[error("unexpected tool calls in stream")]
81    UnexpectedToolCalls,
82
83    /// A frame could not be read as a provider stream event.
84    #[error("invalid stream event: {0}")]
85    InvalidStreamEvent(String),
86
87    /// The gateway does not implement the events API. No request was sent.
88    #[error("stream events are not supported by this gateway")]
89    StreamEventsUnsupported,
90
91    /// Building the request, connecting, or reading the response body failed.
92    #[error("stream request failed: {0}")]
93    RequestFailed(#[source] MojenticError),
94}
95
96impl StreamEventError {
97    /// A provider error for an error frame inside the stream.
98    pub(crate) fn provider_frame(error: Value) -> Self {
99        StreamEventError::ProviderError {
100            status: None,
101            error,
102        }
103    }
104
105    /// A provider error for a non-success HTTP response.
106    fn provider_status(status: u16, body: &str) -> Self {
107        let error = match serde_json::from_str::<Value>(body) {
108            Ok(Value::Object(mut fields)) if fields.contains_key("error") => {
109                fields.remove("error").unwrap_or(Value::Null)
110            }
111            Ok(value) => value,
112            Err(_) => Value::String(body.to_string()),
113        };
114        StreamEventError::ProviderError {
115            status: Some(status),
116            error,
117        }
118    }
119}
120
121/// Turns one provider stream line into zero or more events.
122///
123/// A parser owns the provider's completion rules. It returns a terminal event
124/// when the line settles the outcome; the driver stops reading after it.
125pub(crate) trait FrameParser {
126    fn parse_line(&mut self, line: &str) -> Vec<StreamEvent>;
127
128    /// Evidence seen so far, for a stream that ends without a terminal marker.
129    fn partial_evidence(&self) -> Option<ResponseEvidence>;
130}
131
132/// Send one streaming request and translate its lines into events.
133///
134/// The returned stream owns the HTTP response. Dropping the stream drops the
135/// response body, which closes the connection and cancels the request.
136pub(crate) fn drive_event_stream<'a, P>(
137    request: reqwest::RequestBuilder,
138    mut parser: P,
139    clock: std::sync::Arc<dyn crate::llm::stream_timing::StreamClock>,
140    provider: &'static str,
141) -> StreamEventStream<'a>
142where
143    P: FrameParser + Send + 'a,
144{
145    let timing = crate::llm::stream_timing::StreamTiming::new(clock);
146    let measure = timing.clone();
147    let mut source: StreamEventStream<'a> = Box::pin(async_stream::stream! {
148        measure.dispatch();
149        let response = match request.send().await {
150            Ok(response) => response,
151            Err(error) => {
152                yield StreamEvent::Error(StreamEventError::RequestFailed(error.into()));
153                return;
154            }
155        };
156
157        let status = response.status();
158        if !status.is_success() {
159            let body = response.text().await.unwrap_or_default();
160            yield StreamEvent::Error(StreamEventError::provider_status(status.as_u16(), &body));
161            return;
162        }
163
164        let mut bytes = response.bytes_stream();
165        let mut lines = LineBuffer::default();
166        while let Some(chunk) = bytes.next().await {
167            let chunk = match chunk {
168                Ok(chunk) => chunk,
169                Err(error) => {
170                    yield StreamEvent::Error(StreamEventError::RequestFailed(error.into()));
171                    return;
172                }
173            };
174            for line in lines.push(&chunk) {
175                if let Ok(text) = std::str::from_utf8(&line) { measure.observe_line(text); }
176                let events = match std::str::from_utf8(&line) {
177                    Ok(text) => parser.parse_line(text.trim_end_matches('\r')),
178                    Err(_) => vec![invalid_event("frame is not valid UTF-8")],
179                };
180                for event in events {
181                    let terminal = event.is_terminal();
182                    yield event;
183                    if terminal {
184                        return;
185                    }
186                }
187            }
188        }
189
190        yield StreamEvent::Error(StreamEventError::IncompleteStream(
191            parser.partial_evidence().map(Box::new),
192        ));
193    });
194    Box::pin(async_stream::stream! {
195        while let Some(mut event) = source.next().await {
196            if event.is_terminal() {
197                let metrics = timing.finish(crate::llm::gateway::StreamMetrics { provider: provider.into(), ..Default::default() });
198                let evidence = match &mut event {
199                    StreamEvent::Completed(evidence) => Some(evidence),
200                    StreamEvent::Error(StreamEventError::IncompleteCompletion(evidence)) => Some(evidence.as_mut()),
201                    StreamEvent::Error(StreamEventError::IncompleteStream(Some(evidence))) => Some(evidence.as_mut()),
202                    _ => None,
203                };
204                if let Some(evidence) = evidence {
205                    crate::llm::stream_timing::retain_timing(evidence, &metrics);
206                }
207                yield StreamEvent::Metrics(metrics);
208            }
209            yield event;
210        }
211    })
212}
213
214/// An [`StreamEvent::Error`] for a frame that is not a valid provider event.
215pub(crate) fn invalid_event(reason: impl Into<String>) -> StreamEvent {
216    StreamEvent::Error(StreamEventError::InvalidStreamEvent(reason.into()))
217}
218
219/// Splits a byte stream into complete newline-terminated lines.
220///
221/// Bytes after the last newline wait for the next chunk, so a frame split
222/// across network reads (including inside a UTF-8 sequence) stays intact.
223/// A trailing partial line at end of stream is never returned: a stream cut
224/// mid-frame is incomplete, not malformed.
225#[derive(Default)]
226pub(crate) struct LineBuffer {
227    pending: Vec<u8>,
228}
229
230impl LineBuffer {
231    pub(crate) fn has_pending(&self) -> bool {
232        !self.pending.is_empty()
233    }
234
235    pub(crate) fn push(&mut self, chunk: &[u8]) -> Vec<Vec<u8>> {
236        self.pending.extend_from_slice(chunk);
237        let Some(last_newline) = self.pending.iter().rposition(|byte| *byte == b'\n') else {
238            return Vec::new();
239        };
240        let rest = self.pending.split_off(last_newline + 1);
241        let complete = std::mem::replace(&mut self.pending, rest);
242        complete[..complete.len() - 1]
243            .split(|byte| *byte == b'\n')
244            .map(<[u8]>::to_vec)
245            .collect()
246    }
247}
248
249#[cfg(test)]
250pub(crate) mod testing {
251    //! A local HTTP server that streams forever, for cancellation tests.
252
253    use tokio::io::{AsyncReadExt, AsyncWriteExt};
254    use tokio::net::TcpListener;
255    use tokio::sync::oneshot;
256
257    /// Serve one chunked response that sends `first_frame` and then never ends.
258    ///
259    /// Returns the base URL and a receiver that resolves when the client
260    /// closes the connection.
261    pub(crate) async fn endless_stream_server(
262        first_frame: &'static str,
263    ) -> (String, oneshot::Receiver<()>) {
264        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
265        let address = listener.local_addr().expect("test server address");
266        let (closed_tx, closed_rx) = oneshot::channel();
267
268        tokio::spawn(async move {
269            let (mut socket, _) = listener.accept().await.expect("accept client");
270            read_request(&mut socket).await;
271            let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n";
272            let chunk = format!("{:x}\r\n{}\r\n", first_frame.len(), first_frame);
273            socket.write_all(head.as_bytes()).await.expect("write head");
274            socket.write_all(chunk.as_bytes()).await.expect("write first frame");
275
276            // The response never ends. A read of zero bytes means the client hung up.
277            let mut buffer = [0u8; 64];
278            while let Ok(read) = socket.read(&mut buffer).await {
279                if read == 0 {
280                    break;
281                }
282            }
283            let _ = closed_tx.send(());
284        });
285
286        (format!("http://{address}"), closed_rx)
287    }
288
289    /// Serve one chunked response that sends each of `parts` as its own
290    /// network write, pausing between them so the client reads them apart.
291    ///
292    /// Returns the base URL.
293    pub(crate) async fn split_body_server(parts: Vec<Vec<u8>>) -> String {
294        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
295        let address = listener.local_addr().expect("test server address");
296
297        tokio::spawn(async move {
298            let (mut socket, _) = listener.accept().await.expect("accept client");
299            read_request(&mut socket).await;
300            let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n";
301            socket.write_all(head.as_bytes()).await.expect("write head");
302            for part in parts {
303                let mut chunk = format!("{:x}\r\n", part.len()).into_bytes();
304                chunk.extend_from_slice(&part);
305                chunk.extend_from_slice(b"\r\n");
306                socket.write_all(&chunk).await.expect("write part");
307                socket.flush().await.expect("flush part");
308                tokio::time::sleep(std::time::Duration::from_millis(30)).await;
309            }
310            socket.write_all(b"0\r\n\r\n").await.expect("write end");
311            socket.flush().await.expect("flush end");
312        });
313
314        format!("http://{address}")
315    }
316
317    /// `body` split into two parts inside the first multi-byte UTF-8 character.
318    pub(crate) fn split_inside_first_multibyte_char(body: &str) -> Vec<Vec<u8>> {
319        let (index, _) = body
320            .char_indices()
321            .find(|(_, c)| c.len_utf8() > 1)
322            .expect("body has a multi-byte character");
323        let bytes = body.as_bytes();
324        vec![bytes[..index + 1].to_vec(), bytes[index + 1..].to_vec()]
325    }
326
327    async fn read_request(socket: &mut tokio::net::TcpStream) {
328        let mut request = Vec::new();
329        let mut buffer = [0u8; 4096];
330        loop {
331            let read = socket.read(&mut buffer).await.expect("read request");
332            request.extend_from_slice(&buffer[..read]);
333            let text = String::from_utf8_lossy(&request);
334            if let Some(head_end) = text.find("\r\n\r\n") {
335                let length = text[..head_end]
336                    .lines()
337                    .find_map(|line| {
338                        let (name, value) = line.split_once(':')?;
339                        name.eq_ignore_ascii_case("content-length")
340                            .then(|| value.trim().parse::<usize>().ok())?
341                    })
342                    .unwrap_or(0);
343                if request.len() >= head_end + 4 + length {
344                    return;
345                }
346            }
347            if read == 0 {
348                return;
349            }
350        }
351    }
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn line_buffer_returns_only_complete_lines() {
360        let mut lines = LineBuffer::default();
361
362        assert!(lines.push(b"data: {\"a\"").is_empty());
363        assert_eq!(lines.push(b":1}\n\ndata: [DO"), vec![b"data: {\"a\":1}".to_vec(), vec![]]);
364        assert_eq!(lines.push(b"NE]\n"), vec![b"data: [DONE]".to_vec()]);
365    }
366
367    #[test]
368    fn line_buffer_keeps_utf8_sequences_split_across_chunks() {
369        let mut lines = LineBuffer::default();
370        let text = "héllo\n".as_bytes();
371
372        assert!(lines.push(&text[..2]).is_empty());
373        let complete = lines.push(&text[2..]);
374
375        assert_eq!(complete, vec!["héllo".as_bytes().to_vec()]);
376    }
377
378    #[test]
379    fn only_content_is_non_terminal() {
380        assert!(!StreamEvent::Content("x".into()).is_terminal());
381        assert!(StreamEvent::Completed(ResponseEvidence::default()).is_terminal());
382        assert!(StreamEvent::Error(StreamEventError::IncompleteStream(None)).is_terminal());
383    }
384
385    #[test]
386    fn http_provider_error_keeps_status_and_error_field() {
387        let error = StreamEventError::provider_status(429, r#"{"error":{"message":"slow"}}"#);
388
389        assert!(matches!(
390            &error,
391            StreamEventError::ProviderError { status: Some(429), error } if error["message"] == "slow"
392        ));
393        assert!(error.to_string().starts_with("provider error (HTTP 429)"));
394    }
395
396    #[test]
397    fn http_provider_error_keeps_a_non_json_body_as_text() {
398        let error = StreamEventError::provider_status(502, "Bad Gateway");
399
400        assert!(matches!(
401            error,
402            StreamEventError::ProviderError { status: Some(502), error } if error == "Bad Gateway"
403        ));
404    }
405
406    #[test]
407    fn incomplete_completion_names_its_finish_reason() {
408        let error = StreamEventError::IncompleteCompletion(Box::new(ResponseEvidence {
409            finish_reason: Some("length".into()),
410            ..Default::default()
411        }));
412
413        assert_eq!(error.to_string(), "incomplete completion (finish reason: length)");
414    }
415}