mojentic/llm/
stream_timing.rs1use super::gateway::{StreamChunk, StreamMetrics};
3use crate::error::Result;
4use futures::{Stream, StreamExt};
5use serde_json::Value;
6use std::{
7 pin::Pin,
8 sync::{Arc, Mutex},
9 time::{Duration, Instant},
10};
11
12pub trait StreamClock: Send + Sync {
18 fn now(&self) -> Duration;
20}
21struct MonotonicClock(Instant);
22impl StreamClock for MonotonicClock {
23 fn now(&self) -> Duration {
24 self.0.elapsed()
25 }
26}
27pub(crate) fn system_clock() -> Arc<dyn StreamClock> {
28 Arc::new(MonotonicClock(Instant::now()))
29}
30
31#[derive(Default)]
32struct Measurement {
33 start: Option<Duration>,
34 first: Option<(u64, &'static str)>,
35 duration: Option<u64>,
36 usage: Option<Value>,
37}
38#[derive(Clone)]
40pub(crate) struct StreamTiming {
41 clock: Arc<dyn StreamClock>,
42 measurement: Arc<Mutex<Measurement>>,
43}
44fn millis(duration: Duration) -> u64 {
45 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
46}
47impl StreamTiming {
48 pub(crate) fn new(clock: Arc<dyn StreamClock>) -> Self {
49 Self {
50 clock,
51 measurement: Default::default(),
52 }
53 }
54 pub(crate) fn dispatch(&self) {
55 let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
56 state.usage = None;
59 if state.start.is_none() {
60 state.start = Some(self.clock.now());
61 }
62 }
63 pub(crate) fn observe_line(&self, line: &str) {
64 let text = line.trim().strip_prefix("data:").unwrap_or(line).trim();
65 if let Ok(frame) = serde_json::from_str::<Value>(text) {
66 self.observe(&frame);
67 }
68 }
69 fn observe(&self, frame: &Value) {
70 if frame["model"] == "keepalive" {
71 return;
72 }
73 let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
74 if frame["usage"].is_object() {
75 state.usage = Some(frame["usage"].clone());
76 }
77 if state.first.is_some() {
78 return;
79 }
80 let delta = if frame["message"].is_object() {
81 &frame["message"]
82 } else {
83 &frame["choices"][0]["delta"]
84 };
85 let nonempty = |value: &Value| value.as_str().is_some_and(|text| !text.is_empty());
86 let kind = if nonempty(&delta["thinking"]) || nonempty(&delta["reasoning_content"]) {
87 Some("thinking")
88 } else if nonempty(&delta["content"]) {
89 Some("content")
90 } else if delta["tool_calls"].as_array().is_some_and(|calls| {
91 calls.iter().any(|call| {
92 let function = &call["function"];
93 nonempty(&function["arguments"]) || function["arguments"].is_object()
94 })
95 }) {
96 Some("tool_call")
97 } else {
98 None
99 };
100 if let (Some(start), Some(kind)) = (state.start, kind) {
101 state.first = Some((millis(self.clock.now().saturating_sub(start)), kind));
102 }
103 }
104 pub(crate) fn finish(&self, mut metrics: StreamMetrics) -> StreamMetrics {
105 let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
106 if let Some(start) = state.start {
107 let duration = *state
108 .duration
109 .get_or_insert_with(|| millis(self.clock.now().saturating_sub(start)));
110 metrics.request_duration_ms = Some(duration);
111 }
112 metrics.time_to_first_token_ms = state.first.map(|(ms, _)| ms);
113 metrics.first_token_kind = state.first.map(|(_, kind)| kind.to_string());
114 metrics.provider_usage = state.usage.clone();
115 metrics
116 }
117 pub(crate) fn legacy<'a>(
118 self,
119 mut source: Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>,
120 provider: &'static str,
121 ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
122 Box::pin(async_stream::stream! {
123 let mut reported = false;
124 while let Some(item) = source.next().await {
125 match item {
126 Ok(StreamChunk::Metrics(metrics)) => {
127 reported = true;
128 yield Ok(StreamChunk::Metrics(self.finish(metrics)));
129 }
130 Err(error) => {
131 let metrics = self.finish(StreamMetrics { provider: provider.into(), ..Default::default() });
132 if metrics.request_duration_ms.is_some() { yield Ok(StreamChunk::Metrics(metrics)); }
133 yield Err(error);
134 return;
135 }
136 item => yield item,
137 }
138 }
139 if !reported {
140 yield Ok(StreamChunk::Metrics(self.finish(StreamMetrics { provider: provider.into(), ..Default::default() })));
141 }
142 })
143 }
144}
145
146pub(crate) fn retain_timing(
148 evidence: &mut super::models::ResponseEvidence,
149 metrics: &StreamMetrics,
150) {
151 evidence
152 .metadata
153 .insert("time_to_first_token_ms".into(), metrics.time_to_first_token_ms.into());
154 evidence
155 .metadata
156 .insert("first_token_kind".into(), metrics.first_token_kind.clone().into());
157 evidence
158 .metadata
159 .insert("request_duration_ms".into(), metrics.request_duration_ms.into());
160}