Skip to main content

mojentic/llm/
stream_timing.rs

1//! Client timing for a logical streamed completion.
2use super::gateway::{StreamChunk, StreamMetrics};
3use crate::error::Result;
4use futures::{Stream, StreamExt};
5use serde_json::Value;
6use std::{
7    pin::Pin,
8    sync::{Arc, Mutex},
9    time::{Duration, Instant},
10};
11
12/// Monotonic time source for client stream measurements.
13///
14/// Implementations must return nondecreasing elapsed times from a fixed origin.
15/// A custom clock permits deterministic measurements in embedded environments
16/// and contract tests; normal gateways use [`Instant`].
17pub trait StreamClock: Send + Sync {
18    /// Elapsed time from this clock's fixed origin.
19    fn now(&self) -> Duration;
20}
21struct MonotonicClock(Instant);
22impl StreamClock for MonotonicClock {
23    fn now(&self) -> Duration {
24        self.0.elapsed()
25    }
26}
27pub(crate) fn system_clock() -> Arc<dyn StreamClock> {
28    Arc::new(MonotonicClock(Instant::now()))
29}
30
31#[derive(Default)]
32struct Measurement {
33    start: Option<Duration>,
34    first: Option<(u64, &'static str)>,
35    duration: Option<u64>,
36    usage: Option<Value>,
37}
38/// Shared only between a request driver and its output adapter.
39#[derive(Clone)]
40pub(crate) struct StreamTiming {
41    clock: Arc<dyn StreamClock>,
42    measurement: Arc<Mutex<Measurement>>,
43}
44fn millis(duration: Duration) -> u64 {
45    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
46}
47impl StreamTiming {
48    pub(crate) fn new(clock: Arc<dyn StreamClock>) -> Self {
49        Self {
50            clock,
51            measurement: Default::default(),
52        }
53    }
54    pub(crate) fn dispatch(&self) {
55        let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
56        // Provider usage belongs to a wire attempt; the client origin belongs
57        // to the logical completion and survives admitted retries.
58        state.usage = None;
59        if state.start.is_none() {
60            state.start = Some(self.clock.now());
61        }
62    }
63    pub(crate) fn observe_line(&self, line: &str) {
64        let text = line.trim().strip_prefix("data:").unwrap_or(line).trim();
65        if let Ok(frame) = serde_json::from_str::<Value>(text) {
66            self.observe(&frame);
67        }
68    }
69    fn observe(&self, frame: &Value) {
70        if frame["model"] == "keepalive" {
71            return;
72        }
73        let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
74        if frame["usage"].is_object() {
75            state.usage = Some(frame["usage"].clone());
76        }
77        if state.first.is_some() {
78            return;
79        }
80        let delta = if frame["message"].is_object() {
81            &frame["message"]
82        } else {
83            &frame["choices"][0]["delta"]
84        };
85        let nonempty = |value: &Value| value.as_str().is_some_and(|text| !text.is_empty());
86        let kind = if nonempty(&delta["thinking"]) || nonempty(&delta["reasoning_content"]) {
87            Some("thinking")
88        } else if nonempty(&delta["content"]) {
89            Some("content")
90        } else if delta["tool_calls"].as_array().is_some_and(|calls| {
91            calls.iter().any(|call| {
92                let function = &call["function"];
93                nonempty(&function["arguments"]) || function["arguments"].is_object()
94            })
95        }) {
96            Some("tool_call")
97        } else {
98            None
99        };
100        if let (Some(start), Some(kind)) = (state.start, kind) {
101            state.first = Some((millis(self.clock.now().saturating_sub(start)), kind));
102        }
103    }
104    pub(crate) fn finish(&self, mut metrics: StreamMetrics) -> StreamMetrics {
105        let mut state = self.measurement.lock().unwrap_or_else(|e| e.into_inner());
106        if let Some(start) = state.start {
107            let duration = *state
108                .duration
109                .get_or_insert_with(|| millis(self.clock.now().saturating_sub(start)));
110            metrics.request_duration_ms = Some(duration);
111        }
112        metrics.time_to_first_token_ms = state.first.map(|(ms, _)| ms);
113        metrics.first_token_kind = state.first.map(|(_, kind)| kind.to_string());
114        metrics.provider_usage = state.usage.clone();
115        metrics
116    }
117    pub(crate) fn legacy<'a>(
118        self,
119        mut source: Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>,
120        provider: &'static str,
121    ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>> {
122        Box::pin(async_stream::stream! {
123            let mut reported = false;
124            while let Some(item) = source.next().await {
125                match item {
126                    Ok(StreamChunk::Metrics(metrics)) => {
127                        reported = true;
128                        yield Ok(StreamChunk::Metrics(self.finish(metrics)));
129                    }
130                    Err(error) => {
131                        let metrics = self.finish(StreamMetrics { provider: provider.into(), ..Default::default() });
132                        if metrics.request_duration_ms.is_some() { yield Ok(StreamChunk::Metrics(metrics)); }
133                        yield Err(error);
134                        return;
135                    }
136                    item => yield item,
137                }
138            }
139            if !reported {
140                yield Ok(StreamChunk::Metrics(self.finish(StreamMetrics { provider: provider.into(), ..Default::default() })));
141            }
142        })
143    }
144}
145
146/// Attach only client fields; provider evidence remains unchanged.
147pub(crate) fn retain_timing(
148    evidence: &mut super::models::ResponseEvidence,
149    metrics: &StreamMetrics,
150) {
151    evidence
152        .metadata
153        .insert("time_to_first_token_ms".into(), metrics.time_to_first_token_ms.into());
154    evidence
155        .metadata
156        .insert("first_token_kind".into(), metrics.first_token_kind.clone().into());
157    evidence
158        .metadata
159        .insert("request_duration_ms".into(), metrics.request_duration_ms.into());
160}