Skip to main content

mojentic/llm/recovery/
engine.rs

1use super::{
2    adapter::{RecoveryGateway, RecoveryProvider},
3    frames::Frames,
4    *,
5};
6use crate::llm::stream_timing::{retain_timing, StreamTiming};
7use crate::llm::{
8    gateway::{StreamMetrics, StreamProgress},
9    models::LlmGatewayResponse,
10    stream_events::LineBuffer,
11};
12use futures::{Stream, StreamExt};
13use std::future::Future;
14use std::{
15    pin::Pin,
16    time::{Duration, Instant, UNIX_EPOCH},
17};
18use uuid::Uuid;
19
20/// Output and nonsemantic telemetry for one logical completion.
21/// Completed and Failed are terminal; progress never proves remote termination.
22pub enum RecoveryStreamEvent {
23    /// Decoded provider frame metadata; does not count as delivered semantic output.
24    Progress {
25        identity: Identity,
26        progress: StreamProgress,
27    },
28    /// Original provider-reported final metrics; absent when unsupported, never estimated.
29    /// Final client timing is on the terminal report/error, preserving cancellation checkpoints.
30    Metrics {
31        identity: Identity,
32        metrics: StreamMetrics,
33    },
34    Content(String),
35    Reasoning(String),
36    /// Raw provider tool fragment, for explicit inspection only; never execute it.
37    ToolFragment(serde_json::Value),
38    Completed(Box<Recovered<LlmGatewayResponse>>),
39    Failed(RecoveryError),
40}
41#[derive(Debug)]
42pub(crate) struct PrivateEvidence {
43    pub bytes: Vec<u8>,
44    original: Option<Box<dyn std::error::Error + Send + Sync>>,
45}
46impl PrivateEvidence {
47    pub fn new(bytes: Vec<u8>) -> Self {
48        Self {
49            bytes,
50            original: None,
51        }
52    }
53    pub fn from_error(error: impl std::error::Error + Send + Sync + 'static) -> Self {
54        Self {
55            bytes: error.to_string().into_bytes(),
56            original: Some(Box::new(error)),
57        }
58    }
59}
60impl std::fmt::Display for PrivateEvidence {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        write!(f, "{}", String::from_utf8_lossy(&self.bytes))
63    }
64}
65impl std::error::Error for PrivateEvidence {
66    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
67        self.original.as_deref().map(|e| e as &(dyn std::error::Error + 'static))
68    }
69}
70struct Attempt<'a> {
71    policy: &'a RecoveryPolicy,
72    provider: Provider,
73    operation: Operation,
74    logical: Uuid,
75    number: u32,
76    current: Failure,
77    history: Vec<Failure>,
78    first_failure: Option<Instant>,
79    sent_current: bool,
80    stream_timing: Option<StreamTiming>,
81    raw: Vec<u8>,
82    delivered: std::sync::Arc<std::sync::Mutex<SemanticProgress>>,
83}
84impl<'a> Attempt<'a> {
85    fn new(policy: &'a RecoveryPolicy, provider: Provider, operation: Operation) -> Self {
86        Self {
87            policy,
88            provider,
89            operation,
90            logical: Uuid::new_v4(),
91            number: 0,
92            current: Failure::new(provider, operation, Category::Transport),
93            history: vec![],
94            first_failure: None,
95            sent_current: false,
96            stream_timing: None,
97            raw: Vec::new(),
98            delivered: Default::default(),
99        }
100    }
101    fn emit(&self, transition: Transition, delay: Option<Duration>, failed: bool) {
102        if let Some(observer) = &self.policy.observer {
103            observer.observe(&LifecycleEvent {
104                transition,
105                identity: self.current.identity.clone(),
106                phase: self.current.phase,
107                progress: self.progress(),
108                failure: failed.then(|| self.current.clone()),
109                delay,
110            });
111        }
112    }
113    fn progress(&self) -> Progress {
114        let mut progress = self.current.progress.clone();
115        progress.delivered = *self.delivered.lock().unwrap_or_else(|e| e.into_inner());
116        progress
117    }
118    fn classify(&mut self) {
119        self.current.progress = self.progress();
120        let f = &mut self.current;
121        f.classification = if f.category == Category::Cancellation {
122            Classification {
123                eligible: false,
124                reason: Reason::Cancelled,
125            }
126        } else if matches!(f.http_status, Some(400 | 401 | 403)) {
127            // A body transport error cannot undo a known permanent HTTP rejection.
128            Classification {
129                eligible: false,
130                reason: Reason::Permanent,
131            }
132        } else if f.progress.observed.any() || f.progress.delivered.any() {
133            Classification {
134                eligible: false,
135                reason: Reason::SemanticOutput,
136            }
137        } else if self.operation != Operation::Streaming
138            && f.http_status.is_some_and(|s| (200..300).contains(&s))
139            && f.progress.raw_bytes > 0
140        {
141            // An incomplete JSON body may already contain inference output. Fail closed.
142            Classification {
143                eligible: false,
144                reason: Reason::PartialResponse,
145            }
146        } else {
147            let eligible = match f.category {
148                Category::Http => {
149                    self.policy.retryable_categories.contains(&Category::Http)
150                        && f.http_status
151                            .is_some_and(|s| self.policy.retryable_statuses.contains(&s))
152                        && !matches!(f.http_status, Some(400 | 401 | 403))
153                }
154                Category::Transport => {
155                    self.policy.retryable_categories.contains(&Category::Transport)
156                }
157                Category::ClientTimeout => {
158                    self.policy.retryable_categories.contains(&Category::ClientTimeout)
159                }
160                _ => false,
161            };
162            Classification {
163                eligible,
164                reason: if eligible {
165                    Reason::Transient
166                } else if f.category == Category::Protocol {
167                    Reason::Malformed
168                } else {
169                    Reason::Permanent
170                },
171            }
172        };
173    }
174    fn failure(
175        &mut self,
176        category: Category,
177        cause: impl std::error::Error + Send + Sync + 'static,
178    ) {
179        self.current.set_response_bytes(&self.raw);
180        self.current.category = category;
181        self.current = self.current.clone().with_cause(cause);
182        self.classify();
183    }
184    fn cancel(&mut self) {
185        self.current.set_response_bytes(&self.raw);
186        self.current.category = Category::Cancellation;
187        self.classify();
188    }
189    // Attempt helpers return provisional errors. Only retry finalizes cancellation,
190    // after recording the failed send, so observers receive one terminal event.
191    fn cancelled(&mut self) -> RecoveryError {
192        self.cancel();
193        self.emit(Transition::Cancelled, None, true);
194        self.end(Outcome::Cancelled)
195    }
196    fn end(&self, outcome: Outcome) -> RecoveryError {
197        match outcome {
198            Outcome::Interrupted => self.emit(Transition::Interrupted, None, true),
199            Outcome::Exhausted => self.emit(Transition::Exhausted, None, true),
200            _ => {}
201        }
202        RecoveryError {
203            stream_metrics: None,
204            outcome,
205            failure: Box::new(self.current.clone()),
206            history: self.history.clone(),
207        }
208    }
209    fn expires(&self) -> Option<Instant> {
210        let budget = self
211            .first_failure
212            .zip(self.policy.budget)
213            .and_then(|(start, budget)| start.checked_add(budget));
214        match (self.policy.deadline, budget) {
215            (Some(a), Some(b)) => Some(a.min(b)),
216            (a, b) => a.or(b),
217        }
218    }
219    fn expired(&self) -> bool {
220        self.expires().is_some_and(|end| self.policy.timing.monotonic() >= end)
221    }
222    async fn retry(&mut self, call: &RecoveryCall) -> Result<(), RecoveryError> {
223        self.classify();
224        if self.sent_current {
225            self.history.push(self.current.clone());
226            self.emit(Transition::AttemptFailed, None, true);
227            self.sent_current = false;
228        }
229        if call.cancellation.is_cancelled() || self.current.category == Category::Cancellation {
230            return Err(self.cancelled());
231        }
232        if self.current.progress.observed.any() || self.current.progress.delivered.any() {
233            return Err(self.end(Outcome::Interrupted));
234        }
235        if !self.current.classification.eligible {
236            return Err(self.end(Outcome::Failed));
237        }
238        if self.number >= self.policy.max_attempts.get() {
239            return Err(self.end(Outcome::Exhausted));
240        }
241        self.first_failure.get_or_insert_with(|| self.policy.timing.monotonic());
242        if self.expired() {
243            return Err(self.end(Outcome::BudgetExpired));
244        }
245        let mut ceiling = self.policy.base_delay.min(self.policy.delay_ceiling);
246        for _ in 1..self.number {
247            if ceiling >= self.policy.delay_ceiling || ceiling.is_zero() {
248                break;
249            }
250            ceiling = ceiling.saturating_mul(2).min(self.policy.delay_ceiling);
251        }
252        let floor = self.current.retry_after.delay().unwrap_or_default();
253        if floor > self.policy.delay_ceiling {
254            return Err(self.end(Outcome::DelayCeiling));
255        }
256        let delay = self.policy.timing.jitter(ceiling).min(ceiling).max(floor);
257        if self.expires().is_some_and(|end| {
258            self.policy
259                .timing
260                .monotonic()
261                .checked_add(delay)
262                .is_none_or(|start| start >= end)
263        }) {
264            return Err(self.end(Outcome::BudgetExpired));
265        }
266        if let Some(admission) = &self.policy.admission {
267            self.emit(Transition::AdmissionPending, None, true);
268            let context = AdmissionContext {
269                failure: self.current.clone(),
270                next_attempt: self.number + 1,
271                previous_identity: self
272                    .current
273                    .identity
274                    .clone()
275                    .ok_or_else(|| self.end(Outcome::Failed))?,
276            };
277            let deadline_delay = self
278                .expires()
279                .map(|end| end.saturating_duration_since(self.policy.timing.monotonic()));
280            let expiry = async {
281                if let Some(delay) = deadline_delay {
282                    self.policy.timing.sleep(delay).await;
283                } else {
284                    std::future::pending::<()>().await;
285                }
286            };
287            let decision = tokio::select! { biased;
288                _ = call.cancellation.cancelled() => { return Err(self.cancelled()); },
289                decision = admission.decide(context) => decision,
290                _ = expiry => return Err(self.end(Outcome::BudgetExpired)),
291            };
292            match decision {
293                AdmissionDecision::Allow => self.emit(Transition::AdmissionAllowed, None, true),
294                AdmissionDecision::Reject => {
295                    self.emit(Transition::AdmissionRejected, None, true);
296                    return Err(self.end(Outcome::AdmissionRejected));
297                }
298                AdmissionDecision::Unresolved => {
299                    self.emit(Transition::AdmissionUnresolved, None, true);
300                    return Err(self.end(Outcome::AdmissionUnresolved));
301                }
302            }
303        } else if self.provider.local()
304            && self.current.acceptance != Acceptance::No
305            && !self.current.remote_termination_confirmed
306        {
307            self.emit(Transition::AdmissionRequired, None, true);
308            return Err(self.end(Outcome::AdmissionRequired));
309        }
310        if self.expired() {
311            return Err(self.end(Outcome::BudgetExpired));
312        }
313        if self.expires().is_some_and(|end| {
314            self.policy
315                .timing
316                .monotonic()
317                .checked_add(delay)
318                .is_none_or(|start| start >= end)
319        }) {
320            return Err(self.end(Outcome::BudgetExpired));
321        }
322        self.emit(Transition::DelayScheduled, Some(delay), true);
323        tokio::select! { biased;
324            _ = call.cancellation.cancelled() => { return Err(self.cancelled()); },
325            _ = self.policy.timing.sleep(delay) => {},
326        }
327        if call.cancellation.is_cancelled() {
328            return Err(self.cancelled());
329        }
330        if self.expired() {
331            return Err(self.end(Outcome::BudgetExpired));
332        }
333        Ok(())
334    }
335    fn wire<'b>(
336        &mut self,
337        event: impl FnOnce(Identity) -> WireEvent<'b>,
338    ) -> Result<(), RecoveryError> {
339        if let (Some(observer), Some(id)) = (&self.policy.wire_observer, &self.current.identity) {
340            if let Err(cause) = observer.observe(event(id.clone())) {
341                self.current.category = Category::Protocol;
342                self.current = self.current.clone().with_boxed_cause(cause);
343                self.current.set_response_bytes(&self.raw);
344                self.classify();
345                if self.sent_current {
346                    self.history.push(self.current.clone());
347                    self.emit(Transition::AttemptFailed, None, true);
348                    self.sent_current = false;
349                } else {
350                    self.current.identity = None;
351                }
352                return Err(self.end(Outcome::ObserverFailed));
353            }
354        }
355        Ok(())
356    }
357    async fn send(
358        &mut self,
359        client: &reqwest::Client,
360        request: &reqwest::Request,
361        call: &RecoveryCall,
362    ) -> Result<reqwest::Response, RecoveryError> {
363        if call.cancellation.is_cancelled() {
364            self.cancel();
365            return Err(self.end(Outcome::Cancelled));
366        }
367        if self.number > 0 && self.expired() {
368            return Err(self.end(Outcome::BudgetExpired));
369        }
370        let request = request.try_clone().ok_or_else(|| self.end(Outcome::Failed))?;
371        self.sent_current = false;
372        self.raw.clear();
373        self.current = Failure::new(self.provider, self.operation, Category::Transport);
374        self.current.identity = Some(Identity {
375            logical_request_id: self.logical,
376            attempt_id: Uuid::new_v4(),
377            wire_attempt: self.number + 1,
378        });
379        self.wire(|id| WireEvent::Request {
380            identity: id,
381            request: &request,
382        })?;
383        if call.cancellation.is_cancelled() {
384            self.current.identity = None;
385            self.cancel();
386            return Err(self.end(Outcome::Cancelled));
387        }
388        if self.number > 0 && self.expired() {
389            self.current.identity = None;
390            return Err(self.end(Outcome::BudgetExpired));
391        }
392        self.current.phase = None;
393        if self.number > 0 {
394            self.emit(Transition::RetryStarted, None, false);
395        }
396        self.emit(Transition::AttemptStarted, None, false);
397        if call.cancellation.is_cancelled() {
398            self.current.identity = None;
399            self.cancel();
400            return Err(self.end(Outcome::Cancelled));
401        }
402        if self.number > 0 && self.expired() {
403            self.current.identity = None;
404            return Err(self.end(Outcome::BudgetExpired));
405        }
406        let metadata_request = request.try_clone();
407        let mut execution = Box::pin(client.execute(request));
408        let mut dispatched = false;
409        let response = tokio::select! { biased;
410            _ = call.cancellation.cancelled() => {
411                if !dispatched { self.current.identity = None; }
412                else { self.number += 1; self.sent_current = true; }
413                self.cancel(); return Err(self.end(Outcome::Cancelled));
414            },
415            response = std::future::poll_fn(|cx| { if !dispatched { if let Some(timing) = &self.stream_timing { timing.dispatch(); } } dispatched = true; execution.as_mut().poll(cx) }) => response,
416        };
417        self.number += 1;
418        self.sent_current = true;
419        match response {
420            Ok(response) => {
421                self.current.set_response_headers(response.headers());
422                self.current.http_status = Some(response.status().as_u16());
423                self.current.progress.headers_received = true;
424                self.current.phase = Some(Phase::Streaming);
425                self.current.retry_after = parse_retry_after(
426                    response.headers().get("retry-after"),
427                    self.policy.timing.wall(),
428                );
429                self.current.provider_request_id = response
430                    .headers()
431                    .get("x-request-id")
432                    .and_then(|v| v.to_str().ok())
433                    .and_then(ValidatedMetadata::request_id)
434                    .filter(|id| {
435                        metadata_request
436                            .as_ref()
437                            .is_some_and(|request| !echoes_request(id.as_str(), request))
438                    });
439                self.wire(|id| WireEvent::Headers {
440                    identity: id,
441                    status: response.status().as_u16(),
442                    headers: response.headers(),
443                })?;
444                Ok(response)
445            }
446            Err(cause) => {
447                if cause.is_connect() {
448                    self.current.phase = Some(Phase::Connecting);
449                }
450                let category = if cause.is_timeout() {
451                    Category::ClientTimeout
452                } else {
453                    Category::Transport
454                };
455                self.failure(category, cause);
456                Err(self.end(Outcome::Failed))
457            }
458        }
459    }
460    fn report(&self) -> RecoveryReport {
461        RecoveryReport {
462            stream_metrics: None,
463            identity: self.current.identity.clone().unwrap_or(Identity {
464                logical_request_id: self.logical,
465                attempt_id: Uuid::nil(),
466                wire_attempt: self.number,
467            }),
468            progress: self.progress(),
469            history: self.history.clone(),
470        }
471    }
472    fn decode_failure(&mut self, mut cause: PrivateEvidence) {
473        let body = serde_json::from_slice::<serde_json::Value>(&cause.bytes).ok();
474        let category = if body.as_ref().is_some_and(|body| !body["error"].is_null()) {
475            Category::ProviderResponse
476        } else {
477            Category::Protocol
478        };
479        self.current.provider_code =
480            body.and_then(|body| body["error"]["code"].as_str().and_then(ValidatedMetadata::code));
481        if let Some(original) = cause.original.take() {
482            self.current.set_response_bytes(&self.raw);
483            self.current.category = category;
484            self.current = self.current.clone().with_boxed_cause(original);
485            self.classify();
486        } else {
487            self.failure(category, cause);
488        }
489    }
490    fn http_failure(&mut self, bytes: Vec<u8>) {
491        self.current.provider_code = serde_json::from_slice::<serde_json::Value>(&bytes)
492            .ok()
493            .and_then(|v| v["error"]["code"].as_str().and_then(ValidatedMetadata::code));
494        self.failure(Category::Http, PrivateEvidence::new(bytes));
495    }
496}
497// A syntactically valid request ID can still be an echoed credential or prompt.
498// Prefer omission when its spelling occurs in the immutable request.
499fn echoes_request(value: &str, request: &reqwest::Request) -> bool {
500    let candidates = [value, value.strip_prefix("req_").unwrap_or(value)];
501    candidates.iter().any(|candidate| {
502        let bytes = candidate.as_bytes();
503        let contains = |source: &[u8]| source.windows(bytes.len()).any(|part| part == bytes);
504        request.headers().values().any(|header| contains(header.as_bytes()))
505            || request.body().and_then(|body| body.as_bytes()).is_some_and(contains)
506    })
507}
508fn parse_retry_after(
509    value: Option<&reqwest::header::HeaderValue>,
510    wall: std::time::SystemTime,
511) -> RetryAfter {
512    let Some(value) = value else {
513        return RetryAfter::Absent;
514    };
515    let Ok(value) = value.to_str() else {
516        return RetryAfter::Invalid;
517    };
518    let value = value.trim();
519    if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) {
520        return value.parse().map(RetryAfter::Seconds).unwrap_or(RetryAfter::Invalid);
521    }
522    let value = value.trim();
523    // Resolve obsolete two-digit years against the observed wall clock before
524    // validating the weekday. Chrono's fixed %y pivot is not HTTP's moving pivot.
525    let obsolete = || {
526        use chrono::Datelike;
527        let (weekday, rest) = value.split_once(", ")?;
528        let (date, time) = rest.split_once(' ')?;
529        let (day_month, short_year) = date.rsplit_once('-')?;
530        if short_year.len() != 2 || !short_year.bytes().all(|b| b.is_ascii_digit()) {
531            return None;
532        }
533        let horizon = chrono::DateTime::<chrono::Utc>::from(wall).year().saturating_add(50);
534        let mut year = horizon.div_euclid(100) * 100 + short_year.parse::<i32>().ok()?;
535        if year > horizon {
536            year -= 100;
537        }
538        let expanded = format!("{weekday}, {day_month}-{year:04} {time}");
539        chrono::NaiveDateTime::parse_from_str(&expanded, "%A, %d-%b-%Y %H:%M:%S GMT").ok()
540    };
541    let parsed = chrono::NaiveDateTime::parse_from_str(value, "%a, %d %b %Y %H:%M:%S GMT")
542        .ok()
543        .or_else(obsolete)
544        .or_else(|| chrono::NaiveDateTime::parse_from_str(value, "%a %b %e %H:%M:%S %Y").ok());
545    match parsed {
546        Some(date) => {
547            let seconds = date.and_utc().timestamp();
548            let remaining = if seconds < 0 {
549                Duration::ZERO
550            } else {
551                UNIX_EPOCH
552                    .checked_add(Duration::from_secs(seconds as u64))
553                    .and_then(|date| date.duration_since(wall).ok())
554                    .unwrap_or_default()
555            };
556            RetryAfter::Date {
557                unix_seconds: seconds,
558                remaining,
559            }
560        }
561        None => RetryAfter::Invalid,
562    }
563}
564async fn body(
565    state: &mut Attempt<'_>,
566    response: reqwest::Response,
567    call: &RecoveryCall,
568) -> Result<Vec<u8>, RecoveryError> {
569    let mut bytes = response.bytes_stream();
570    let mut collected = Vec::new();
571    loop {
572        let chunk = tokio::select! { biased;
573            _ = call.cancellation.cancelled() => { state.cancel(); state.current = state.current.clone().with_cause(PrivateEvidence::new(collected)); return Err(state.end(Outcome::Cancelled)); },
574            chunk = bytes.next() => chunk,
575        };
576        match chunk {
577            Some(Ok(chunk)) => {
578                state.current.progress.raw_bytes =
579                    state.current.progress.raw_bytes.saturating_add(chunk.len() as u64);
580                state.raw.extend_from_slice(&chunk);
581                collected.extend_from_slice(&chunk);
582                if state.operation != Operation::Streaming
583                    && state.current.http_status.is_some_and(|s| (200..300).contains(&s))
584                {
585                    super::frames::observe_response(
586                        state.provider,
587                        &collected,
588                        &mut state.current.progress.observed,
589                    );
590                }
591                // Capture failure still retains semantic evidence already received.
592                state.wire(|id| WireEvent::Body {
593                    identity: id,
594                    bytes: &chunk,
595                })?;
596            }
597            Some(Err(cause)) => {
598                // Once headers establish an HTTP failure, its status governs eligibility.
599                // Preserve the body transport cause privately without reclassifying it.
600                let category =
601                    if state.current.http_status.is_some_and(|s| !(200..300).contains(&s)) {
602                        Category::Http
603                    } else {
604                        Category::Transport
605                    };
606                state.failure(category, cause);
607                return Err(state.end(Outcome::Failed));
608            }
609            None => return Ok(collected),
610        }
611    }
612}
613pub(super) async fn complete<G: RecoveryProvider>(
614    gateway: &RecoveryGateway<G>,
615    request: reqwest::Request,
616    operation: Operation,
617    call: &RecoveryCall,
618) -> Result<Recovered<LlmGatewayResponse<serde_json::Value>>, RecoveryError> {
619    let mut state = Attempt::new(&gateway.policy, gateway.inner.recovery_provider(), operation);
620    let structured_output = operation == Operation::Structured
621        || request
622            .body()
623            .and_then(|body| body.as_bytes())
624            .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(bytes).ok())
625            .is_some_and(|body| {
626                matches!(
627                    body["response_format"]["type"].as_str(),
628                    Some("json_object" | "json_schema")
629                )
630            });
631    loop {
632        let result = async {
633            let response = state.send(&gateway.client, &request, call).await?;
634            let success = response.status().is_success();
635            let collected = body(&mut state, response, call).await?;
636            if !success {
637                state.http_failure(collected);
638                return Err(state.end(Outcome::Failed));
639            }
640            state.current.phase = Some(Phase::Decoding);
641            match super::frames::decode(
642                state.provider,
643                &collected,
644                operation == Operation::Structured,
645            ) {
646                Ok(mut response) => {
647                    if state.provider == Provider::Omlx && structured_output {
648                        if let Some(warning) = state
649                            .current
650                            .inspect_response_headers()
651                            .and_then(crate::llm::gateways::omlx::format_warning_headers)
652                        {
653                            response.evidence.metadata.insert(
654                                crate::llm::gateways::omlx::RESPONSE_FORMAT_WARNING.into(),
655                                serde_json::Value::String(warning),
656                            );
657                        }
658                    }
659                    super::frames::response_progress(
660                        &response,
661                        &mut state.current.progress.observed,
662                    );
663                    *state.delivered.lock().unwrap_or_else(|e| e.into_inner()) =
664                        state.current.progress.observed;
665                    Ok(response)
666                }
667                Err(cause) => {
668                    state.decode_failure(cause);
669                    Err(state.end(Outcome::Failed))
670                }
671            }
672        }
673        .await;
674        match result {
675            Ok(value) => {
676                state.emit(Transition::AttemptSucceeded, None, false);
677                return Ok(Recovered {
678                    value,
679                    report: state.report(),
680                });
681            }
682            Err(error) if error.outcome == Outcome::ObserverFailed => return Err(error),
683            Err(_) => state.retry(call).await?,
684        }
685    }
686}
687pub(super) fn stream<G: RecoveryProvider>(
688    gateway: &RecoveryGateway<G>,
689    request: reqwest::Request,
690    call: &RecoveryCall,
691) -> Pin<Box<dyn Stream<Item = RecoveryStreamEvent> + Send>> {
692    let owner = StreamOwner {
693        client: gateway.client.clone(),
694        policy: gateway.policy.clone(),
695        provider: gateway.inner.recovery_provider(),
696        timing: StreamTiming::new(gateway.inner.stream_clock()),
697    };
698    let call = RecoveryCall {
699        cancellation: call.cancellation.child_token(),
700    };
701    Box::pin(async_stream::stream! {
702        let _guard = call.cancellation.clone().drop_guard();
703        let delivered = std::sync::Arc::new(std::sync::Mutex::new(SemanticProgress::default()));
704        let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
705        let worker_call = call.clone();
706        let worker_delivered = delivered.clone();
707        tokio::spawn(async move {
708            let timing = owner.timing.clone();
709            let mut provider_metrics = StreamMetrics { provider: format!("{:?}", owner.provider).to_lowercase(), ..Default::default() };
710            let mut source = stream_owned(owner, request, worker_call.clone(), worker_delivered);
711            loop {
712                // Reserve delivery capacity before advancing the provider stream. In
713                // particular, do not finalize success behind buffered terminal metadata
714                // while a paused consumer can still cancel the active attempt.
715                let permit = tokio::select! { biased;
716                    _ = worker_call.cancellation.cancelled() => None,
717                    permit = sender.reserve() => match permit {
718                        Ok(permit) => Some(permit),
719                        Err(_) => break,
720                    },
721                };
722                let Some(mut event) = source.next().await else { break; };
723                match &mut event {
724                    RecoveryStreamEvent::Metrics { metrics, .. } => provider_metrics = metrics.clone(),
725                    RecoveryStreamEvent::Completed(result) => {
726                        let metrics = timing.finish(provider_metrics.clone());
727                        retain_timing(&mut result.value.evidence, &metrics);
728                        result.report.stream_metrics = Some(Box::new(metrics));
729                    },
730                    RecoveryStreamEvent::Failed(error) => error.stream_metrics = Some(Box::new(timing.finish(provider_metrics.clone()))),
731                    _ => {},
732                }
733                let terminal = matches!(event, RecoveryStreamEvent::Completed(_) | RecoveryStreamEvent::Failed(_));
734                if let Some(permit) = permit { permit.send(event); }
735                else if terminal { let _ = sender.send(event).await; }
736                if terminal { break; }
737            }
738        });
739        while let Some(mut event) = receiver.recv().await {
740            if call.cancellation.is_cancelled() && !matches!(event, RecoveryStreamEvent::Failed(_) | RecoveryStreamEvent::Completed(_)) { continue; }
741            {
742                let mut delivered = delivered.lock().unwrap_or_else(|e| e.into_inner());
743                super::frames::delivered(&event, &mut delivered);
744                match &mut event {
745                    RecoveryStreamEvent::Failed(error) => { error.failure.progress.delivered = *delivered;
746                        if let Some(last) = error.history.last_mut().filter(|last| last.identity == error.failure.identity) { last.progress.delivered = *delivered; }
747                    },
748                    RecoveryStreamEvent::Completed(result) => { delivered.completed_tool_calls = result.value.tool_calls.len() as u64; result.report.progress.delivered = *delivered; },
749                    _ => {},
750                }
751            }
752            yield event;
753        }
754    })
755}
756struct StreamOwner {
757    client: reqwest::Client,
758    policy: RecoveryPolicy,
759    provider: Provider,
760    timing: StreamTiming,
761}
762fn stream_owned(
763    gateway: StreamOwner,
764    request: reqwest::Request,
765    call: RecoveryCall,
766    delivered: std::sync::Arc<std::sync::Mutex<SemanticProgress>>,
767) -> Pin<Box<dyn Stream<Item = RecoveryStreamEvent> + Send>> {
768    Box::pin(async_stream::stream! {
769        let mut state = Attempt::new(&gateway.policy, gateway.provider, Operation::Streaming);
770        state.delivered = delivered;
771        state.stream_timing = Some(gateway.timing.clone());
772        loop {
773            let response = match state.send(&gateway.client, &request, &call).await {
774                Ok(response) => response,
775                Err(error) => { if error.outcome == Outcome::ObserverFailed { yield RecoveryStreamEvent::Failed(error); break; } match state.retry(&call).await { Ok(()) => continue, Err(error) => { yield RecoveryStreamEvent::Failed(error); break; } } },
776            };
777            if !response.status().is_success() {
778                match body(&mut state, response, &call).await { Ok(bytes) => state.http_failure(bytes), Err(error) if error.outcome == Outcome::ObserverFailed => { yield RecoveryStreamEvent::Failed(error); break; }, Err(_) => {} }
779            } else {
780                let mut bytes = response.bytes_stream();
781                let mut lines = LineBuffer::default();
782                let mut frames = Frames::new(state.provider);
783                let mut failed = false;
784                let mut raw_evidence = Vec::new();
785                while !frames.terminal {
786                    let chunk = tokio::select! { biased;
787                        _ = call.cancellation.cancelled() => { state.cancel(); failed = true; None },
788                        chunk = bytes.next() => chunk,
789                    };
790                    let Some(chunk) = chunk else { if !failed { state.failure(if lines.has_pending() { Category::Protocol } else { Category::Transport }, PrivateEvidence::new(b"stream ended without terminal proof".to_vec())); } break; };
791                    let chunk = match chunk { Ok(chunk) => chunk, Err(cause) => { state.failure(if lines.has_pending() { Category::Protocol } else { Category::Transport }, cause); break; } };
792                    state.current.progress.raw_bytes = state.current.progress.raw_bytes.saturating_add(chunk.len() as u64);
793                    raw_evidence.extend_from_slice(&chunk);
794                    state.raw.extend_from_slice(&chunk);
795                    // Account for all semantic evidence in a received chunk before yielding any part.
796                    let mut output = Vec::new();
797                    for line in lines.push(&chunk) {
798                        if let Ok(text) = std::str::from_utf8(&line) { gateway.timing.observe_line(text); }
799                        let (events, result) = frames.parse(&line, &mut state.current.progress.observed, state.current.identity.as_ref());
800                        output.extend(events);
801                        if let Err(cause) = result { state.decode_failure(cause); failed = true; break; }
802                    }
803                    if let Err(error) = state.wire(|id| WireEvent::Body { identity: id, bytes: &chunk }) { yield RecoveryStreamEvent::Failed(error); return; }
804                    for event in output {
805                        if call.cancellation.is_cancelled() { state.cancel(); failed = true; break; }
806                        yield event;
807                    }
808                    if failed { break; }
809                }
810                // A terminal-only chunk has no output loop in which to notice a
811                // wire hook cancelling the call. Cancellation still wins over success.
812                if call.cancellation.is_cancelled() { state.cancel(); failed = true; }
813                drop(bytes);
814                if state.current.category == Category::Cancellation { state.current = state.current.clone().with_cause(PrivateEvidence::new(raw_evidence)); }
815                if frames.terminal && !failed {
816                    match frames.response() {
817                        Ok(value) => { state.emit(Transition::AttemptSucceeded, None, false); yield RecoveryStreamEvent::Completed(Box::new(Recovered { value, report: state.report() })); return; },
818                        Err(cause) => state.decode_failure(cause),
819                    }
820                }
821            }
822            match state.retry(&call).await { Ok(()) => {}, Err(error) => { yield RecoveryStreamEvent::Failed(error); break; } }
823        }
824    })
825}