1use super::{
2 adapter::{RecoveryGateway, RecoveryProvider},
3 frames::Frames,
4 *,
5};
6use crate::llm::stream_timing::{retain_timing, StreamTiming};
7use crate::llm::{
8 gateway::{StreamMetrics, StreamProgress},
9 models::LlmGatewayResponse,
10 stream_events::LineBuffer,
11};
12use futures::{Stream, StreamExt};
13use std::future::Future;
14use std::{
15 pin::Pin,
16 time::{Duration, Instant, UNIX_EPOCH},
17};
18use uuid::Uuid;
19
20pub enum RecoveryStreamEvent {
23 Progress {
25 identity: Identity,
26 progress: StreamProgress,
27 },
28 Metrics {
31 identity: Identity,
32 metrics: StreamMetrics,
33 },
34 Content(String),
35 Reasoning(String),
36 ToolFragment(serde_json::Value),
38 Completed(Box<Recovered<LlmGatewayResponse>>),
39 Failed(RecoveryError),
40}
41#[derive(Debug)]
42pub(crate) struct PrivateEvidence {
43 pub bytes: Vec<u8>,
44 original: Option<Box<dyn std::error::Error + Send + Sync>>,
45}
46impl PrivateEvidence {
47 pub fn new(bytes: Vec<u8>) -> Self {
48 Self {
49 bytes,
50 original: None,
51 }
52 }
53 pub fn from_error(error: impl std::error::Error + Send + Sync + 'static) -> Self {
54 Self {
55 bytes: error.to_string().into_bytes(),
56 original: Some(Box::new(error)),
57 }
58 }
59}
60impl std::fmt::Display for PrivateEvidence {
61 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 write!(f, "{}", String::from_utf8_lossy(&self.bytes))
63 }
64}
65impl std::error::Error for PrivateEvidence {
66 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
67 self.original.as_deref().map(|e| e as &(dyn std::error::Error + 'static))
68 }
69}
70struct Attempt<'a> {
71 policy: &'a RecoveryPolicy,
72 provider: Provider,
73 operation: Operation,
74 logical: Uuid,
75 number: u32,
76 current: Failure,
77 history: Vec<Failure>,
78 first_failure: Option<Instant>,
79 sent_current: bool,
80 stream_timing: Option<StreamTiming>,
81 raw: Vec<u8>,
82 delivered: std::sync::Arc<std::sync::Mutex<SemanticProgress>>,
83}
84impl<'a> Attempt<'a> {
85 fn new(policy: &'a RecoveryPolicy, provider: Provider, operation: Operation) -> Self {
86 Self {
87 policy,
88 provider,
89 operation,
90 logical: Uuid::new_v4(),
91 number: 0,
92 current: Failure::new(provider, operation, Category::Transport),
93 history: vec![],
94 first_failure: None,
95 sent_current: false,
96 stream_timing: None,
97 raw: Vec::new(),
98 delivered: Default::default(),
99 }
100 }
101 fn emit(&self, transition: Transition, delay: Option<Duration>, failed: bool) {
102 if let Some(observer) = &self.policy.observer {
103 observer.observe(&LifecycleEvent {
104 transition,
105 identity: self.current.identity.clone(),
106 phase: self.current.phase,
107 progress: self.progress(),
108 failure: failed.then(|| self.current.clone()),
109 delay,
110 });
111 }
112 }
113 fn progress(&self) -> Progress {
114 let mut progress = self.current.progress.clone();
115 progress.delivered = *self.delivered.lock().unwrap_or_else(|e| e.into_inner());
116 progress
117 }
118 fn classify(&mut self) {
119 self.current.progress = self.progress();
120 let f = &mut self.current;
121 f.classification = if f.category == Category::Cancellation {
122 Classification {
123 eligible: false,
124 reason: Reason::Cancelled,
125 }
126 } else if matches!(f.http_status, Some(400 | 401 | 403)) {
127 Classification {
129 eligible: false,
130 reason: Reason::Permanent,
131 }
132 } else if f.progress.observed.any() || f.progress.delivered.any() {
133 Classification {
134 eligible: false,
135 reason: Reason::SemanticOutput,
136 }
137 } else if self.operation != Operation::Streaming
138 && f.http_status.is_some_and(|s| (200..300).contains(&s))
139 && f.progress.raw_bytes > 0
140 {
141 Classification {
143 eligible: false,
144 reason: Reason::PartialResponse,
145 }
146 } else {
147 let eligible = match f.category {
148 Category::Http => {
149 self.policy.retryable_categories.contains(&Category::Http)
150 && f.http_status
151 .is_some_and(|s| self.policy.retryable_statuses.contains(&s))
152 && !matches!(f.http_status, Some(400 | 401 | 403))
153 }
154 Category::Transport => {
155 self.policy.retryable_categories.contains(&Category::Transport)
156 }
157 Category::ClientTimeout => {
158 self.policy.retryable_categories.contains(&Category::ClientTimeout)
159 }
160 _ => false,
161 };
162 Classification {
163 eligible,
164 reason: if eligible {
165 Reason::Transient
166 } else if f.category == Category::Protocol {
167 Reason::Malformed
168 } else {
169 Reason::Permanent
170 },
171 }
172 };
173 }
174 fn failure(
175 &mut self,
176 category: Category,
177 cause: impl std::error::Error + Send + Sync + 'static,
178 ) {
179 self.current.set_response_bytes(&self.raw);
180 self.current.category = category;
181 self.current = self.current.clone().with_cause(cause);
182 self.classify();
183 }
184 fn cancel(&mut self) {
185 self.current.set_response_bytes(&self.raw);
186 self.current.category = Category::Cancellation;
187 self.classify();
188 }
189 fn cancelled(&mut self) -> RecoveryError {
192 self.cancel();
193 self.emit(Transition::Cancelled, None, true);
194 self.end(Outcome::Cancelled)
195 }
196 fn end(&self, outcome: Outcome) -> RecoveryError {
197 match outcome {
198 Outcome::Interrupted => self.emit(Transition::Interrupted, None, true),
199 Outcome::Exhausted => self.emit(Transition::Exhausted, None, true),
200 _ => {}
201 }
202 RecoveryError {
203 stream_metrics: None,
204 outcome,
205 failure: Box::new(self.current.clone()),
206 history: self.history.clone(),
207 }
208 }
209 fn expires(&self) -> Option<Instant> {
210 let budget = self
211 .first_failure
212 .zip(self.policy.budget)
213 .and_then(|(start, budget)| start.checked_add(budget));
214 match (self.policy.deadline, budget) {
215 (Some(a), Some(b)) => Some(a.min(b)),
216 (a, b) => a.or(b),
217 }
218 }
219 fn expired(&self) -> bool {
220 self.expires().is_some_and(|end| self.policy.timing.monotonic() >= end)
221 }
222 async fn retry(&mut self, call: &RecoveryCall) -> Result<(), RecoveryError> {
223 self.classify();
224 if self.sent_current {
225 self.history.push(self.current.clone());
226 self.emit(Transition::AttemptFailed, None, true);
227 self.sent_current = false;
228 }
229 if call.cancellation.is_cancelled() || self.current.category == Category::Cancellation {
230 return Err(self.cancelled());
231 }
232 if self.current.progress.observed.any() || self.current.progress.delivered.any() {
233 return Err(self.end(Outcome::Interrupted));
234 }
235 if !self.current.classification.eligible {
236 return Err(self.end(Outcome::Failed));
237 }
238 if self.number >= self.policy.max_attempts.get() {
239 return Err(self.end(Outcome::Exhausted));
240 }
241 self.first_failure.get_or_insert_with(|| self.policy.timing.monotonic());
242 if self.expired() {
243 return Err(self.end(Outcome::BudgetExpired));
244 }
245 let mut ceiling = self.policy.base_delay.min(self.policy.delay_ceiling);
246 for _ in 1..self.number {
247 if ceiling >= self.policy.delay_ceiling || ceiling.is_zero() {
248 break;
249 }
250 ceiling = ceiling.saturating_mul(2).min(self.policy.delay_ceiling);
251 }
252 let floor = self.current.retry_after.delay().unwrap_or_default();
253 if floor > self.policy.delay_ceiling {
254 return Err(self.end(Outcome::DelayCeiling));
255 }
256 let delay = self.policy.timing.jitter(ceiling).min(ceiling).max(floor);
257 if self.expires().is_some_and(|end| {
258 self.policy
259 .timing
260 .monotonic()
261 .checked_add(delay)
262 .is_none_or(|start| start >= end)
263 }) {
264 return Err(self.end(Outcome::BudgetExpired));
265 }
266 if let Some(admission) = &self.policy.admission {
267 self.emit(Transition::AdmissionPending, None, true);
268 let context = AdmissionContext {
269 failure: self.current.clone(),
270 next_attempt: self.number + 1,
271 previous_identity: self
272 .current
273 .identity
274 .clone()
275 .ok_or_else(|| self.end(Outcome::Failed))?,
276 };
277 let deadline_delay = self
278 .expires()
279 .map(|end| end.saturating_duration_since(self.policy.timing.monotonic()));
280 let expiry = async {
281 if let Some(delay) = deadline_delay {
282 self.policy.timing.sleep(delay).await;
283 } else {
284 std::future::pending::<()>().await;
285 }
286 };
287 let decision = tokio::select! { biased;
288 _ = call.cancellation.cancelled() => { return Err(self.cancelled()); },
289 decision = admission.decide(context) => decision,
290 _ = expiry => return Err(self.end(Outcome::BudgetExpired)),
291 };
292 match decision {
293 AdmissionDecision::Allow => self.emit(Transition::AdmissionAllowed, None, true),
294 AdmissionDecision::Reject => {
295 self.emit(Transition::AdmissionRejected, None, true);
296 return Err(self.end(Outcome::AdmissionRejected));
297 }
298 AdmissionDecision::Unresolved => {
299 self.emit(Transition::AdmissionUnresolved, None, true);
300 return Err(self.end(Outcome::AdmissionUnresolved));
301 }
302 }
303 } else if self.provider.local()
304 && self.current.acceptance != Acceptance::No
305 && !self.current.remote_termination_confirmed
306 {
307 self.emit(Transition::AdmissionRequired, None, true);
308 return Err(self.end(Outcome::AdmissionRequired));
309 }
310 if self.expired() {
311 return Err(self.end(Outcome::BudgetExpired));
312 }
313 if self.expires().is_some_and(|end| {
314 self.policy
315 .timing
316 .monotonic()
317 .checked_add(delay)
318 .is_none_or(|start| start >= end)
319 }) {
320 return Err(self.end(Outcome::BudgetExpired));
321 }
322 self.emit(Transition::DelayScheduled, Some(delay), true);
323 tokio::select! { biased;
324 _ = call.cancellation.cancelled() => { return Err(self.cancelled()); },
325 _ = self.policy.timing.sleep(delay) => {},
326 }
327 if call.cancellation.is_cancelled() {
328 return Err(self.cancelled());
329 }
330 if self.expired() {
331 return Err(self.end(Outcome::BudgetExpired));
332 }
333 Ok(())
334 }
335 fn wire<'b>(
336 &mut self,
337 event: impl FnOnce(Identity) -> WireEvent<'b>,
338 ) -> Result<(), RecoveryError> {
339 if let (Some(observer), Some(id)) = (&self.policy.wire_observer, &self.current.identity) {
340 if let Err(cause) = observer.observe(event(id.clone())) {
341 self.current.category = Category::Protocol;
342 self.current = self.current.clone().with_boxed_cause(cause);
343 self.current.set_response_bytes(&self.raw);
344 self.classify();
345 if self.sent_current {
346 self.history.push(self.current.clone());
347 self.emit(Transition::AttemptFailed, None, true);
348 self.sent_current = false;
349 } else {
350 self.current.identity = None;
351 }
352 return Err(self.end(Outcome::ObserverFailed));
353 }
354 }
355 Ok(())
356 }
357 async fn send(
358 &mut self,
359 client: &reqwest::Client,
360 request: &reqwest::Request,
361 call: &RecoveryCall,
362 ) -> Result<reqwest::Response, RecoveryError> {
363 if call.cancellation.is_cancelled() {
364 self.cancel();
365 return Err(self.end(Outcome::Cancelled));
366 }
367 if self.number > 0 && self.expired() {
368 return Err(self.end(Outcome::BudgetExpired));
369 }
370 let request = request.try_clone().ok_or_else(|| self.end(Outcome::Failed))?;
371 self.sent_current = false;
372 self.raw.clear();
373 self.current = Failure::new(self.provider, self.operation, Category::Transport);
374 self.current.identity = Some(Identity {
375 logical_request_id: self.logical,
376 attempt_id: Uuid::new_v4(),
377 wire_attempt: self.number + 1,
378 });
379 self.wire(|id| WireEvent::Request {
380 identity: id,
381 request: &request,
382 })?;
383 if call.cancellation.is_cancelled() {
384 self.current.identity = None;
385 self.cancel();
386 return Err(self.end(Outcome::Cancelled));
387 }
388 if self.number > 0 && self.expired() {
389 self.current.identity = None;
390 return Err(self.end(Outcome::BudgetExpired));
391 }
392 self.current.phase = None;
393 if self.number > 0 {
394 self.emit(Transition::RetryStarted, None, false);
395 }
396 self.emit(Transition::AttemptStarted, None, false);
397 if call.cancellation.is_cancelled() {
398 self.current.identity = None;
399 self.cancel();
400 return Err(self.end(Outcome::Cancelled));
401 }
402 if self.number > 0 && self.expired() {
403 self.current.identity = None;
404 return Err(self.end(Outcome::BudgetExpired));
405 }
406 let metadata_request = request.try_clone();
407 let mut execution = Box::pin(client.execute(request));
408 let mut dispatched = false;
409 let response = tokio::select! { biased;
410 _ = call.cancellation.cancelled() => {
411 if !dispatched { self.current.identity = None; }
412 else { self.number += 1; self.sent_current = true; }
413 self.cancel(); return Err(self.end(Outcome::Cancelled));
414 },
415 response = std::future::poll_fn(|cx| { if !dispatched { if let Some(timing) = &self.stream_timing { timing.dispatch(); } } dispatched = true; execution.as_mut().poll(cx) }) => response,
416 };
417 self.number += 1;
418 self.sent_current = true;
419 match response {
420 Ok(response) => {
421 self.current.set_response_headers(response.headers());
422 self.current.http_status = Some(response.status().as_u16());
423 self.current.progress.headers_received = true;
424 self.current.phase = Some(Phase::Streaming);
425 self.current.retry_after = parse_retry_after(
426 response.headers().get("retry-after"),
427 self.policy.timing.wall(),
428 );
429 self.current.provider_request_id = response
430 .headers()
431 .get("x-request-id")
432 .and_then(|v| v.to_str().ok())
433 .and_then(ValidatedMetadata::request_id)
434 .filter(|id| {
435 metadata_request
436 .as_ref()
437 .is_some_and(|request| !echoes_request(id.as_str(), request))
438 });
439 self.wire(|id| WireEvent::Headers {
440 identity: id,
441 status: response.status().as_u16(),
442 headers: response.headers(),
443 })?;
444 Ok(response)
445 }
446 Err(cause) => {
447 if cause.is_connect() {
448 self.current.phase = Some(Phase::Connecting);
449 }
450 let category = if cause.is_timeout() {
451 Category::ClientTimeout
452 } else {
453 Category::Transport
454 };
455 self.failure(category, cause);
456 Err(self.end(Outcome::Failed))
457 }
458 }
459 }
460 fn report(&self) -> RecoveryReport {
461 RecoveryReport {
462 stream_metrics: None,
463 identity: self.current.identity.clone().unwrap_or(Identity {
464 logical_request_id: self.logical,
465 attempt_id: Uuid::nil(),
466 wire_attempt: self.number,
467 }),
468 progress: self.progress(),
469 history: self.history.clone(),
470 }
471 }
472 fn decode_failure(&mut self, mut cause: PrivateEvidence) {
473 let body = serde_json::from_slice::<serde_json::Value>(&cause.bytes).ok();
474 let category = if body.as_ref().is_some_and(|body| !body["error"].is_null()) {
475 Category::ProviderResponse
476 } else {
477 Category::Protocol
478 };
479 self.current.provider_code =
480 body.and_then(|body| body["error"]["code"].as_str().and_then(ValidatedMetadata::code));
481 if let Some(original) = cause.original.take() {
482 self.current.set_response_bytes(&self.raw);
483 self.current.category = category;
484 self.current = self.current.clone().with_boxed_cause(original);
485 self.classify();
486 } else {
487 self.failure(category, cause);
488 }
489 }
490 fn http_failure(&mut self, bytes: Vec<u8>) {
491 self.current.provider_code = serde_json::from_slice::<serde_json::Value>(&bytes)
492 .ok()
493 .and_then(|v| v["error"]["code"].as_str().and_then(ValidatedMetadata::code));
494 self.failure(Category::Http, PrivateEvidence::new(bytes));
495 }
496}
497fn echoes_request(value: &str, request: &reqwest::Request) -> bool {
500 let candidates = [value, value.strip_prefix("req_").unwrap_or(value)];
501 candidates.iter().any(|candidate| {
502 let bytes = candidate.as_bytes();
503 let contains = |source: &[u8]| source.windows(bytes.len()).any(|part| part == bytes);
504 request.headers().values().any(|header| contains(header.as_bytes()))
505 || request.body().and_then(|body| body.as_bytes()).is_some_and(contains)
506 })
507}
508fn parse_retry_after(
509 value: Option<&reqwest::header::HeaderValue>,
510 wall: std::time::SystemTime,
511) -> RetryAfter {
512 let Some(value) = value else {
513 return RetryAfter::Absent;
514 };
515 let Ok(value) = value.to_str() else {
516 return RetryAfter::Invalid;
517 };
518 let value = value.trim();
519 if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) {
520 return value.parse().map(RetryAfter::Seconds).unwrap_or(RetryAfter::Invalid);
521 }
522 let value = value.trim();
523 let obsolete = || {
526 use chrono::Datelike;
527 let (weekday, rest) = value.split_once(", ")?;
528 let (date, time) = rest.split_once(' ')?;
529 let (day_month, short_year) = date.rsplit_once('-')?;
530 if short_year.len() != 2 || !short_year.bytes().all(|b| b.is_ascii_digit()) {
531 return None;
532 }
533 let horizon = chrono::DateTime::<chrono::Utc>::from(wall).year().saturating_add(50);
534 let mut year = horizon.div_euclid(100) * 100 + short_year.parse::<i32>().ok()?;
535 if year > horizon {
536 year -= 100;
537 }
538 let expanded = format!("{weekday}, {day_month}-{year:04} {time}");
539 chrono::NaiveDateTime::parse_from_str(&expanded, "%A, %d-%b-%Y %H:%M:%S GMT").ok()
540 };
541 let parsed = chrono::NaiveDateTime::parse_from_str(value, "%a, %d %b %Y %H:%M:%S GMT")
542 .ok()
543 .or_else(obsolete)
544 .or_else(|| chrono::NaiveDateTime::parse_from_str(value, "%a %b %e %H:%M:%S %Y").ok());
545 match parsed {
546 Some(date) => {
547 let seconds = date.and_utc().timestamp();
548 let remaining = if seconds < 0 {
549 Duration::ZERO
550 } else {
551 UNIX_EPOCH
552 .checked_add(Duration::from_secs(seconds as u64))
553 .and_then(|date| date.duration_since(wall).ok())
554 .unwrap_or_default()
555 };
556 RetryAfter::Date {
557 unix_seconds: seconds,
558 remaining,
559 }
560 }
561 None => RetryAfter::Invalid,
562 }
563}
564async fn body(
565 state: &mut Attempt<'_>,
566 response: reqwest::Response,
567 call: &RecoveryCall,
568) -> Result<Vec<u8>, RecoveryError> {
569 let mut bytes = response.bytes_stream();
570 let mut collected = Vec::new();
571 loop {
572 let chunk = tokio::select! { biased;
573 _ = call.cancellation.cancelled() => { state.cancel(); state.current = state.current.clone().with_cause(PrivateEvidence::new(collected)); return Err(state.end(Outcome::Cancelled)); },
574 chunk = bytes.next() => chunk,
575 };
576 match chunk {
577 Some(Ok(chunk)) => {
578 state.current.progress.raw_bytes =
579 state.current.progress.raw_bytes.saturating_add(chunk.len() as u64);
580 state.raw.extend_from_slice(&chunk);
581 collected.extend_from_slice(&chunk);
582 if state.operation != Operation::Streaming
583 && state.current.http_status.is_some_and(|s| (200..300).contains(&s))
584 {
585 super::frames::observe_response(
586 state.provider,
587 &collected,
588 &mut state.current.progress.observed,
589 );
590 }
591 state.wire(|id| WireEvent::Body {
593 identity: id,
594 bytes: &chunk,
595 })?;
596 }
597 Some(Err(cause)) => {
598 let category =
601 if state.current.http_status.is_some_and(|s| !(200..300).contains(&s)) {
602 Category::Http
603 } else {
604 Category::Transport
605 };
606 state.failure(category, cause);
607 return Err(state.end(Outcome::Failed));
608 }
609 None => return Ok(collected),
610 }
611 }
612}
613pub(super) async fn complete<G: RecoveryProvider>(
614 gateway: &RecoveryGateway<G>,
615 request: reqwest::Request,
616 operation: Operation,
617 call: &RecoveryCall,
618) -> Result<Recovered<LlmGatewayResponse<serde_json::Value>>, RecoveryError> {
619 let mut state = Attempt::new(&gateway.policy, gateway.inner.recovery_provider(), operation);
620 let structured_output = operation == Operation::Structured
621 || request
622 .body()
623 .and_then(|body| body.as_bytes())
624 .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(bytes).ok())
625 .is_some_and(|body| {
626 matches!(
627 body["response_format"]["type"].as_str(),
628 Some("json_object" | "json_schema")
629 )
630 });
631 loop {
632 let result = async {
633 let response = state.send(&gateway.client, &request, call).await?;
634 let success = response.status().is_success();
635 let collected = body(&mut state, response, call).await?;
636 if !success {
637 state.http_failure(collected);
638 return Err(state.end(Outcome::Failed));
639 }
640 state.current.phase = Some(Phase::Decoding);
641 match super::frames::decode(
642 state.provider,
643 &collected,
644 operation == Operation::Structured,
645 ) {
646 Ok(mut response) => {
647 if state.provider == Provider::Omlx && structured_output {
648 if let Some(warning) = state
649 .current
650 .inspect_response_headers()
651 .and_then(crate::llm::gateways::omlx::format_warning_headers)
652 {
653 response.evidence.metadata.insert(
654 crate::llm::gateways::omlx::RESPONSE_FORMAT_WARNING.into(),
655 serde_json::Value::String(warning),
656 );
657 }
658 }
659 super::frames::response_progress(
660 &response,
661 &mut state.current.progress.observed,
662 );
663 *state.delivered.lock().unwrap_or_else(|e| e.into_inner()) =
664 state.current.progress.observed;
665 Ok(response)
666 }
667 Err(cause) => {
668 state.decode_failure(cause);
669 Err(state.end(Outcome::Failed))
670 }
671 }
672 }
673 .await;
674 match result {
675 Ok(value) => {
676 state.emit(Transition::AttemptSucceeded, None, false);
677 return Ok(Recovered {
678 value,
679 report: state.report(),
680 });
681 }
682 Err(error) if error.outcome == Outcome::ObserverFailed => return Err(error),
683 Err(_) => state.retry(call).await?,
684 }
685 }
686}
687pub(super) fn stream<G: RecoveryProvider>(
688 gateway: &RecoveryGateway<G>,
689 request: reqwest::Request,
690 call: &RecoveryCall,
691) -> Pin<Box<dyn Stream<Item = RecoveryStreamEvent> + Send>> {
692 let owner = StreamOwner {
693 client: gateway.client.clone(),
694 policy: gateway.policy.clone(),
695 provider: gateway.inner.recovery_provider(),
696 timing: StreamTiming::new(gateway.inner.stream_clock()),
697 };
698 let call = RecoveryCall {
699 cancellation: call.cancellation.child_token(),
700 };
701 Box::pin(async_stream::stream! {
702 let _guard = call.cancellation.clone().drop_guard();
703 let delivered = std::sync::Arc::new(std::sync::Mutex::new(SemanticProgress::default()));
704 let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
705 let worker_call = call.clone();
706 let worker_delivered = delivered.clone();
707 tokio::spawn(async move {
708 let timing = owner.timing.clone();
709 let mut provider_metrics = StreamMetrics { provider: format!("{:?}", owner.provider).to_lowercase(), ..Default::default() };
710 let mut source = stream_owned(owner, request, worker_call.clone(), worker_delivered);
711 loop {
712 let permit = tokio::select! { biased;
716 _ = worker_call.cancellation.cancelled() => None,
717 permit = sender.reserve() => match permit {
718 Ok(permit) => Some(permit),
719 Err(_) => break,
720 },
721 };
722 let Some(mut event) = source.next().await else { break; };
723 match &mut event {
724 RecoveryStreamEvent::Metrics { metrics, .. } => provider_metrics = metrics.clone(),
725 RecoveryStreamEvent::Completed(result) => {
726 let metrics = timing.finish(provider_metrics.clone());
727 retain_timing(&mut result.value.evidence, &metrics);
728 result.report.stream_metrics = Some(Box::new(metrics));
729 },
730 RecoveryStreamEvent::Failed(error) => error.stream_metrics = Some(Box::new(timing.finish(provider_metrics.clone()))),
731 _ => {},
732 }
733 let terminal = matches!(event, RecoveryStreamEvent::Completed(_) | RecoveryStreamEvent::Failed(_));
734 if let Some(permit) = permit { permit.send(event); }
735 else if terminal { let _ = sender.send(event).await; }
736 if terminal { break; }
737 }
738 });
739 while let Some(mut event) = receiver.recv().await {
740 if call.cancellation.is_cancelled() && !matches!(event, RecoveryStreamEvent::Failed(_) | RecoveryStreamEvent::Completed(_)) { continue; }
741 {
742 let mut delivered = delivered.lock().unwrap_or_else(|e| e.into_inner());
743 super::frames::delivered(&event, &mut delivered);
744 match &mut event {
745 RecoveryStreamEvent::Failed(error) => { error.failure.progress.delivered = *delivered;
746 if let Some(last) = error.history.last_mut().filter(|last| last.identity == error.failure.identity) { last.progress.delivered = *delivered; }
747 },
748 RecoveryStreamEvent::Completed(result) => { delivered.completed_tool_calls = result.value.tool_calls.len() as u64; result.report.progress.delivered = *delivered; },
749 _ => {},
750 }
751 }
752 yield event;
753 }
754 })
755}
756struct StreamOwner {
757 client: reqwest::Client,
758 policy: RecoveryPolicy,
759 provider: Provider,
760 timing: StreamTiming,
761}
762fn stream_owned(
763 gateway: StreamOwner,
764 request: reqwest::Request,
765 call: RecoveryCall,
766 delivered: std::sync::Arc<std::sync::Mutex<SemanticProgress>>,
767) -> Pin<Box<dyn Stream<Item = RecoveryStreamEvent> + Send>> {
768 Box::pin(async_stream::stream! {
769 let mut state = Attempt::new(&gateway.policy, gateway.provider, Operation::Streaming);
770 state.delivered = delivered;
771 state.stream_timing = Some(gateway.timing.clone());
772 loop {
773 let response = match state.send(&gateway.client, &request, &call).await {
774 Ok(response) => response,
775 Err(error) => { if error.outcome == Outcome::ObserverFailed { yield RecoveryStreamEvent::Failed(error); break; } match state.retry(&call).await { Ok(()) => continue, Err(error) => { yield RecoveryStreamEvent::Failed(error); break; } } },
776 };
777 if !response.status().is_success() {
778 match body(&mut state, response, &call).await { Ok(bytes) => state.http_failure(bytes), Err(error) if error.outcome == Outcome::ObserverFailed => { yield RecoveryStreamEvent::Failed(error); break; }, Err(_) => {} }
779 } else {
780 let mut bytes = response.bytes_stream();
781 let mut lines = LineBuffer::default();
782 let mut frames = Frames::new(state.provider);
783 let mut failed = false;
784 let mut raw_evidence = Vec::new();
785 while !frames.terminal {
786 let chunk = tokio::select! { biased;
787 _ = call.cancellation.cancelled() => { state.cancel(); failed = true; None },
788 chunk = bytes.next() => chunk,
789 };
790 let Some(chunk) = chunk else { if !failed { state.failure(if lines.has_pending() { Category::Protocol } else { Category::Transport }, PrivateEvidence::new(b"stream ended without terminal proof".to_vec())); } break; };
791 let chunk = match chunk { Ok(chunk) => chunk, Err(cause) => { state.failure(if lines.has_pending() { Category::Protocol } else { Category::Transport }, cause); break; } };
792 state.current.progress.raw_bytes = state.current.progress.raw_bytes.saturating_add(chunk.len() as u64);
793 raw_evidence.extend_from_slice(&chunk);
794 state.raw.extend_from_slice(&chunk);
795 let mut output = Vec::new();
797 for line in lines.push(&chunk) {
798 if let Ok(text) = std::str::from_utf8(&line) { gateway.timing.observe_line(text); }
799 let (events, result) = frames.parse(&line, &mut state.current.progress.observed, state.current.identity.as_ref());
800 output.extend(events);
801 if let Err(cause) = result { state.decode_failure(cause); failed = true; break; }
802 }
803 if let Err(error) = state.wire(|id| WireEvent::Body { identity: id, bytes: &chunk }) { yield RecoveryStreamEvent::Failed(error); return; }
804 for event in output {
805 if call.cancellation.is_cancelled() { state.cancel(); failed = true; break; }
806 yield event;
807 }
808 if failed { break; }
809 }
810 if call.cancellation.is_cancelled() { state.cancel(); failed = true; }
813 drop(bytes);
814 if state.current.category == Category::Cancellation { state.current = state.current.clone().with_cause(PrivateEvidence::new(raw_evidence)); }
815 if frames.terminal && !failed {
816 match frames.response() {
817 Ok(value) => { state.emit(Transition::AttemptSucceeded, None, false); yield RecoveryStreamEvent::Completed(Box::new(Recovered { value, report: state.report() })); return; },
818 Err(cause) => state.decode_failure(cause),
819 }
820 }
821 }
822 match state.retry(&call).await { Ok(()) => {}, Err(error) => { yield RecoveryStreamEvent::Failed(error); break; } }
823 }
824 })
825}