mojentic/llm/recovery/
policy.rs1use super::types::*;
2use async_trait::async_trait;
3use std::{
4 collections::BTreeSet,
5 num::NonZeroU32,
6 sync::Arc,
7 time::{Duration, Instant, SystemTime},
8};
9use tokio_util::sync::CancellationToken;
10
11#[async_trait]
13pub trait Timing: Send + Sync {
14 fn monotonic(&self) -> Instant;
15 fn wall(&self) -> SystemTime;
16 fn jitter(&self, ceiling: Duration) -> Duration;
17 async fn sleep(&self, delay: Duration);
18}
19pub struct SystemTiming;
20#[async_trait]
21impl Timing for SystemTiming {
22 fn monotonic(&self) -> Instant {
23 Instant::now()
24 }
25 fn wall(&self) -> SystemTime {
26 SystemTime::now()
27 }
28 fn jitter(&self, ceiling: Duration) -> Duration {
29 let random = UuidBytes::random();
30 Duration::from_nanos(
31 (ceiling.as_nanos().min(u64::MAX as u128) * random as u128 / u64::MAX as u128) as u64,
32 )
33 }
34 async fn sleep(&self, delay: Duration) {
35 tokio::time::sleep(delay).await;
36 }
37}
38struct UuidBytes;
39impl UuidBytes {
40 fn random() -> u64 {
41 let bytes = *uuid::Uuid::new_v4().as_bytes();
42 u64::from_le_bytes([
43 bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[9], bytes[10],
44 ])
45 }
46}
47#[derive(Debug, Clone)]
48pub struct AdmissionContext {
49 pub failure: Failure,
50 pub next_attempt: u32,
51 pub previous_identity: Identity,
52}
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum AdmissionDecision {
55 Allow,
56 Reject,
57 Unresolved,
59}
60#[async_trait]
63pub trait Admission: Send + Sync {
64 async fn decide(&self, context: AdmissionContext) -> AdmissionDecision;
65}
66pub trait LifecycleObserver: Send + Sync {
67 fn observe(&self, event: &LifecycleEvent);
68}
69pub enum WireEvent<'a> {
71 Request {
72 identity: Identity,
73 request: &'a reqwest::Request,
74 },
75 Headers {
76 identity: Identity,
77 status: u16,
78 headers: &'a reqwest::header::HeaderMap,
79 },
80 Body {
81 identity: Identity,
82 bytes: &'a [u8],
83 },
84}
85pub trait WireObserver: Send + Sync {
86 fn observe(&self, event: WireEvent<'_>)
88 -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
89}
90#[derive(Clone)]
91pub struct RecoveryPolicy {
92 pub max_attempts: NonZeroU32,
93 pub base_delay: Duration,
94 pub delay_ceiling: Duration,
95 pub retryable_statuses: BTreeSet<u16>,
96 pub retryable_categories: BTreeSet<Category>,
98 pub budget: Option<Duration>,
99 pub deadline: Option<Instant>,
100 pub timing: Arc<dyn Timing>,
101 pub admission: Option<Arc<dyn Admission>>,
102 pub observer: Option<Arc<dyn LifecycleObserver>>,
103 pub wire_observer: Option<Arc<dyn WireObserver>>,
104}
105impl Default for RecoveryPolicy {
106 fn default() -> Self {
107 Self {
108 max_attempts: NonZeroU32::MIN,
109 base_delay: Duration::from_millis(100),
110 delay_ceiling: Duration::from_secs(30),
111 retryable_statuses: [429, 500, 502, 503, 504].into_iter().collect(),
112 retryable_categories: [Category::Transport, Category::Http].into_iter().collect(),
113 budget: None,
114 deadline: None,
115 timing: Arc::new(SystemTiming),
116 admission: None,
117 observer: None,
118 wire_observer: None,
119 }
120 }
121}
122#[derive(Clone, Default)]
124pub struct RecoveryCall {
125 pub cancellation: CancellationToken,
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
132pub enum RecoveryPolicyError {
133 #[error("retryable status must be an HTTP error status: {status}")]
135 InvalidStatus { status: u16 },
136 #[error("unsupported retryable category: {category:?}")]
138 UnsupportedCategory { category: Category },
139}
140impl RecoveryPolicy {
141 pub fn validate(&self) -> Result<(), RecoveryPolicyError> {
148 for &status in &self.retryable_statuses {
149 if !(400..=599).contains(&status) {
150 return Err(RecoveryPolicyError::InvalidStatus { status });
151 }
152 }
153 for &category in &self.retryable_categories {
154 if !matches!(category, Category::Transport | Category::Http | Category::ClientTimeout) {
155 return Err(RecoveryPolicyError::UnsupportedCategory { category });
156 }
157 }
158 Ok(())
159 }
160}