Skip to main content

mojentic/llm/recovery/
policy.rs

1use super::types::*;
2use async_trait::async_trait;
3use std::{
4    collections::BTreeSet,
5    num::NonZeroU32,
6    sync::Arc,
7    time::{Duration, Instant, SystemTime},
8};
9use tokio_util::sync::CancellationToken;
10
11/// Inject both monotonic and wall time, sleep, and full jitter for deterministic tests.
12#[async_trait]
13pub trait Timing: Send + Sync {
14    fn monotonic(&self) -> Instant;
15    fn wall(&self) -> SystemTime;
16    fn jitter(&self, ceiling: Duration) -> Duration;
17    async fn sleep(&self, delay: Duration);
18}
19pub struct SystemTiming;
20#[async_trait]
21impl Timing for SystemTiming {
22    fn monotonic(&self) -> Instant {
23        Instant::now()
24    }
25    fn wall(&self) -> SystemTime {
26        SystemTime::now()
27    }
28    fn jitter(&self, ceiling: Duration) -> Duration {
29        let random = UuidBytes::random();
30        Duration::from_nanos(
31            (ceiling.as_nanos().min(u64::MAX as u128) * random as u128 / u64::MAX as u128) as u64,
32        )
33    }
34    async fn sleep(&self, delay: Duration) {
35        tokio::time::sleep(delay).await;
36    }
37}
38struct UuidBytes;
39impl UuidBytes {
40    fn random() -> u64 {
41        let bytes = *uuid::Uuid::new_v4().as_bytes();
42        u64::from_le_bytes([
43            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[9], bytes[10],
44        ])
45    }
46}
47#[derive(Debug, Clone)]
48pub struct AdmissionContext {
49    pub failure: Failure,
50    pub next_attempt: u32,
51    pub previous_identity: Identity,
52}
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum AdmissionDecision {
55    Allow,
56    Reject,
57    /// The hook completed without granting or rejecting permission. Never resend.
58    Unresolved,
59}
60/// Only `Allow` grants permission. `Unresolved` completes without a decision and stops
61/// recovery; a pending future stays pending until a decision, cancellation, or expiry.
62#[async_trait]
63pub trait Admission: Send + Sync {
64    async fn decide(&self, context: AdmissionContext) -> AdmissionDecision;
65}
66pub trait LifecycleObserver: Send + Sync {
67    fn observe(&self, event: &LifecycleEvent);
68}
69/// Wire capture is explicit and sensitive. The library never stores captures by default.
70pub enum WireEvent<'a> {
71    Request {
72        identity: Identity,
73        request: &'a reqwest::Request,
74    },
75    Headers {
76        identity: Identity,
77        status: u16,
78        headers: &'a reqwest::header::HeaderMap,
79    },
80    Body {
81        identity: Identity,
82        bytes: &'a [u8],
83    },
84}
85pub trait WireObserver: Send + Sync {
86    /// A capture failure terminates recovery; it never causes a resend.
87    fn observe(&self, event: WireEvent<'_>)
88        -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
89}
90#[derive(Clone)]
91pub struct RecoveryPolicy {
92    pub max_attempts: NonZeroU32,
93    pub base_delay: Duration,
94    pub delay_ceiling: Duration,
95    pub retryable_statuses: BTreeSet<u16>,
96    /// Protocol and provider-response errors remain ineligible without a proven provider policy.
97    pub retryable_categories: BTreeSet<Category>,
98    pub budget: Option<Duration>,
99    pub deadline: Option<Instant>,
100    pub timing: Arc<dyn Timing>,
101    pub admission: Option<Arc<dyn Admission>>,
102    pub observer: Option<Arc<dyn LifecycleObserver>>,
103    pub wire_observer: Option<Arc<dyn WireObserver>>,
104}
105impl Default for RecoveryPolicy {
106    fn default() -> Self {
107        Self {
108            max_attempts: NonZeroU32::MIN,
109            base_delay: Duration::from_millis(100),
110            delay_ceiling: Duration::from_secs(30),
111            retryable_statuses: [429, 500, 502, 503, 504].into_iter().collect(),
112            retryable_categories: [Category::Transport, Category::Http].into_iter().collect(),
113            budget: None,
114            deadline: None,
115            timing: Arc::new(SystemTiming),
116            admission: None,
117            observer: None,
118            wire_observer: None,
119        }
120    }
121}
122/// Per-call cancellation, independent of remote inference termination.
123#[derive(Clone, Default)]
124pub struct RecoveryCall {
125    pub cancellation: CancellationToken,
126}
127
128/// Invalid recovery options, rejected before a wrapper can dispatch a request.
129/// Positive attempt counts and nonnegative finite delays are enforced by Rust's
130/// `NonZeroU32` and `Duration` types. Base delays above the ceiling are valid.
131#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
132pub enum RecoveryPolicyError {
133    /// Only HTTP error statuses can be selected for recovery.
134    #[error("retryable status must be an HTTP error status: {status}")]
135    InvalidStatus { status: u16 },
136    /// No provider policy proves replay safe for this category.
137    #[error("unsupported retryable category: {category:?}")]
138    UnsupportedCategory { category: Category },
139}
140impl RecoveryPolicy {
141    /// Validate options before creating a recovery gateway.
142    ///
143    /// # Errors
144    /// Returns a concrete policy error for non-error HTTP statuses or categories
145    /// without a supported recovery policy. Permanent statuses remain ineligible
146    /// even when explicitly selected.
147    pub fn validate(&self) -> Result<(), RecoveryPolicyError> {
148        for &status in &self.retryable_statuses {
149            if !(400..=599).contains(&status) {
150                return Err(RecoveryPolicyError::InvalidStatus { status });
151            }
152        }
153        for &category in &self.retryable_categories {
154            if !matches!(category, Category::Transport | Category::Http | Category::ClientTimeout) {
155                return Err(RecoveryPolicyError::UnsupportedCategory { category });
156            }
157        }
158        Ok(())
159    }
160}