Skip to main content

LlmGateway

Trait LlmGateway 

Source
pub trait LlmGateway: Send + Sync {
    // Required methods
    fn complete<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>(
        &'life0 self,
        model: &'life1 str,
        messages: &'life2 [LlmMessage],
        tools: Option<&'life3 [Box<dyn LlmTool>]>,
        config: &'life4 CompletionConfig,
    ) -> Pin<Box<dyn Future<Output = Result<LlmGatewayResponse>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait,
             'life4: 'async_trait;
    fn complete_json<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        model: &'life1 str,
        messages: &'life2 [LlmMessage],
        schema: Value,
        config: &'life3 CompletionConfig,
    ) -> Pin<Box<dyn Future<Output = Result<Value>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;
    fn get_available_models<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn calculate_embeddings<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        text: &'life1 str,
        model: Option<&'life2 str>,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn complete_stream<'a>(
        &'a self,
        model: &'a str,
        messages: &'a [LlmMessage],
        tools: Option<&'a [Box<dyn LlmTool>]>,
        config: &'a CompletionConfig,
    ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>;

    // Provided methods
    fn stream_clock(&self) -> Arc<dyn StreamClock> ⓘ { ... }
    fn get_model_context_window<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _model: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<ModelContextWindow>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn complete_json_response<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        model: &'life1 str,
        messages: &'life2 [LlmMessage],
        schema: Value,
        config: &'life3 CompletionConfig,
    ) -> Pin<Box<dyn Future<Output = Result<LlmGatewayResponse<Value>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait { ... }
    fn complete_stream_events<'a>(
        &'a self,
        _model: &'a str,
        _messages: &'a [LlmMessage],
        _config: &'a CompletionConfig,
    ) -> Result<StreamEventStream<'a>, StreamEventError> { ... }
}
Expand description

Abstract interface for LLM providers

Required Methods§

Source

fn complete<'life0, 'life1, 'life2, 'life3, 'life4, 'async_trait>( &'life0 self, model: &'life1 str, messages: &'life2 [LlmMessage], tools: Option<&'life3 [Box<dyn LlmTool>]>, config: &'life4 CompletionConfig, ) -> Pin<Box<dyn Future<Output = Result<LlmGatewayResponse>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait, 'life4: 'async_trait,

Complete an LLM request with text response

Source

fn complete_json<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, model: &'life1 str, messages: &'life2 [LlmMessage], schema: Value, config: &'life3 CompletionConfig, ) -> Pin<Box<dyn Future<Output = Result<Value>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Complete an LLM request with structured JSON response

Source

fn get_available_models<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Get list of available models

Source

fn calculate_embeddings<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, text: &'life1 str, model: Option<&'life2 str>, ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Calculate embeddings for text

Source

fn complete_stream<'a>( &'a self, model: &'a str, messages: &'a [LlmMessage], tools: Option<&'a [Box<dyn LlmTool>]>, config: &'a CompletionConfig, ) -> Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send + 'a>>

Stream LLM responses chunk by chunk

Returns a stream that yields either content chunks or tool calls. Tool calls will be accumulated and yielded when complete.

Provided Methods§

Source

fn stream_clock(&self) -> Arc<dyn StreamClock> ⓘ

Monotonic clock used by maintained streaming adapters.

Source

fn get_model_context_window<'life0, 'life1, 'async_trait>( &'life0 self, _model: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<ModelContextWindow>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Query advertised context capacity without changing subsequent requests.

Third-party gateways default to not advertised.

§Errors

Returns the gateway’s existing transport, status or malformed-body error.

Source

fn complete_json_response<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, model: &'life1 str, messages: &'life2 [LlmMessage], schema: Value, config: &'life3 CompletionConfig, ) -> Pin<Box<dyn Future<Output = Result<LlmGatewayResponse<Value>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Complete a structured JSON request and keep the provider’s evidence.

The response carries the parsed object, the raw content when the gateway has it, and crate::llm::models::ResponseEvidence. The default implementation wraps LlmGateway::complete_json and reports no evidence; gateways that see provider metadata override it.

Source

fn complete_stream_events<'a>( &'a self, _model: &'a str, _messages: &'a [LlmMessage], _config: &'a CompletionConfig, ) -> Result<StreamEventStream<'a>, StreamEventError>

Stream one turn as crate::llm::StreamEvents with terminal completion evidence.

No tools are sent. The returned stream sends exactly one request when first polled and ends with exactly one terminal event. Dropping it cancels the request.

§Errors

Returns StreamEventError::StreamEventsUnsupported when the gateway does not implement the events API (the default), or StreamEventError::RequestFailed when the request cannot be built. No request is sent in either case. The broker turns either into a single terminal error event.

Dyn Compatibility§

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors§