Custom Providers

Custom Providers

Codive supports multiple LLM providers out of the box. You can also add support for new providers by implementing the Provider trait.

Built-in Providers

Provider	Models	Status
Anthropic	Claude 3.5 Sonnet, Claude Opus 4, Claude Sonnet 4	Stable
OpenAI	GPT-4, GPT-4 Turbo	Stable

The Provider Trait

Providers implement the Provider trait to connect Codive to different LLMs:

use async_trait::async_trait;
use futures::Stream;
use std::pin::Pin;

#[async_trait]
pub trait Provider: Send + Sync {
    /// Provider name for configuration
    fn name(&self) -> &str;

    /// List available models
    fn available_models(&self) -> Vec<ModelInfo>;

    /// Send a message and get a complete response
    async fn complete(
        &self,
        request: CompletionRequest,
    ) -> Result<CompletionResponse, ProviderError>;

    /// Stream a response token by token
    async fn stream(
        &self,
        request: CompletionRequest,
    ) -> Result<Pin<Box<dyn Stream<Item = StreamEvent> + Send>>, ProviderError>;
}

Creating a Custom Provider

Let’s implement a provider for a hypothetical “LocalLLM” service:

1. Create the provider module

   Create src/providers/local_llm.rs:

   use async_trait::async_trait;
   use futures::{Stream, StreamExt};
   use reqwest::Client;
   use serde::{Deserialize, Serialize};
   use std::pin::Pin;
   
   use crate::provider::{
       CompletionRequest, CompletionResponse, ModelInfo,
       Provider, ProviderError, StreamEvent,
   };
   
   pub struct LocalLlmProvider {
       client: Client,
       base_url: String,
       model: String,
   }
   
   impl LocalLlmProvider {
       pub fn new(base_url: &str, model: &str) -> Self {
           Self {
               client: Client::new(),
               base_url: base_url.to_string(),
               model: model.to_string(),
           }
       }
   }
   
   #[derive(Serialize)]
   struct LocalLlmRequest {
       model: String,
       messages: Vec<Message>,
       stream: bool,
       max_tokens: Option<u32>,
       temperature: Option<f32>,
   }
   
   #[derive(Serialize, Deserialize)]
   struct Message {
       role: String,
       content: String,
   }
   
   #[derive(Deserialize)]
   struct LocalLlmResponse {
       id: String,
       content: String,
       usage: Usage,
   }
   
   #[derive(Deserialize)]
   struct Usage {
       prompt_tokens: u32,
       completion_tokens: u32,
   }
   
   #[async_trait]
   impl Provider for LocalLlmProvider {
       fn name(&self) -> &str {
           "local_llm"
       }
   
       fn available_models(&self) -> Vec<ModelInfo> {
           vec![
               ModelInfo {
                   id: "local-7b".to_string(),
                   name: "Local 7B".to_string(),
                   context_window: 4096,
                   max_output_tokens: 2048,
               },
               ModelInfo {
                   id: "local-13b".to_string(),
                   name: "Local 13B".to_string(),
                   context_window: 8192,
                   max_output_tokens: 4096,
               },
           ]
       }
   
       async fn complete(
           &self,
           request: CompletionRequest,
       ) -> Result<CompletionResponse, ProviderError> {
           let messages: Vec<Message> = request
               .messages
               .iter()
               .map(|m| Message {
                   role: m.role.clone(),
                   content: m.content.clone(),
               })
               .collect();
   
           let local_request = LocalLlmRequest {
               model: self.model.clone(),
               messages,
               stream: false,
               max_tokens: request.max_tokens,
               temperature: request.temperature,
           };
   
           let response = self
               .client
               .post(format!("{}/v1/chat/completions", self.base_url))
               .json(&local_request)
               .send()
               .await
               .map_err(|e| ProviderError::Network(e.to_string()))?;
   
           if !response.status().is_success() {
               let error = response.text().await.unwrap_or_default();
               return Err(ProviderError::Api(error));
           }
   
           let local_response: LocalLlmResponse = response
               .json()
               .await
               .map_err(|e| ProviderError::Parse(e.to_string()))?;
   
           Ok(CompletionResponse {
               id: local_response.id,
               content: local_response.content,
               tool_calls: vec![],
               usage: crate::provider::Usage {
                   input_tokens: local_response.usage.prompt_tokens,
                   output_tokens: local_response.usage.completion_tokens,
               },
           })
       }
   
       async fn stream(
           &self,
           request: CompletionRequest,
       ) -> Result<Pin<Box<dyn Stream<Item = StreamEvent> + Send>>, ProviderError> {
           let messages: Vec<Message> = request
               .messages
               .iter()
               .map(|m| Message {
                   role: m.role.clone(),
                   content: m.content.clone(),
               })
               .collect();
   
           let local_request = LocalLlmRequest {
               model: self.model.clone(),
               messages,
               stream: true,
               max_tokens: request.max_tokens,
               temperature: request.temperature,
           };
   
           let response = self
               .client
               .post(format!("{}/v1/chat/completions", self.base_url))
               .json(&local_request)
               .send()
               .await
               .map_err(|e| ProviderError::Network(e.to_string()))?;
   
           let stream = response
               .bytes_stream()
               .map(|chunk| {
                   match chunk {
                       Ok(bytes) => {
                           // Parse SSE events
                           let text = String::from_utf8_lossy(&bytes);
                           StreamEvent::Delta(text.to_string())
                       }
                       Err(e) => StreamEvent::Error(e.to_string()),
                   }
               });
   
           Ok(Box::pin(stream))
       }
   }

2. Register the provider

   In src/providers/mod.rs:

   mod anthropic;
   mod openai;
   mod local_llm;
   
   pub use anthropic::AnthropicProvider;
   pub use openai::OpenAiProvider;
   pub use local_llm::LocalLlmProvider;
   
   pub fn create_provider(config: &ProviderConfig) -> Box<dyn Provider> {
       match config.name.as_str() {
           "anthropic" => Box::new(AnthropicProvider::from_config(config)),
           "openai" => Box::new(OpenAiProvider::from_config(config)),
           "local_llm" => Box::new(LocalLlmProvider::new(
               &config.base_url.unwrap_or("http://localhost:8080".to_string()),
               &config.model,
           )),
           _ => panic!("Unknown provider: {}", config.name),
       }
   }

3. Configure the provider

   In codive.toml:

   [provider]
   name = "local_llm"
   model = "local-13b"
   base_url = "http://localhost:8080"

Provider Types

Request/Response Types

/// Request sent to the provider
pub struct CompletionRequest {
    /// Conversation messages
    pub messages: Vec<ChatMessage>,

    /// Available tools the model can call
    pub tools: Vec<ToolDefinition>,

    /// Maximum tokens to generate
    pub max_tokens: Option<u32>,

    /// Temperature (0.0 - 1.0)
    pub temperature: Option<f32>,

    /// Stop sequences
    pub stop: Option<Vec<String>>,
}

/// Response from the provider
pub struct CompletionResponse {
    /// Unique response ID
    pub id: String,

    /// Generated text content
    pub content: String,

    /// Tool calls made by the model
    pub tool_calls: Vec<ToolCall>,

    /// Token usage
    pub usage: Usage,
}

/// Streaming events
pub enum StreamEvent {
    /// Text delta
    Delta(String),

    /// Tool call started
    ToolCallStart { id: String, name: String },

    /// Tool call argument delta
    ToolCallDelta { id: String, arguments: String },

    /// Stream completed
    Done,

    /// Error occurred
    Error(String),
}

Chat Messages

pub struct ChatMessage {
    /// Role: "system", "user", "assistant", "tool"
    pub role: String,

    /// Message content
    pub content: String,

    /// Optional tool call ID (for tool responses)
    pub tool_call_id: Option<String>,
}

Tool Calling

Providers must support tool calling for Codive to work effectively:

/// Tool definition sent to the model
pub struct ToolDefinition {
    pub name: String,
    pub description: String,
    pub parameters: serde_json::Value,
}

/// Tool call from the model
pub struct ToolCall {
    pub id: String,
    pub name: String,
    pub arguments: serde_json::Value,
}

Note

Not all LLMs support function/tool calling. For models without native tool support, you may need to implement a prompt-based approach where tools are described in the system prompt and parsed from the response.

Error Handling

Define clear error types:

pub enum ProviderError {
    /// Network/connection error
    Network(String),

    /// API error (rate limit, invalid request, etc.)
    Api(String),

    /// Authentication error
    Auth(String),

    /// Response parsing error
    Parse(String),

    /// Model not available
    ModelNotFound(String),

    /// Context length exceeded
    ContextTooLong { max: usize, actual: usize },
}

Configuration

Support provider-specific configuration:

#[derive(Deserialize)]
pub struct ProviderConfig {
    pub name: String,
    pub model: String,
    pub base_url: Option<String>,
    pub api_key: Option<String>,  // Prefer env vars
    pub temperature: Option<f32>,
    pub max_tokens: Option<u32>,
    pub timeout: Option<u64>,
}

Example configuration:

[provider]
name = "local_llm"
model = "local-13b"
base_url = "http://localhost:8080"
temperature = 0.7
max_tokens = 4096
timeout = 60

Testing Providers

Test your provider implementation:

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_complete() {
        let provider = LocalLlmProvider::new(
            "http://localhost:8080",
            "local-7b"
        );

        let request = CompletionRequest {
            messages: vec![
                ChatMessage {
                    role: "user".to_string(),
                    content: "Hello, world!".to_string(),
                    tool_call_id: None,
                },
            ],
            tools: vec![],
            max_tokens: Some(100),
            temperature: None,
            stop: None,
        };

        let response = provider.complete(request).await.unwrap();
        assert!(!response.content.is_empty());
    }

    #[tokio::test]
    async fn test_streaming() {
        let provider = LocalLlmProvider::new(
            "http://localhost:8080",
            "local-7b"
        );

        let request = CompletionRequest {
            messages: vec![
                ChatMessage {
                    role: "user".to_string(),
                    content: "Count to 5".to_string(),
                    tool_call_id: None,
                },
            ],
            tools: vec![],
            max_tokens: Some(100),
            temperature: None,
            stop: None,
        };

        let mut stream = provider.stream(request).await.unwrap();
        let mut output = String::new();

        while let Some(event) = stream.next().await {
            if let StreamEvent::Delta(text) = event {
                output.push_str(&text);
            }
        }

        assert!(!output.is_empty());
    }
}

Next Steps

• Add custom tools
• Set up IDE integration