diff --git a/.gitignore b/.gitignore index 43a48ce78cb..3a7b0e4da3d 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ crates/docs_preprocessor/actions.json .nixos-test-history .local* +.cache diff --git a/Cargo.lock b/Cargo.lock index fa8b39b7e6e..7db15b1c295 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,6 +5398,7 @@ dependencies = [ "lsp", "menu", "node_runtime", + "ollama", "open_ai", "parking_lot", "postage", diff --git a/crates/edit_prediction/Cargo.toml b/crates/edit_prediction/Cargo.toml index 27b67e5d570..d8a26c016c8 100644 --- a/crates/edit_prediction/Cargo.toml +++ b/crates/edit_prediction/Cargo.toml @@ -44,6 +44,7 @@ language_model.workspace = true log.workspace = true lsp.workspace = true menu.workspace = true +ollama.workspace = true open_ai.workspace = true postage.workspace = true pretty_assertions.workspace = true diff --git a/crates/edit_prediction/src/fim.rs b/crates/edit_prediction/src/fim.rs index 349fc308318..5482fc1fb2a 100644 --- a/crates/edit_prediction/src/fim.rs +++ b/crates/edit_prediction/src/fim.rs @@ -7,7 +7,7 @@ use anyhow::{Context as _, Result, anyhow}; use gpui::{App, AppContext as _, Entity, Task}; use language::{ Anchor, Buffer, BufferSnapshot, EditPredictionPromptFormat, ToOffset, ToPoint as _, - language_settings::all_language_settings, + ZetaVersion, language_settings::all_language_settings, }; use std::{path::Path, sync::Arc, time::Instant}; use zeta_prompt::{Zeta2PromptInput, compute_editable_and_context_ranges}; @@ -165,6 +165,30 @@ pub fn request_prediction( }) } +/// Infers the FIM prompt format from an Ollama/OpenAI-compatible model name. +/// Returns `None` if the model isn't a known FIM-capable model. +pub fn infer_prompt_format(model: &str) -> Option { + let model_base = model.split(':').next().unwrap_or(model); + + Some(match model_base { + "zeta2" => EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2), + "zeta2.1" => EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2_1), + model_base if model_base.to_ascii_lowercase().contains("sweep-next-edit") => { + EditPredictionPromptFormat::Sweep + } + "codellama" | "code-llama" => EditPredictionPromptFormat::CodeLlama, + "starcoder" | "starcoder2" | "starcoderbase" => EditPredictionPromptFormat::StarCoder, + "deepseek-coder" | "deepseek-coder-v2" => EditPredictionPromptFormat::DeepseekCoder, + "qwen2.5-coder" | "qwen-coder" | "qwen" => EditPredictionPromptFormat::Qwen, + "codegemma" => EditPredictionPromptFormat::CodeGemma, + "codestral" | "mistral" => EditPredictionPromptFormat::Codestral, + "glm" | "glm-4" | "glm-4.5" => EditPredictionPromptFormat::Glm, + _ => { + return None; + } + }) +} + fn format_fim_prompt( prompt_format: EditPredictionPromptFormat, prefix: &str, @@ -242,3 +266,59 @@ fn clean_fim_completion(response: &str) -> String { result } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn infer_prompt_format_matches_known_model_families() { + assert_eq!( + infer_prompt_format("qwen2.5-coder:3b"), + Some(EditPredictionPromptFormat::Qwen) + ); + assert_eq!( + infer_prompt_format("codellama:7b"), + Some(EditPredictionPromptFormat::CodeLlama) + ); + assert_eq!( + infer_prompt_format("deepseek-coder-v2:16b"), + Some(EditPredictionPromptFormat::DeepseekCoder) + ); + assert_eq!( + infer_prompt_format("starcoder2:3b"), + Some(EditPredictionPromptFormat::StarCoder) + ); + assert_eq!( + infer_prompt_format("codestral:latest"), + Some(EditPredictionPromptFormat::Codestral) + ); + assert_eq!( + infer_prompt_format("glm-4:9b"), + Some(EditPredictionPromptFormat::Glm) + ); + } + + #[test] + fn infer_prompt_format_matches_zeta_and_sweep() { + assert_eq!( + infer_prompt_format("zeta2"), + Some(EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2)) + ); + assert_eq!( + infer_prompt_format("zeta2.1"), + Some(EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2_1)) + ); + assert_eq!( + infer_prompt_format("my-sweep-next-edit-v1"), + Some(EditPredictionPromptFormat::Sweep) + ); + } + + #[test] + fn infer_prompt_format_returns_none_for_unsupported_models() { + assert_eq!(infer_prompt_format("llama3:70b"), None); + assert_eq!(infer_prompt_format("phi3:mini"), None); + assert_eq!(infer_prompt_format("nomic-embed-text"), None); + } +} diff --git a/crates/edit_prediction/src/ollama.rs b/crates/edit_prediction/src/ollama.rs index fc0f36d8321..f120e545b2e 100644 --- a/crates/edit_prediction/src/ollama.rs +++ b/crates/edit_prediction/src/ollama.rs @@ -1,11 +1,11 @@ use anyhow::{Context as _, Result}; use futures::AsyncReadExt as _; use gpui::{ - App, SharedString, TaskExt, + SharedString, http_client::{self, HttpClient}, }; use language::language_settings::OpenAiCompatibleEditPredictionSettings; -use language_model::{LanguageModelProviderId, LanguageModelRegistry}; +use ollama::get_models; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -35,32 +35,23 @@ pub(crate) struct OllamaGenerateResponse { pub response: String, } -const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("ollama"); - -pub fn is_available(cx: &App) -> bool { - LanguageModelRegistry::read_global(cx) - .provider(&PROVIDER_ID) - .is_some_and(|provider| provider.is_authenticated(cx)) -} - -pub fn ensure_authenticated(cx: &mut App) { - if let Some(provider) = LanguageModelRegistry::read_global(cx).provider(&PROVIDER_ID) { - provider.authenticate(cx).detach_and_log_err(cx); - } -} - -pub fn fetch_models(cx: &mut App) -> Vec { - let Some(provider) = LanguageModelRegistry::read_global(cx).provider(&PROVIDER_ID) else { - return Vec::new(); - }; - provider.authenticate(cx).detach_and_log_err(cx); - let mut models: Vec = provider - .provided_models(cx) - .into_iter() - .map(|model| model.id().0) - .collect(); +/// Fetches models from the Ollama server at `api_url` directly, independent +/// of the Assistant's Ollama language model provider. Only returns models +/// with a known completion prompt format, since edit predictions need +/// FIM-style models rather than chat models. +pub async fn fetch_models_from_server( + http_client: Arc, + api_url: &str, +) -> Result> { + let mut models: Vec = + get_models(http_client.as_ref(), api_url, None, &Default::default()) + .await? + .into_iter() + .filter(|model| crate::fim::infer_prompt_format(&model.name).is_some()) + .map(|model| SharedString::new(model.name)) + .collect(); models.sort(); - models + Ok(models) } pub(crate) async fn make_request( @@ -106,3 +97,37 @@ pub(crate) async fn make_request( serde_json::from_str(&body).context("Failed to parse Ollama response")?; Ok(ollama_response) } + +#[cfg(test)] +mod tests { + use super::*; + use gpui::{TestAppContext, http_client::FakeHttpClient}; + + #[gpui::test] + async fn fetch_models_from_server_filters_to_fim_capable_models(_cx: &mut TestAppContext) { + let http_client = FakeHttpClient::create(|_request| async move { + Ok(http_client::Response::builder().status(200).body( + http_client::AsyncBody::from( + r#"{"models":[ + {"name":"qwen2.5-coder:3b","modified_at":"","size":0,"digest":"","details":{"format":"gguf","family":"qwen2","families":null,"parameter_size":"3B","quantization_level":"Q4_0"}}, + {"name":"llama3:70b","modified_at":"","size":0,"digest":"","details":{"format":"gguf","family":"llama","families":null,"parameter_size":"70B","quantization_level":"Q4_0"}}, + {"name":"nomic-embed-text","modified_at":"","size":0,"digest":"","details":{"format":"gguf","family":"nomic-bert","families":null,"parameter_size":"137M","quantization_level":"F16"}}, + {"name":"codestral:latest","modified_at":"","size":0,"digest":"","details":{"format":"gguf","family":"mistral","families":null,"parameter_size":"22B","quantization_level":"Q4_0"}} + ]}"#, + ), + )?) + }); + + let models = fetch_models_from_server(http_client, "http://localhost:11434") + .await + .unwrap(); + + assert_eq!( + models, + vec![ + SharedString::from("codestral:latest"), + SharedString::from("qwen2.5-coder:3b"), + ] + ); + } +} diff --git a/crates/edit_prediction_ui/src/edit_prediction_button.rs b/crates/edit_prediction_ui/src/edit_prediction_button.rs index a2a6eec48d4..0a90a90b763 100644 --- a/crates/edit_prediction_ui/src/edit_prediction_button.rs +++ b/crates/edit_prediction_ui/src/edit_prediction_button.rs @@ -559,7 +559,6 @@ impl EditPredictionButton { cx.observe_global::(move |_, cx| cx.notify()) .detach(); - edit_prediction::ollama::ensure_authenticated(cx); let mercury_api_token_task = edit_prediction::mercury::load_mercury_api_token(cx); let open_ai_compatible_api_token_task = edit_prediction::open_ai_compatible::load_open_ai_compatible_api_token(cx); @@ -1497,7 +1496,11 @@ pub fn get_available_providers(cx: &mut App) -> Vec { providers.push(EditPredictionProvider::Codestral); } - if edit_prediction::ollama::is_available(cx) { + if all_language_settings(None, cx) + .edit_predictions + .ollama + .is_some() + { providers.push(EditPredictionProvider::Ollama); } diff --git a/crates/settings_ui/src/components/ollama_model_picker.rs b/crates/settings_ui/src/components/ollama_model_picker.rs index 8a5af331d63..6acb4a5ef91 100644 --- a/crates/settings_ui/src/components/ollama_model_picker.rs +++ b/crates/settings_ui/src/components/ollama_model_picker.rs @@ -14,47 +14,90 @@ use crate::{ type OllamaModelPicker = Picker; +const DEFAULT_OLLAMA_API_URL: &str = "http://localhost:11434"; + +fn matches_for_query(models: &[SharedString], query: &str) -> Vec { + let query_lower = query.to_lowercase(); + models + .iter() + .enumerate() + .filter(|(_, model)| query.is_empty() || model.to_lowercase().contains(&query_lower)) + .map(|(index, model)| StringMatch { + candidate_id: index, + string: model.to_string(), + positions: Vec::new(), + score: 0.0, + }) + .collect() +} + struct OllamaModelPickerDelegate { models: Vec, filtered_models: Vec, selected_index: usize, on_model_changed: Arc, + loading: bool, + _fetch_models_task: Option>, } impl OllamaModelPickerDelegate { fn new( current_model: SharedString, + api_url: SharedString, on_model_changed: impl Fn(SharedString, &mut Window, &mut App) + 'static, cx: &mut Context, ) -> Self { - let mut models = edit_prediction::ollama::fetch_models(cx); - - let current_in_list = models.contains(¤t_model); - if !current_model.is_empty() && !current_in_list { - models.insert(0, current_model.clone()); + let mut models = Vec::new(); + if !current_model.is_empty() { + models.push(current_model.clone()); } + let filtered_models = matches_for_query(&models, ""); - let selected_index = models - .iter() - .position(|model| *model == current_model) - .unwrap_or(0); - - let filtered_models = models - .iter() - .enumerate() - .map(|(index, model)| StringMatch { - candidate_id: index, - string: model.to_string(), - positions: Vec::new(), - score: 0.0, + let loading = !api_url.is_empty(); + let fetch_models_task = loading.then(|| { + let http_client = cx.http_client(); + cx.spawn(async move |this, cx| { + let result = edit_prediction::ollama::fetch_models_from_server( + http_client, + api_url.as_ref(), + ) + .await; + this.update(cx, move |picker, cx| { + picker.delegate.loading = false; + match result { + Ok(mut fetched_models) => { + if !current_model.is_empty() && !fetched_models.contains(¤t_model) + { + fetched_models.insert(0, current_model.clone()); + } + picker.delegate.models = fetched_models; + } + Err(error) => { + log::warn!("Failed to fetch Ollama models from {api_url}: {error}"); + } + } + let query = picker.query(cx); + picker.delegate.filtered_models = + matches_for_query(&picker.delegate.models, &query); + picker.delegate.selected_index = picker + .delegate + .models + .iter() + .position(|model| *model == current_model) + .unwrap_or(0); + cx.notify(); + }) + .ok(); }) - .collect(); + }); Self { models, filtered_models, - selected_index, + selected_index: 0, on_model_changed: Arc::new(on_model_changed), + loading, + _fetch_models_task: fetch_models_task, } } } @@ -88,27 +131,21 @@ impl PickerDelegate for OllamaModelPickerDelegate { "Search models…".into() } + fn no_matches_text(&self, _window: &mut Window, _cx: &mut App) -> Option { + Some(if self.loading { + "Loading models…".into() + } else { + "No models found. Check your Ollama server URL.".into() + }) + } + fn update_matches( &mut self, query: String, _window: &mut Window, cx: &mut Context, ) -> Task<()> { - let query_lower = query.to_lowercase(); - - self.filtered_models = self - .models - .iter() - .enumerate() - .filter(|(_, model)| query.is_empty() || model.to_lowercase().contains(&query_lower)) - .map(|(index, model)| StringMatch { - candidate_id: index, - string: model.to_string(), - positions: Vec::new(), - score: 0.0, - }) - .collect(); - + self.filtered_models = matches_for_query(&self.models, &query); self.selected_index = 0; cx.notify(); @@ -170,6 +207,25 @@ pub fn render_ollama_model_picker( .map(|m| m.0.clone().into()) .unwrap_or_else(|| "".into()); + let (_, api_url_value) = SettingsStore::global(cx).get_value_from_file( + file.to_settings(), + |settings: &settings::SettingsContent| { + settings + .project + .all_languages + .edit_predictions + .as_ref()? + .ollama + .as_ref()? + .api_url + .as_ref() + }, + ); + let api_url: SharedString = api_url_value + .filter(|api_url| !api_url.is_empty()) + .map(|api_url| SharedString::new(api_url.clone())) + .unwrap_or_else(|| DEFAULT_OLLAMA_API_URL.into()); + let trigger_value: SharedString = if current_value.is_empty() { "Select a model…".into() } else { @@ -188,8 +244,10 @@ pub fn render_ollama_model_picker( Some(cx.new(|cx| { let file = file.clone(); let current_value = current_value.clone(); + let api_url = api_url.clone(); let delegate = OllamaModelPickerDelegate::new( current_value, + api_url, move |model_name, window, cx| { update_settings_file( file.clone(), diff --git a/crates/zed/src/zed/edit_prediction_registry.rs b/crates/zed/src/zed/edit_prediction_registry.rs index 4bae62948fe..1e92ac4f986 100644 --- a/crates/zed/src/zed/edit_prediction_registry.rs +++ b/crates/zed/src/zed/edit_prediction_registry.rs @@ -2,14 +2,11 @@ use client::{Client, UserStore}; use codestral::{CodestralEditPredictionDelegate, load_codestral_api_key}; use collections::HashMap; use copilot::CopilotEditPredictionDelegate; -use edit_prediction::{EditPredictionModel, ZedEditPredictionDelegate}; +use edit_prediction::{EditPredictionModel, ZedEditPredictionDelegate, fim}; use editor::{EditPredictionRequestTrigger, Editor}; use gpui::{AnyWindowHandle, App, AppContext as _, Context, Entity, WeakEntity}; -use language::{ - ZetaVersion, - language_settings::{ - EditPredictionPromptFormat, EditPredictionProvider, all_language_settings, - }, +use language::language_settings::{ + EditPredictionPromptFormat, EditPredictionProvider, all_language_settings, }; use settings::SettingsStore; @@ -130,7 +127,7 @@ fn edit_prediction_provider_config_for_settings(cx: &App) -> Option Option Option { - let model_base = model.split(':').next().unwrap_or(model); - - Some(match model_base { - "zeta2" => EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2), - "zeta2.1" => EditPredictionPromptFormat::Zeta(ZetaVersion::Zeta2_1), - model_base if model_base.to_ascii_lowercase().contains("sweep-next-edit") => { - EditPredictionPromptFormat::Sweep - } - "codellama" | "code-llama" => EditPredictionPromptFormat::CodeLlama, - "starcoder" | "starcoder2" | "starcoderbase" => EditPredictionPromptFormat::StarCoder, - "deepseek-coder" | "deepseek-coder-v2" => EditPredictionPromptFormat::DeepseekCoder, - "qwen2.5-coder" | "qwen-coder" | "qwen" => EditPredictionPromptFormat::Qwen, - "codegemma" => EditPredictionPromptFormat::CodeGemma, - "codestral" | "mistral" => EditPredictionPromptFormat::Codestral, - "glm" | "glm-4" | "glm-4.5" => EditPredictionPromptFormat::Glm, - _ => { - return None; - } - }) -} - #[derive(Copy, Clone, PartialEq, Eq)] enum EditPredictionProviderConfig { Copilot, @@ -370,6 +345,52 @@ mod tests { drop(app_state); } + #[gpui::test] + async fn test_ollama_provider_routes_to_fim_model(cx: &mut TestAppContext) { + let app_state = cx.update(|cx| { + let app_state = AppState::test(cx); + client::init(&app_state.client, cx); + language_model::init(cx); + app_state + }); + + cx.update(|cx| { + cx.update_global::(|store: &mut SettingsStore, cx| { + store.update_user_settings(cx, |settings| { + settings.project.all_languages.edit_predictions = + Some(settings::EditPredictionSettingsContent { + provider: Some(EditPredictionProvider::Ollama), + ollama: Some(settings::OllamaEditPredictionSettingsContent { + api_url: Some("http://localhost:11434".to_string()), + model: Some("qwen2.5-coder:3b".to_string().into()), + prompt_format: Some(EditPredictionPromptFormatContent::Infer), + ..Default::default() + }), + ..Default::default() + }); + }); + }); + }); + + let config = cx.update(|cx| edit_prediction_provider_config_for_settings(cx)); + assert!( + matches!( + config, + Some(EditPredictionProviderConfig::Zed( + EditPredictionModel::Fim { + format: EditPredictionPromptFormat::Qwen, + } + )) + ), + "expected qwen2.5-coder model to infer the Qwen FIM prompt format" + ); + + let provider_name = config.map(|config| config.name()); + assert_eq!(provider_name, Some("FIM")); + + drop(app_state); + } + #[gpui::test] async fn test_subscribe_uses_stale_provider_config_after_settings_change( cx: &mut TestAppContext,