mirror of
https://github.com/zed-industries/zed.git
synced 2026-11-19 06:59:22 +01:00
agent: Fix compaction being billed as un-cached input tokens (#64938)
## Summary Compaction currently bills the entire context window as input tokens instead of cached input tokens, because tool definitions change during compaction requests. Compaction here is separate from native provider compaction. Closes #59528 The fix is only able to be applied to providers which have a way of disabling tools without changing the tool definitions. This notably does not include Anthropic. This PR adds a compaction strategy enum, which describes whether a provider supports disabling tools without causing a cache miss. The new variant checks whether the model supports `tool_choice: none` and includes it in the request, while keeping the tool definitions the same. A provider may support `tool_choice: none`, but still bust the cache, or have models which don't support it. So we need both the enum and `tool_choice: none` check. ## Testing I've added tests for checking request prefix equality during compaction, where this new fix is applied. ## Self-review - [x] I've reviewed my diff for quality, security, reliability, and performance. - [x] UI changes follow the [checklist](https://zed.dev/docs/development/ui-checklist). - [x] Tests cover the new or changed behavior. Release Notes: - Fixed compaction being billed as un-cached input tokens
This commit is contained in:
+306
-49
@@ -43,9 +43,10 @@ use language_model::{
|
||||
CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
|
||||
LanguageModelId, LanguageModelImage, LanguageModelProviderId, LanguageModelRegistry,
|
||||
LanguageModelRequest, LanguageModelRequestMessage, LanguageModelRequestTool,
|
||||
LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse,
|
||||
LanguageModelToolUseId, MessageContent, ProviderErrorCategory, Role, SelectedModel, Speed,
|
||||
StopReason, TokenUsage, ZED_CLOUD_PROVIDER_ID,
|
||||
LanguageModelToolChoice, LanguageModelToolResult, LanguageModelToolResultContent,
|
||||
LanguageModelToolUse, LanguageModelToolUseId, MessageContent, PromptCompactionStrategy,
|
||||
ProviderErrorCategory, Role, SelectedModel, Speed, StopReason, TokenUsage,
|
||||
ZED_CLOUD_PROVIDER_ID,
|
||||
};
|
||||
use project::{Project, trusted_worktrees::TrustedWorktrees};
|
||||
use prompt_store::ProjectContext;
|
||||
@@ -4358,51 +4359,8 @@ impl Thread {
|
||||
let model = self
|
||||
.model()
|
||||
.ok_or_else(|| anyhow!(NoModelConfiguredError))?;
|
||||
let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
|
||||
let tools = if let Some(turn) = self.running_turn.as_ref() {
|
||||
turn.tools
|
||||
.iter()
|
||||
.map(|(tool_name, tool)| {
|
||||
log::trace!("Including tool: {}", tool_name);
|
||||
let mut description = tool.description().to_string();
|
||||
let mut schema = tool.input_schema();
|
||||
// TEMPORARY (sandboxing feature flag): with the flag off,
|
||||
// the fetch and create_directory descriptions/schemas must
|
||||
// not advertise sandbox-dependent behavior (host grants,
|
||||
// out-of-project creation via the `reason` field), since
|
||||
// the corresponding runtime paths are disabled. Restore
|
||||
// the pre-sandboxing model-facing surface here rather than
|
||||
// forking the tools; delete this when the flag is removed
|
||||
// again.
|
||||
if !sandboxing_enabled {
|
||||
if tool_name.as_ref() == FetchTool::NAME {
|
||||
description =
|
||||
"Fetches a URL and returns the content as Markdown.".to_string();
|
||||
} else if tool_name.as_ref() == CreateDirectoryTool::NAME {
|
||||
description = "Creates a new directory at the specified path within \
|
||||
the project. Returns confirmation that the directory was \
|
||||
created.\n\nThis tool creates a directory and all necessary \
|
||||
parent directories. It should be used whenever you need to \
|
||||
create new directories within the project.\nThe only supported \
|
||||
path outside the project is `~/.agents/skills` or a descendant, \
|
||||
for global agent skills."
|
||||
.to_string();
|
||||
if let Some(properties) = schema
|
||||
.get_mut("properties")
|
||||
.and_then(|value| value.as_object_mut())
|
||||
{
|
||||
properties.remove("reason");
|
||||
}
|
||||
}
|
||||
}
|
||||
LanguageModelRequestTool::function(
|
||||
tool_name.to_string(),
|
||||
description,
|
||||
schema,
|
||||
tool.supports_input_streaming(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
build_request_tools(&turn.tools, cx)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
@@ -4812,13 +4770,53 @@ impl Thread {
|
||||
model: &LanguageModel,
|
||||
cx: &App,
|
||||
) -> LanguageModelRequest {
|
||||
let uses_thread_model = self.model().is_some_and(|thread_model| {
|
||||
thread_model.provider_id == model.provider_id && thread_model.id == model.id
|
||||
});
|
||||
let preserves_request_prefix = uses_thread_model
|
||||
&& model.prompt_compaction_strategy()
|
||||
== PromptCompactionStrategy::PreserveRequestPrefix;
|
||||
let enabled_tools = if preserves_request_prefix
|
||||
&& model.supports_tools()
|
||||
&& model.supports_tool_choice(LanguageModelToolChoice::None)
|
||||
{
|
||||
self.enabled_tools(cx)
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
};
|
||||
let available_tools = enabled_tools.keys().cloned().collect();
|
||||
let tools = build_request_tools(&enabled_tools, cx);
|
||||
// Keeping the tool definitions preserves the cached prefix, but
|
||||
// compaction can't execute or retain tool calls.
|
||||
let tool_choice = (!tools.is_empty()).then_some(LanguageModelToolChoice::None);
|
||||
let (thinking_allowed, thinking_effort) = if preserves_request_prefix {
|
||||
(
|
||||
self.thinking_enabled || !model.supports_disabling_thinking(),
|
||||
self.thinking_effort.clone(),
|
||||
)
|
||||
} else {
|
||||
(false, None)
|
||||
};
|
||||
|
||||
let mut request = LanguageModelRequest {
|
||||
thread_id: Some(self.id.to_string()),
|
||||
prompt_cache_key: None,
|
||||
prompt_id: Some(self.prompt_id.to_string()),
|
||||
intent: Some(CompletionIntent::ThreadContextSummarization),
|
||||
messages: self.build_request_messages_until(available_tools, insertion_ix, cx),
|
||||
tools,
|
||||
tool_choice,
|
||||
stop: Vec::new(),
|
||||
temperature: AgentSettings::temperature_for_model(&model, cx),
|
||||
messages: self.build_request_messages_until(Vec::new(), insertion_ix, cx),
|
||||
..Default::default()
|
||||
thinking_allowed,
|
||||
thinking_effort,
|
||||
speed: if preserves_request_prefix {
|
||||
self.speed()
|
||||
} else {
|
||||
None
|
||||
},
|
||||
compact_at_tokens: None,
|
||||
max_output_tokens: None,
|
||||
};
|
||||
|
||||
request.messages.push(LanguageModelRequestMessage {
|
||||
@@ -4950,6 +4948,55 @@ fn total_input_tokens(usage: language_model::TokenUsage) -> u64 {
|
||||
.saturating_add(usage.cache_read_input_tokens)
|
||||
}
|
||||
|
||||
fn build_request_tools(
|
||||
tools: &BTreeMap<SharedString, Arc<dyn AnyAgentTool>>,
|
||||
cx: &App,
|
||||
) -> Vec<LanguageModelRequestTool> {
|
||||
let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
|
||||
tools
|
||||
.iter()
|
||||
.map(|(tool_name, tool)| {
|
||||
log::trace!("Including tool: {}", tool_name);
|
||||
let mut description = tool.description().to_string();
|
||||
let mut schema = tool.input_schema();
|
||||
// TEMPORARY (sandboxing feature flag): with the flag off,
|
||||
// the fetch and create_directory descriptions/schemas must
|
||||
// not advertise sandbox-dependent behavior (host grants,
|
||||
// out-of-project creation via the `reason` field), since
|
||||
// the corresponding runtime paths are disabled. Restore
|
||||
// the pre-sandboxing model-facing surface here rather than
|
||||
// forking the tools; delete this when the flag is removed
|
||||
// again.
|
||||
if !sandboxing_enabled {
|
||||
if tool_name.as_ref() == FetchTool::NAME {
|
||||
description = "Fetches a URL and returns the content as Markdown.".to_string();
|
||||
} else if tool_name.as_ref() == CreateDirectoryTool::NAME {
|
||||
description = "Creates a new directory at the specified path within \
|
||||
the project. Returns confirmation that the directory was \
|
||||
created.\n\nThis tool creates a directory and all necessary \
|
||||
parent directories. It should be used whenever you need to \
|
||||
create new directories within the project.\nThe only supported \
|
||||
path outside the project is `~/.agents/skills` or a descendant, \
|
||||
for global agent skills."
|
||||
.to_string();
|
||||
if let Some(properties) = schema
|
||||
.get_mut("properties")
|
||||
.and_then(|value| value.as_object_mut())
|
||||
{
|
||||
properties.remove("reason");
|
||||
}
|
||||
}
|
||||
}
|
||||
LanguageModelRequestTool::function(
|
||||
tool_name.to_string(),
|
||||
description,
|
||||
schema,
|
||||
tool.supports_input_streaming(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Reserves output without subtracting it from an independent input ceiling.
|
||||
fn compaction_input_capacity(
|
||||
input_limit: u64,
|
||||
@@ -8579,6 +8626,216 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn enable_replay_image_tool(thread: &Entity<Thread>, cx: &mut App) {
|
||||
let profile_id = thread.read(cx).profile().clone();
|
||||
let mut settings = AgentSettings::get_global(cx).clone();
|
||||
settings
|
||||
.profiles
|
||||
.get_mut(&profile_id)
|
||||
.expect("thread profile should exist")
|
||||
.tools
|
||||
.insert(ReplayImageTool::NAME.into(), true);
|
||||
AgentSettings::override_global(settings, cx);
|
||||
thread.update(cx, |thread, _cx| thread.add_tool(ReplayImageTool));
|
||||
}
|
||||
|
||||
fn supports_tools_and_tool_choice(model: &mut LanguageModel) {
|
||||
model.supports_tools = true;
|
||||
model.tool_choice_support = language_model::LanguageModelToolChoiceSupport::ALL;
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_prompt_compaction_uses_default_strategy_without_opt_in(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
|
||||
let model = fake.update_model("fake", |model| {
|
||||
supports_tools_and_tool_choice(model);
|
||||
model.supports_thinking = true;
|
||||
});
|
||||
|
||||
let request = cx.update(|cx| {
|
||||
enable_replay_image_tool(&thread, cx);
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(model.clone(), cx);
|
||||
thread.set_thinking_enabled(true, cx);
|
||||
thread.set_thinking_effort(Some("high".to_string()), cx);
|
||||
thread.set_speed(Speed::Fast, cx);
|
||||
assert_eq!(thread.enabled_tools(cx).len(), 1);
|
||||
thread
|
||||
.messages
|
||||
.push(user_text_message(ClientUserMessageId::new(), "old user"));
|
||||
thread.build_compaction_request(thread.messages.len(), &model, cx)
|
||||
})
|
||||
});
|
||||
|
||||
assert!(request.tools.is_empty());
|
||||
assert_eq!(request.tool_choice, None);
|
||||
assert!(!request.thinking_allowed);
|
||||
assert_eq!(request.thinking_effort, None);
|
||||
assert_eq!(request.speed, None);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_prompt_compaction_reuses_request_prefix_when_enabled(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
|
||||
let model = fake.update_model("fake", |model| {
|
||||
supports_tools_and_tool_choice(model);
|
||||
model.supports_thinking = true;
|
||||
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
|
||||
});
|
||||
|
||||
cx.update(|cx| {
|
||||
enable_replay_image_tool(&thread, cx);
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(model.clone(), cx);
|
||||
thread.set_thinking_enabled(true, cx);
|
||||
thread.set_thinking_effort(Some("high".to_string()), cx);
|
||||
thread.set_speed(Speed::Fast, cx);
|
||||
});
|
||||
});
|
||||
|
||||
let _events = cx
|
||||
.update(|cx| {
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.send(ClientUserMessageId::new(), vec!["context to compact"], cx)
|
||||
})
|
||||
})
|
||||
.unwrap();
|
||||
cx.run_until_parked();
|
||||
|
||||
let completion_request = fake.pending_completions().pop().unwrap();
|
||||
assert_eq!(completion_request.tools.len(), 1);
|
||||
fake.send_text(&model, &completion_request, "assistant response");
|
||||
fake.end_stream(&model, &completion_request);
|
||||
cx.run_until_parked();
|
||||
|
||||
let _events = cx
|
||||
.update(|cx| {
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.compact(ClientUserMessageId::new(), cx)
|
||||
})
|
||||
})
|
||||
.unwrap();
|
||||
cx.run_until_parked();
|
||||
|
||||
let compaction_request = fake.pending_completions().pop().unwrap();
|
||||
assert_eq!(
|
||||
compaction_request.intent,
|
||||
Some(CompletionIntent::ThreadContextSummarization)
|
||||
);
|
||||
assert_eq!(compaction_request.tools, completion_request.tools);
|
||||
assert_eq!(
|
||||
compaction_request.tool_choice,
|
||||
Some(LanguageModelToolChoice::None)
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_request.thinking_allowed,
|
||||
completion_request.thinking_allowed
|
||||
);
|
||||
assert_eq!(
|
||||
compaction_request.thinking_effort,
|
||||
completion_request.thinking_effort
|
||||
);
|
||||
assert_eq!(completion_request.speed, Some(Speed::Fast));
|
||||
assert_eq!(compaction_request.speed, completion_request.speed);
|
||||
// The compaction request extends the previous request with the
|
||||
// assistant response and the compaction prompt.
|
||||
assert_eq!(
|
||||
compaction_request.messages.len(),
|
||||
completion_request.messages.len() + 2
|
||||
);
|
||||
for (compaction_message, completion_message) in compaction_request
|
||||
.messages
|
||||
.iter()
|
||||
.zip(&completion_request.messages)
|
||||
{
|
||||
assert_eq!(compaction_message.role, completion_message.role);
|
||||
assert_eq!(compaction_message.content, completion_message.content);
|
||||
assert_eq!(
|
||||
compaction_message.reasoning_details,
|
||||
completion_message.reasoning_details
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_prompt_compaction_omits_tool_choice_without_tools(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
|
||||
let model = fake.update_model("fake", |model| {
|
||||
supports_tools_and_tool_choice(model);
|
||||
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
|
||||
});
|
||||
|
||||
let request = cx.update(|cx| {
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(model.clone(), cx);
|
||||
thread
|
||||
.messages
|
||||
.push(user_text_message(ClientUserMessageId::new(), "old user"));
|
||||
thread.build_compaction_request(thread.messages.len(), &model, cx)
|
||||
})
|
||||
});
|
||||
|
||||
assert!(request.tools.is_empty());
|
||||
assert_eq!(request.tool_choice, None);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_prompt_compaction_omits_tools_without_tool_choice_support(
|
||||
cx: &mut TestAppContext,
|
||||
) {
|
||||
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
|
||||
let model = fake.update_model("fake", |model| {
|
||||
model.supports_tools = true;
|
||||
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
|
||||
});
|
||||
|
||||
let request = cx.update(|cx| {
|
||||
enable_replay_image_tool(&thread, cx);
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(model.clone(), cx);
|
||||
assert_eq!(thread.enabled_tools(cx).len(), 1);
|
||||
thread
|
||||
.messages
|
||||
.push(user_text_message(ClientUserMessageId::new(), "old user"));
|
||||
thread.build_compaction_request(thread.messages.len(), &model, cx)
|
||||
})
|
||||
});
|
||||
|
||||
assert!(request.tools.is_empty());
|
||||
assert_eq!(request.tool_choice, None);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn test_prompt_compaction_rebuilds_prompt_for_different_model(cx: &mut TestAppContext) {
|
||||
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
|
||||
let thread_model = fake.update_model("fake", |model| model.supports_thinking = true);
|
||||
let compaction_model = fake.update_model("compaction", |model| {
|
||||
supports_tools_and_tool_choice(model);
|
||||
model.supports_thinking = true;
|
||||
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
|
||||
});
|
||||
|
||||
let request = cx.update(|cx| {
|
||||
enable_replay_image_tool(&thread, cx);
|
||||
thread.update(cx, |thread, cx| {
|
||||
thread.set_model(thread_model.clone(), cx);
|
||||
thread.set_thinking_enabled(true, cx);
|
||||
thread.set_thinking_effort(Some("high".to_string()), cx);
|
||||
thread.set_speed(Speed::Fast, cx);
|
||||
thread
|
||||
.messages
|
||||
.push(user_text_message(ClientUserMessageId::new(), "old user"));
|
||||
thread.build_compaction_request(thread.messages.len(), &compaction_model, cx)
|
||||
})
|
||||
});
|
||||
|
||||
assert!(request.tools.is_empty());
|
||||
assert_eq!(request.tool_choice, None);
|
||||
assert!(!request.thinking_allowed);
|
||||
assert_eq!(request.thinking_effort, None);
|
||||
assert_eq!(request.speed, None);
|
||||
}
|
||||
|
||||
/// When `agent.compaction_model` is configured, manual `/compact` streams
|
||||
/// to the configured model rather than the thread's primary model.
|
||||
#[gpui::test]
|
||||
|
||||
@@ -139,6 +139,9 @@ pub struct LanguageModel {
|
||||
pub supports_explicit_compaction_output_limit: bool,
|
||||
/// The provider-enforced input size required for explicit compaction.
|
||||
pub minimum_explicit_compaction_input_tokens: Option<u64>,
|
||||
/// How Zed builds prompt-based compaction requests. This does not affect
|
||||
/// provider-native compaction.
|
||||
pub prompt_compaction_strategy: PromptCompactionStrategy,
|
||||
pub supports_images: bool,
|
||||
pub supports_tools: bool,
|
||||
pub tool_choice_support: LanguageModelToolChoiceSupport,
|
||||
@@ -155,6 +158,16 @@ pub struct LanguageModel {
|
||||
pub max_output_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub enum PromptCompactionStrategy {
|
||||
#[default]
|
||||
RebuildPrompt,
|
||||
/// Preserve tools, system instructions, conversation history, and reasoning
|
||||
/// settings while disabling tool calls. Providers must only opt in when
|
||||
/// `tool_choice: none` is enforced without invalidating that cached prefix.
|
||||
PreserveRequestPrefix,
|
||||
}
|
||||
|
||||
/// Which [`LanguageModelToolChoice`] values a model accepts.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct LanguageModelToolChoiceSupport {
|
||||
@@ -212,6 +225,7 @@ impl LanguageModel {
|
||||
supports_explicit_compaction: false,
|
||||
supports_explicit_compaction_output_limit: false,
|
||||
minimum_explicit_compaction_input_tokens: None,
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::default(),
|
||||
supports_images: false,
|
||||
supports_tools: false,
|
||||
tool_choice_support: LanguageModelToolChoiceSupport::default(),
|
||||
@@ -333,6 +347,10 @@ impl LanguageModel {
|
||||
self.minimum_explicit_compaction_input_tokens
|
||||
}
|
||||
|
||||
pub fn prompt_compaction_strategy(&self) -> PromptCompactionStrategy {
|
||||
self.prompt_compaction_strategy
|
||||
}
|
||||
|
||||
pub fn supports_images(&self) -> bool {
|
||||
self.supports_images
|
||||
}
|
||||
|
||||
@@ -14,8 +14,8 @@ use language_model::{
|
||||
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
|
||||
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
|
||||
LanguageModelToolChoiceSupport, LanguageModelToolResultContent, LanguageModelToolUse,
|
||||
MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role, StopReason,
|
||||
TokenUsage, env_var, unavailable_error,
|
||||
MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
|
||||
Role, StopReason, TokenUsage, env_var, unavailable_error,
|
||||
};
|
||||
pub use settings::DeepseekAvailableModel as AvailableModel;
|
||||
use settings::{Settings, SettingsStore};
|
||||
@@ -330,6 +330,7 @@ fn language_model(model: &deepseek::Model) -> LanguageModel {
|
||||
Arc::default()
|
||||
},
|
||||
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
supports_images: model.supports_images(),
|
||||
max_output_tokens: model.max_output_tokens(),
|
||||
..LanguageModel::new(
|
||||
|
||||
@@ -11,8 +11,8 @@ use language_model::{
|
||||
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
|
||||
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
|
||||
LanguageModelToolChoice, LanguageModelToolChoiceSupport, LanguageModelToolResultContent,
|
||||
MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role,
|
||||
SubPageProviderSettings, env_var, unavailable_error,
|
||||
MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
|
||||
Role, SubPageProviderSettings, env_var, unavailable_error,
|
||||
};
|
||||
use llama_cpp::{
|
||||
LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,
|
||||
@@ -570,6 +570,7 @@ impl LlamaCppLanguageModelProvider {
|
||||
None => LanguageModelName::from(display_name.to_string()),
|
||||
};
|
||||
LanguageModel {
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
supports_tools: capabilities.supports_tools,
|
||||
tool_choice_support: LanguageModelToolChoiceSupport {
|
||||
auto: capabilities.supports_tools,
|
||||
|
||||
@@ -12,7 +12,8 @@ use language_model::{
|
||||
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
|
||||
LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
|
||||
LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ModelRateLimiters,
|
||||
ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var, unavailable_error,
|
||||
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage,
|
||||
env_var, unavailable_error,
|
||||
};
|
||||
pub use mistral::{MISTRAL_API_URL, StreamResponse};
|
||||
pub use settings::MistralAvailableModel as AvailableModel;
|
||||
@@ -327,6 +328,7 @@ fn language_model(model: &mistral::Model) -> LanguageModel {
|
||||
any: supports_tools,
|
||||
none: supports_tools,
|
||||
},
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
supports_images: model.supports_images(),
|
||||
supports_thinking: model.supports_thinking(),
|
||||
supports_disabling_thinking: model.supports_disabling_thinking(),
|
||||
|
||||
@@ -11,7 +11,8 @@ use language_model::{
|
||||
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
|
||||
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
|
||||
LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID, OPEN_AI_PROVIDER_NAME,
|
||||
ProviderSettingsView, RateLimiter, env_var, stream_in_background, unavailable_error,
|
||||
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, stream_in_background,
|
||||
unavailable_error,
|
||||
};
|
||||
use open_ai::{
|
||||
ResponseStreamEvent,
|
||||
@@ -707,6 +708,7 @@ fn language_model(model: &open_ai::Model) -> LanguageModel {
|
||||
supports_tools: true,
|
||||
supports_images,
|
||||
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
supports_streaming_tools: true,
|
||||
supports_thinking: supports_selectable_thinking_effort(model),
|
||||
supports_fast_mode: model.supports_priority(),
|
||||
|
||||
@@ -9,8 +9,8 @@ use language_model::{
|
||||
LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
|
||||
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
|
||||
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
|
||||
ModelRateLimiters, ProviderSettingsView, RateLimiter, SubPageProviderSettings,
|
||||
unavailable_error,
|
||||
ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
|
||||
SubPageProviderSettings, unavailable_error,
|
||||
};
|
||||
use open_ai::{
|
||||
ResponseStreamEvent,
|
||||
@@ -95,6 +95,7 @@ impl OpenAiCompatibleLanguageModelProvider {
|
||||
any: model.capabilities.tools,
|
||||
none: true,
|
||||
},
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::RebuildPrompt,
|
||||
supports_streaming_tools: true,
|
||||
supports_thinking: default_thinking_reasoning_effort(model).is_some(),
|
||||
supported_effort_levels: supported_thinking_effort_levels(model).into(),
|
||||
|
||||
@@ -10,8 +10,8 @@ use language_model::{
|
||||
LanguageModelClient, LanguageModelCompletionError, LanguageModelCompletionStream,
|
||||
LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProvider,
|
||||
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
|
||||
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView,
|
||||
RateLimiter, env_var, unavailable_error,
|
||||
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters,
|
||||
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, unavailable_error,
|
||||
};
|
||||
pub use settings::XaiAvailableModel as AvailableModel;
|
||||
use settings::{Settings, SettingsStore};
|
||||
@@ -395,6 +395,7 @@ fn language_model(model: &x_ai::Model) -> LanguageModel {
|
||||
supports_images: model.supports_images(),
|
||||
supports_streaming_tools: true,
|
||||
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
|
||||
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
supports_thinking: model.supports_reasoning_effort(),
|
||||
supported_effort_levels: supported_thinking_effort_levels(model).into(),
|
||||
max_output_tokens: model.max_output_tokens(),
|
||||
|
||||
@@ -23,8 +23,9 @@ use language_model::{
|
||||
LanguageModelCompletionEvent, LanguageModelCompletionStream, LanguageModelEffortLevel,
|
||||
LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
|
||||
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID,
|
||||
OPEN_AI_PROVIDER_NAME, ProviderErrorCategory, RateLimiter, X_AI_PROVIDER_ID,
|
||||
X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME, unavailable_error,
|
||||
OPEN_AI_PROVIDER_NAME, PromptCompactionStrategy, ProviderErrorCategory, RateLimiter,
|
||||
X_AI_PROVIDER_ID, X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME,
|
||||
unavailable_error,
|
||||
};
|
||||
|
||||
use schemars::JsonSchema;
|
||||
@@ -546,6 +547,11 @@ pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel
|
||||
none: true,
|
||||
any: model.provider != Anthropic || anthropic::supports_forced_tool_use(&model.id.0),
|
||||
},
|
||||
// Anthropic invalidates cached messages when `tool_choice` changes.
|
||||
prompt_compaction_strategy: match model.provider {
|
||||
OpenAi | XAi => PromptCompactionStrategy::PreserveRequestPrefix,
|
||||
Anthropic | Google => PromptCompactionStrategy::RebuildPrompt,
|
||||
},
|
||||
supports_split_token_display: matches!(model.provider, OpenAi | XAi),
|
||||
max_output_tokens: Some(model.max_output_tokens as u64),
|
||||
..LanguageModel::new(
|
||||
@@ -2140,6 +2146,18 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_compaction_preserves_prefix_only_for_supported_upstreams() {
|
||||
assert_eq!(
|
||||
language_model(&cloud_test_config()).prompt_compaction_strategy,
|
||||
PromptCompactionStrategy::PreserveRequestPrefix
|
||||
);
|
||||
assert_eq!(
|
||||
language_model(&cloud_anthropic_test_config()).prompt_compaction_strategy,
|
||||
PromptCompactionStrategy::RebuildPrompt
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_resolves_current_model_list_and_rejects_unlisted_models() {
|
||||
let listed = language_model(&cloud_test_config());
|
||||
|
||||
Reference in New Issue
Block a user