agent: Fix compaction being billed as un-cached input tokens (#64938)

## Summary

Compaction currently bills the entire context window as input tokens
instead of cached input tokens, because tool definitions change during
compaction requests. Compaction here is separate from native provider
compaction.

Closes #59528

The fix is only able to be applied to providers which have a way of
disabling tools without changing the tool definitions. This notably does
not include Anthropic.

This PR adds a compaction strategy enum, which describes whether a
provider supports disabling tools without causing a cache miss. The new
variant checks whether the model supports `tool_choice: none` and
includes it in the request, while keeping the tool definitions the same.

A provider may support `tool_choice: none`, but still bust the cache, or
have models which don't support it. So we need both the enum and
`tool_choice: none` check.

## Testing

I've added tests for checking request prefix equality during compaction,
where this new fix is applied.

## Self-review

- [x] I've reviewed my diff for quality, security, reliability, and
performance.
- [x] UI changes follow the
[checklist](https://zed.dev/docs/development/ui-checklist).
- [x] Tests cover the new or changed behavior.

Release Notes:

- Fixed compaction being billed as un-cached input tokens
This commit is contained in:
tidely
2026-11-04 13:36:23 +00:00
committed by GitHub
parent ad54befbc6
commit ead5672e16
9 changed files with 362 additions and 61 deletions
+306 -49
View File
@@ -43,9 +43,10 @@ use language_model::{
CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelId, LanguageModelImage, LanguageModelProviderId, LanguageModelRegistry,
LanguageModelRequest, LanguageModelRequestMessage, LanguageModelRequestTool,
LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse,
LanguageModelToolUseId, MessageContent, ProviderErrorCategory, Role, SelectedModel, Speed,
StopReason, TokenUsage, ZED_CLOUD_PROVIDER_ID,
LanguageModelToolChoice, LanguageModelToolResult, LanguageModelToolResultContent,
LanguageModelToolUse, LanguageModelToolUseId, MessageContent, PromptCompactionStrategy,
ProviderErrorCategory, Role, SelectedModel, Speed, StopReason, TokenUsage,
ZED_CLOUD_PROVIDER_ID,
};
use project::{Project, trusted_worktrees::TrustedWorktrees};
use prompt_store::ProjectContext;
@@ -4358,51 +4359,8 @@ impl Thread {
let model = self
.model()
.ok_or_else(|| anyhow!(NoModelConfiguredError))?;
let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
let tools = if let Some(turn) = self.running_turn.as_ref() {
turn.tools
.iter()
.map(|(tool_name, tool)| {
log::trace!("Including tool: {}", tool_name);
let mut description = tool.description().to_string();
let mut schema = tool.input_schema();
// TEMPORARY (sandboxing feature flag): with the flag off,
// the fetch and create_directory descriptions/schemas must
// not advertise sandbox-dependent behavior (host grants,
// out-of-project creation via the `reason` field), since
// the corresponding runtime paths are disabled. Restore
// the pre-sandboxing model-facing surface here rather than
// forking the tools; delete this when the flag is removed
// again.
if !sandboxing_enabled {
if tool_name.as_ref() == FetchTool::NAME {
description =
"Fetches a URL and returns the content as Markdown.".to_string();
} else if tool_name.as_ref() == CreateDirectoryTool::NAME {
description = "Creates a new directory at the specified path within \
the project. Returns confirmation that the directory was \
created.\n\nThis tool creates a directory and all necessary \
parent directories. It should be used whenever you need to \
create new directories within the project.\nThe only supported \
path outside the project is `~/.agents/skills` or a descendant, \
for global agent skills."
.to_string();
if let Some(properties) = schema
.get_mut("properties")
.and_then(|value| value.as_object_mut())
{
properties.remove("reason");
}
}
}
LanguageModelRequestTool::function(
tool_name.to_string(),
description,
schema,
tool.supports_input_streaming(),
)
})
.collect::<Vec<_>>()
build_request_tools(&turn.tools, cx)
} else {
Vec::new()
};
@@ -4812,13 +4770,53 @@ impl Thread {
model: &LanguageModel,
cx: &App,
) -> LanguageModelRequest {
let uses_thread_model = self.model().is_some_and(|thread_model| {
thread_model.provider_id == model.provider_id && thread_model.id == model.id
});
let preserves_request_prefix = uses_thread_model
&& model.prompt_compaction_strategy()
== PromptCompactionStrategy::PreserveRequestPrefix;
let enabled_tools = if preserves_request_prefix
&& model.supports_tools()
&& model.supports_tool_choice(LanguageModelToolChoice::None)
{
self.enabled_tools(cx)
} else {
BTreeMap::new()
};
let available_tools = enabled_tools.keys().cloned().collect();
let tools = build_request_tools(&enabled_tools, cx);
// Keeping the tool definitions preserves the cached prefix, but
// compaction can't execute or retain tool calls.
let tool_choice = (!tools.is_empty()).then_some(LanguageModelToolChoice::None);
let (thinking_allowed, thinking_effort) = if preserves_request_prefix {
(
self.thinking_enabled || !model.supports_disabling_thinking(),
self.thinking_effort.clone(),
)
} else {
(false, None)
};
let mut request = LanguageModelRequest {
thread_id: Some(self.id.to_string()),
prompt_cache_key: None,
prompt_id: Some(self.prompt_id.to_string()),
intent: Some(CompletionIntent::ThreadContextSummarization),
messages: self.build_request_messages_until(available_tools, insertion_ix, cx),
tools,
tool_choice,
stop: Vec::new(),
temperature: AgentSettings::temperature_for_model(&model, cx),
messages: self.build_request_messages_until(Vec::new(), insertion_ix, cx),
..Default::default()
thinking_allowed,
thinking_effort,
speed: if preserves_request_prefix {
self.speed()
} else {
None
},
compact_at_tokens: None,
max_output_tokens: None,
};
request.messages.push(LanguageModelRequestMessage {
@@ -4950,6 +4948,55 @@ fn total_input_tokens(usage: language_model::TokenUsage) -> u64 {
.saturating_add(usage.cache_read_input_tokens)
}
fn build_request_tools(
tools: &BTreeMap<SharedString, Arc<dyn AnyAgentTool>>,
cx: &App,
) -> Vec<LanguageModelRequestTool> {
let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
tools
.iter()
.map(|(tool_name, tool)| {
log::trace!("Including tool: {}", tool_name);
let mut description = tool.description().to_string();
let mut schema = tool.input_schema();
// TEMPORARY (sandboxing feature flag): with the flag off,
// the fetch and create_directory descriptions/schemas must
// not advertise sandbox-dependent behavior (host grants,
// out-of-project creation via the `reason` field), since
// the corresponding runtime paths are disabled. Restore
// the pre-sandboxing model-facing surface here rather than
// forking the tools; delete this when the flag is removed
// again.
if !sandboxing_enabled {
if tool_name.as_ref() == FetchTool::NAME {
description = "Fetches a URL and returns the content as Markdown.".to_string();
} else if tool_name.as_ref() == CreateDirectoryTool::NAME {
description = "Creates a new directory at the specified path within \
the project. Returns confirmation that the directory was \
created.\n\nThis tool creates a directory and all necessary \
parent directories. It should be used whenever you need to \
create new directories within the project.\nThe only supported \
path outside the project is `~/.agents/skills` or a descendant, \
for global agent skills."
.to_string();
if let Some(properties) = schema
.get_mut("properties")
.and_then(|value| value.as_object_mut())
{
properties.remove("reason");
}
}
}
LanguageModelRequestTool::function(
tool_name.to_string(),
description,
schema,
tool.supports_input_streaming(),
)
})
.collect()
}
/// Reserves output without subtracting it from an independent input ceiling.
fn compaction_input_capacity(
input_limit: u64,
@@ -8579,6 +8626,216 @@ mod tests {
);
}
fn enable_replay_image_tool(thread: &Entity<Thread>, cx: &mut App) {
let profile_id = thread.read(cx).profile().clone();
let mut settings = AgentSettings::get_global(cx).clone();
settings
.profiles
.get_mut(&profile_id)
.expect("thread profile should exist")
.tools
.insert(ReplayImageTool::NAME.into(), true);
AgentSettings::override_global(settings, cx);
thread.update(cx, |thread, _cx| thread.add_tool(ReplayImageTool));
}
fn supports_tools_and_tool_choice(model: &mut LanguageModel) {
model.supports_tools = true;
model.tool_choice_support = language_model::LanguageModelToolChoiceSupport::ALL;
}
#[gpui::test]
async fn test_prompt_compaction_uses_default_strategy_without_opt_in(cx: &mut TestAppContext) {
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let model = fake.update_model("fake", |model| {
supports_tools_and_tool_choice(model);
model.supports_thinking = true;
});
let request = cx.update(|cx| {
enable_replay_image_tool(&thread, cx);
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
thread.set_thinking_enabled(true, cx);
thread.set_thinking_effort(Some("high".to_string()), cx);
thread.set_speed(Speed::Fast, cx);
assert_eq!(thread.enabled_tools(cx).len(), 1);
thread
.messages
.push(user_text_message(ClientUserMessageId::new(), "old user"));
thread.build_compaction_request(thread.messages.len(), &model, cx)
})
});
assert!(request.tools.is_empty());
assert_eq!(request.tool_choice, None);
assert!(!request.thinking_allowed);
assert_eq!(request.thinking_effort, None);
assert_eq!(request.speed, None);
}
#[gpui::test]
async fn test_prompt_compaction_reuses_request_prefix_when_enabled(cx: &mut TestAppContext) {
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let model = fake.update_model("fake", |model| {
supports_tools_and_tool_choice(model);
model.supports_thinking = true;
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
});
cx.update(|cx| {
enable_replay_image_tool(&thread, cx);
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
thread.set_thinking_enabled(true, cx);
thread.set_thinking_effort(Some("high".to_string()), cx);
thread.set_speed(Speed::Fast, cx);
});
});
let _events = cx
.update(|cx| {
thread.update(cx, |thread, cx| {
thread.send(ClientUserMessageId::new(), vec!["context to compact"], cx)
})
})
.unwrap();
cx.run_until_parked();
let completion_request = fake.pending_completions().pop().unwrap();
assert_eq!(completion_request.tools.len(), 1);
fake.send_text(&model, &completion_request, "assistant response");
fake.end_stream(&model, &completion_request);
cx.run_until_parked();
let _events = cx
.update(|cx| {
thread.update(cx, |thread, cx| {
thread.compact(ClientUserMessageId::new(), cx)
})
})
.unwrap();
cx.run_until_parked();
let compaction_request = fake.pending_completions().pop().unwrap();
assert_eq!(
compaction_request.intent,
Some(CompletionIntent::ThreadContextSummarization)
);
assert_eq!(compaction_request.tools, completion_request.tools);
assert_eq!(
compaction_request.tool_choice,
Some(LanguageModelToolChoice::None)
);
assert_eq!(
compaction_request.thinking_allowed,
completion_request.thinking_allowed
);
assert_eq!(
compaction_request.thinking_effort,
completion_request.thinking_effort
);
assert_eq!(completion_request.speed, Some(Speed::Fast));
assert_eq!(compaction_request.speed, completion_request.speed);
// The compaction request extends the previous request with the
// assistant response and the compaction prompt.
assert_eq!(
compaction_request.messages.len(),
completion_request.messages.len() + 2
);
for (compaction_message, completion_message) in compaction_request
.messages
.iter()
.zip(&completion_request.messages)
{
assert_eq!(compaction_message.role, completion_message.role);
assert_eq!(compaction_message.content, completion_message.content);
assert_eq!(
compaction_message.reasoning_details,
completion_message.reasoning_details
);
}
}
#[gpui::test]
async fn test_prompt_compaction_omits_tool_choice_without_tools(cx: &mut TestAppContext) {
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let model = fake.update_model("fake", |model| {
supports_tools_and_tool_choice(model);
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
});
let request = cx.update(|cx| {
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
thread
.messages
.push(user_text_message(ClientUserMessageId::new(), "old user"));
thread.build_compaction_request(thread.messages.len(), &model, cx)
})
});
assert!(request.tools.is_empty());
assert_eq!(request.tool_choice, None);
}
#[gpui::test]
async fn test_prompt_compaction_omits_tools_without_tool_choice_support(
cx: &mut TestAppContext,
) {
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let model = fake.update_model("fake", |model| {
model.supports_tools = true;
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
});
let request = cx.update(|cx| {
enable_replay_image_tool(&thread, cx);
thread.update(cx, |thread, cx| {
thread.set_model(model.clone(), cx);
assert_eq!(thread.enabled_tools(cx).len(), 1);
thread
.messages
.push(user_text_message(ClientUserMessageId::new(), "old user"));
thread.build_compaction_request(thread.messages.len(), &model, cx)
})
});
assert!(request.tools.is_empty());
assert_eq!(request.tool_choice, None);
}
#[gpui::test]
async fn test_prompt_compaction_rebuilds_prompt_for_different_model(cx: &mut TestAppContext) {
let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
let thread_model = fake.update_model("fake", |model| model.supports_thinking = true);
let compaction_model = fake.update_model("compaction", |model| {
supports_tools_and_tool_choice(model);
model.supports_thinking = true;
model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
});
let request = cx.update(|cx| {
enable_replay_image_tool(&thread, cx);
thread.update(cx, |thread, cx| {
thread.set_model(thread_model.clone(), cx);
thread.set_thinking_enabled(true, cx);
thread.set_thinking_effort(Some("high".to_string()), cx);
thread.set_speed(Speed::Fast, cx);
thread
.messages
.push(user_text_message(ClientUserMessageId::new(), "old user"));
thread.build_compaction_request(thread.messages.len(), &compaction_model, cx)
})
});
assert!(request.tools.is_empty());
assert_eq!(request.tool_choice, None);
assert!(!request.thinking_allowed);
assert_eq!(request.thinking_effort, None);
assert_eq!(request.speed, None);
}
/// When `agent.compaction_model` is configured, manual `/compact` streams
/// to the configured model rather than the thread's primary model.
#[gpui::test]
@@ -139,6 +139,9 @@ pub struct LanguageModel {
pub supports_explicit_compaction_output_limit: bool,
/// The provider-enforced input size required for explicit compaction.
pub minimum_explicit_compaction_input_tokens: Option<u64>,
/// How Zed builds prompt-based compaction requests. This does not affect
/// provider-native compaction.
pub prompt_compaction_strategy: PromptCompactionStrategy,
pub supports_images: bool,
pub supports_tools: bool,
pub tool_choice_support: LanguageModelToolChoiceSupport,
@@ -155,6 +158,16 @@ pub struct LanguageModel {
pub max_output_tokens: Option<u64>,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum PromptCompactionStrategy {
#[default]
RebuildPrompt,
/// Preserve tools, system instructions, conversation history, and reasoning
/// settings while disabling tool calls. Providers must only opt in when
/// `tool_choice: none` is enforced without invalidating that cached prefix.
PreserveRequestPrefix,
}
/// Which [`LanguageModelToolChoice`] values a model accepts.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct LanguageModelToolChoiceSupport {
@@ -212,6 +225,7 @@ impl LanguageModel {
supports_explicit_compaction: false,
supports_explicit_compaction_output_limit: false,
minimum_explicit_compaction_input_tokens: None,
prompt_compaction_strategy: PromptCompactionStrategy::default(),
supports_images: false,
supports_tools: false,
tool_choice_support: LanguageModelToolChoiceSupport::default(),
@@ -333,6 +347,10 @@ impl LanguageModel {
self.minimum_explicit_compaction_input_tokens
}
pub fn prompt_compaction_strategy(&self) -> PromptCompactionStrategy {
self.prompt_compaction_strategy
}
pub fn supports_images(&self) -> bool {
self.supports_images
}
@@ -14,8 +14,8 @@ use language_model::{
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
LanguageModelToolChoiceSupport, LanguageModelToolResultContent, LanguageModelToolUse,
MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role, StopReason,
TokenUsage, env_var, unavailable_error,
MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
Role, StopReason, TokenUsage, env_var, unavailable_error,
};
pub use settings::DeepseekAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -330,6 +330,7 @@ fn language_model(model: &deepseek::Model) -> LanguageModel {
Arc::default()
},
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_images: model.supports_images(),
max_output_tokens: model.max_output_tokens(),
..LanguageModel::new(
@@ -11,8 +11,8 @@ use language_model::{
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
LanguageModelToolChoice, LanguageModelToolChoiceSupport, LanguageModelToolResultContent,
MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role,
SubPageProviderSettings, env_var, unavailable_error,
MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
Role, SubPageProviderSettings, env_var, unavailable_error,
};
use llama_cpp::{
LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,
@@ -570,6 +570,7 @@ impl LlamaCppLanguageModelProvider {
None => LanguageModelName::from(display_name.to_string()),
};
LanguageModel {
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_tools: capabilities.supports_tools,
tool_choice_support: LanguageModelToolChoiceSupport {
auto: capabilities.supports_tools,
@@ -12,7 +12,8 @@ use language_model::{
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ModelRateLimiters,
ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var, unavailable_error,
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage,
env_var, unavailable_error,
};
pub use mistral::{MISTRAL_API_URL, StreamResponse};
pub use settings::MistralAvailableModel as AvailableModel;
@@ -327,6 +328,7 @@ fn language_model(model: &mistral::Model) -> LanguageModel {
any: supports_tools,
none: supports_tools,
},
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_images: model.supports_images(),
supports_thinking: model.supports_thinking(),
supports_disabling_thinking: model.supports_disabling_thinking(),
@@ -11,7 +11,8 @@ use language_model::{
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID, OPEN_AI_PROVIDER_NAME,
ProviderSettingsView, RateLimiter, env_var, stream_in_background, unavailable_error,
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, stream_in_background,
unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -707,6 +708,7 @@ fn language_model(model: &open_ai::Model) -> LanguageModel {
supports_tools: true,
supports_images,
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_streaming_tools: true,
supports_thinking: supports_selectable_thinking_effort(model),
supports_fast_mode: model.supports_priority(),
@@ -9,8 +9,8 @@ use language_model::{
LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
ModelRateLimiters, ProviderSettingsView, RateLimiter, SubPageProviderSettings,
unavailable_error,
ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
SubPageProviderSettings, unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -95,6 +95,7 @@ impl OpenAiCompatibleLanguageModelProvider {
any: model.capabilities.tools,
none: true,
},
prompt_compaction_strategy: PromptCompactionStrategy::RebuildPrompt,
supports_streaming_tools: true,
supports_thinking: default_thinking_reasoning_effort(model).is_some(),
supported_effort_levels: supported_thinking_effort_levels(model).into(),
+3 -2
View File
@@ -10,8 +10,8 @@ use language_model::{
LanguageModelClient, LanguageModelCompletionError, LanguageModelCompletionStream,
LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProvider,
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView,
RateLimiter, env_var, unavailable_error,
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters,
PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, unavailable_error,
};
pub use settings::XaiAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -395,6 +395,7 @@ fn language_model(model: &x_ai::Model) -> LanguageModel {
supports_images: model.supports_images(),
supports_streaming_tools: true,
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_thinking: model.supports_reasoning_effort(),
supported_effort_levels: supported_thinking_effort_levels(model).into(),
max_output_tokens: model.max_output_tokens(),
@@ -23,8 +23,9 @@ use language_model::{
LanguageModelCompletionEvent, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID,
OPEN_AI_PROVIDER_NAME, ProviderErrorCategory, RateLimiter, X_AI_PROVIDER_ID,
X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME, unavailable_error,
OPEN_AI_PROVIDER_NAME, PromptCompactionStrategy, ProviderErrorCategory, RateLimiter,
X_AI_PROVIDER_ID, X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME,
unavailable_error,
};
use schemars::JsonSchema;
@@ -546,6 +547,11 @@ pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel
none: true,
any: model.provider != Anthropic || anthropic::supports_forced_tool_use(&model.id.0),
},
// Anthropic invalidates cached messages when `tool_choice` changes.
prompt_compaction_strategy: match model.provider {
OpenAi | XAi => PromptCompactionStrategy::PreserveRequestPrefix,
Anthropic | Google => PromptCompactionStrategy::RebuildPrompt,
},
supports_split_token_display: matches!(model.provider, OpenAi | XAi),
max_output_tokens: Some(model.max_output_tokens as u64),
..LanguageModel::new(
@@ -2140,6 +2146,18 @@ mod tests {
);
}
#[test]
fn prompt_compaction_preserves_prefix_only_for_supported_upstreams() {
assert_eq!(
language_model(&cloud_test_config()).prompt_compaction_strategy,
PromptCompactionStrategy::PreserveRequestPrefix
);
assert_eq!(
language_model(&cloud_anthropic_test_config()).prompt_compaction_strategy,
PromptCompactionStrategy::RebuildPrompt
);
}
#[test]
fn config_resolves_current_model_list_and_rejects_unlisted_models() {
let listed = language_model(&cloud_test_config());