From 89d3da65789d8728d01a82d284924334418065f7 Mon Sep 17 00:00:00 2001 From: Bennet Bo Fenner Date: Mon, 16 Nov 2026 13:07:38 +0000 Subject: [PATCH] Preserve mid-conversation system prompt updates (#66490) Release Notes: - N/A --- crates/anthropic/src/anthropic.rs | 21 ++++ crates/anthropic/src/completion.rs | 101 ++++++++++++++++-- crates/copilot_chat/src/model.rs | 5 + .../src/anthropic_client.rs | 2 + crates/language_model/src/language_model.rs | 3 + crates/language_model_core/src/request.rs | 4 + .../src/language_models_cloud.rs | 5 + crates/open_ai/src/completion.rs | 45 ++++++-- crates/open_ai/src/open_ai.rs | 3 + 9 files changed, 171 insertions(+), 18 deletions(-) diff --git a/crates/anthropic/src/anthropic.rs b/crates/anthropic/src/anthropic.rs index 7d6060cc980..b457f947252 100644 --- a/crates/anthropic/src/anthropic.rs +++ b/crates/anthropic/src/anthropic.rs @@ -77,6 +77,26 @@ pub fn supports_forced_tool_use(model_id: &str) -> bool { ) } +/// Whether this model accepts system messages inside conversation history. +/// +/// Older models retain the flattened system-prompt representation. +/// +pub fn supports_mid_conversation_system_messages(model_id: &str) -> bool { + matches!( + model_id, + "claude-fable-5-1" + | "claude-mythos-5-1" + | "claude-fable-5" + | "claude-mythos-5" + | "claude-mythos-preview" + | "claude-opus-5-5" + | "claude-opus-5" + | "claude-opus-4-8" + | "claude-sonnet-5-5" + | "claude-haiku-5-5" + ) +} + /// pub const COMPACTION_BETA_HEADER: &str = "compact-2026-01-12"; @@ -784,6 +804,7 @@ pub struct Message { pub enum Role { User, Assistant, + System, } #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs index 95e2b498711..efea1b775e2 100644 --- a/crates/anthropic/src/completion.rs +++ b/crates/anthropic/src/completion.rs @@ -310,6 +310,7 @@ pub fn into_anthropic( let mut new_messages: Vec = Vec::new(); let mut system_message = String::new(); let mut any_message_wants_cache = false; + let supports_system_updates = crate::supports_mid_conversation_system_messages(&model); for message in request.messages { if message.contents_empty() { @@ -358,10 +359,32 @@ pub fn into_anthropic( }); } Role::System => { - if !system_message.is_empty() { - system_message.push_str("\n\n"); + if supports_system_updates && !new_messages.is_empty() { + anyhow::ensure!( + message + .content + .iter() + .all(|content| matches!(content, MessageContent::Text(_))), + "Anthropic system updates must contain only text" + ); + new_messages.push(Message { + role: crate::Role::System, + content: vec![RequestContent::Text { + text: message.string_contents(), + cache_control: (cache_mode == AnthropicPromptCacheMode::Legacy + && message.cache) + .then_some(CacheControl { + cache_type: CacheControlType::Ephemeral, + ttl: None, + }), + }], + }); + } else { + if !system_message.is_empty() { + system_message.push_str("\n\n"); + } + system_message.push_str(&message.string_contents()); } - system_message.push_str(&message.string_contents()); } } } @@ -936,6 +959,12 @@ mod tests { cache: true, reasoning_details: None, }, + LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text("Prefer terse answers.\n".into())], + cache: true, + reasoning_details: None, + }, ], thread_id: None, prompt_cache_key: None, @@ -958,8 +987,8 @@ mod tests { }; let anthropic_request = into_anthropic( - request, - "claude-3-5-sonnet".to_string(), + request.clone(), + "claude-opus-4-8".to_string(), 0.7, 4096, AnthropicModelMode::Default, @@ -974,8 +1003,12 @@ mod tests { ); // No message content block should carry cache_control anymore; the // conversation breakpoint is set via top-level automatic caching. - assert_eq!(anthropic_request.messages.len(), 1); - for block in &anthropic_request.messages[0].content { + assert_eq!(anthropic_request.messages.len(), 2); + for block in anthropic_request + .messages + .iter() + .flat_map(|message| &message.content) + { let cache_control = match block { RequestContent::Text { cache_control, .. } | RequestContent::Thinking { cache_control, .. } @@ -1003,6 +1036,15 @@ mod tests { // System prompt is emitted in array form with a long-TTL breakpoint on // the final text block. + let serialized = serde_json::to_value(&anthropic_request).unwrap(); + assert_eq!(serialized["system"][0]["text"], "You are helpful."); + assert_eq!( + serialized["messages"][1], + serde_json::json!({ + "role": "system", + "content": [{"type": "text", "text": "Prefer terse answers.\n"}] + }), + ); match anthropic_request.system { Some(StringOrContents::Content(ref blocks)) => { assert_eq!(blocks.len(), 1); @@ -1029,6 +1071,51 @@ mod tests { ttl: Some(CacheTtl::OneHour), }) )); + + let mut later_user_message = request.clone(); + later_user_message + .messages + .push(LanguageModelRequestMessage { + role: Role::User, + content: vec![MessageContent::Text("Another user message.".into())], + cache: true, + reasoning_details: None, + }); + let later_user_message = into_anthropic( + later_user_message, + "claude-opus-4-8".to_string(), + 0.7, + 4096, + AnthropicModelMode::Default, + AnthropicPromptCacheMode::Automatic, + &ANTHROPIC_PROVIDER_ID, + ) + .unwrap(); + assert_eq!( + later_user_message + .messages + .iter() + .map(|message| &message.role) + .collect::>(), + vec![&crate::Role::User, &crate::Role::System, &crate::Role::User], + ); + + let legacy_request = into_anthropic( + request, + "claude-3-5-sonnet".to_string(), + 0.7, + 4096, + AnthropicModelMode::Default, + AnthropicPromptCacheMode::Automatic, + &ANTHROPIC_PROVIDER_ID, + ) + .unwrap(); + let serialized = serde_json::to_value(&legacy_request).unwrap(); + assert_eq!( + serialized["system"][0]["text"], + "You are helpful.\n\nPrefer terse answers.\n" + ); + assert_eq!(legacy_request.messages.len(), 1); } #[test] diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs index 1710ef0d738..b10e531da4a 100644 --- a/crates/copilot_chat/src/model.rs +++ b/crates/copilot_chat/src/model.rs @@ -63,6 +63,11 @@ pub fn language_model(model: &CopilotChatModel) -> LanguageModel { supports_streaming_tools: true, supports_images: model.supports_vision(), supports_thinking: model.can_think(), + supports_system_prompt_updates: if model.supports_messages() { + anthropic::supports_mid_conversation_system_messages(model.id()) + } else { + model.supports_response() + }, supported_effort_levels: supported_effort_levels.into(), tool_choice_support: LanguageModelToolChoiceSupport { auto: supports_tools, diff --git a/crates/edit_prediction_cli/src/anthropic_client.rs b/crates/edit_prediction_cli/src/anthropic_client.rs index 402b0bfd4a9..59061a91dc8 100644 --- a/crates/edit_prediction_cli/src/anthropic_client.rs +++ b/crates/edit_prediction_cli/src/anthropic_client.rs @@ -247,6 +247,7 @@ impl BatchingLlmClient { role: match msg.role { Role::User => "user".to_string(), Role::Assistant => "assistant".to_string(), + Role::System => "system".to_string(), }, content: message_content_to_string(&msg.content), }) @@ -571,6 +572,7 @@ impl BatchingLlmClient { role: match msg.role.as_str() { "user" => Role::User, "assistant" => Role::Assistant, + "system" => Role::System, _ => Role::User, }, content: vec![RequestContent::Text { diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs index b9ddec4678c..61e7259a51e 100644 --- a/crates/language_model/src/language_model.rs +++ b/crates/language_model/src/language_model.rs @@ -129,6 +129,8 @@ pub struct LanguageModel { /// request. Only meaningful when `supports_thinking` is `true`. pub supports_disabling_thinking: bool, pub supports_fast_mode: bool, + /// Whether instruction updates can retain their positions in conversation history. + pub supports_system_prompt_updates: bool, /// The effort levels that can be used when thinking. pub supported_effort_levels: Arc<[LanguageModelEffortLevel]>, /// Whether this model supports provider-side automatic context @@ -220,6 +222,7 @@ impl LanguageModel { supports_thinking: false, supports_disabling_thinking: true, supports_fast_mode: false, + supports_system_prompt_updates: false, supported_effort_levels: Arc::default(), supports_server_side_compaction: false, supports_explicit_compaction: false, diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rs index 0d9752e3e8c..fa7d4dbb423 100644 --- a/crates/language_model_core/src/request.rs +++ b/crates/language_model_core/src/request.rs @@ -353,6 +353,10 @@ impl From<&str> for MessageContent { #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] pub struct LanguageModelRequestMessage { + /// Leading system messages form the initial prompt; later ones update the conversation. + /// + /// Supporting providers preserve later system messages at their position. + /// Other providers may flatten them into the initial instructions. pub role: Role, pub content: Vec, pub cache: bool, diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs index a262aca88e9..f4242ce62eb 100644 --- a/crates/language_models_cloud/src/language_models_cloud.rs +++ b/crates/language_models_cloud/src/language_models_cloud.rs @@ -529,6 +529,11 @@ pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel supports_thinking: model.supports_thinking, supports_disabling_thinking: model.supports_disabling_thinking, supports_fast_mode: model.supports_fast_mode, + supports_system_prompt_updates: match model.provider { + Anthropic => anthropic::supports_mid_conversation_system_messages(&model.id.0), + OpenAi => true, + Google | XAi => false, + }, supports_server_side_compaction: model.supports_server_side_compaction, supports_explicit_compaction, supports_explicit_compaction_output_limit: model.provider == Anthropic diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs index 2b8e30bdd2f..367ffe53701 100644 --- a/crates/open_ai/src/completion.rs +++ b/crates/open_ai/src/completion.rs @@ -303,15 +303,13 @@ pub fn into_open_ai_response( let mut replayed_reasoning_item_indexes = HashMap::default(); let mut tool_use_kinds_by_id = HashMap::default(); let mut system_instructions = Vec::new(); + let mut in_initial_system_prompt = true; for (index, message) in messages.into_iter().enumerate() { - // System messages go to the top-level `instructions` field rather - // than the input item list. `instructions` is applied per request - // (the Responses API documents it as a system message inserted into - // the model's context), so the current system prompt survives when - // replaying provider compaction state replaces the accumulated input - // items below. This also matches the Codex backend, which rejects - // system-role input items outright. - if message.role == Role::System { + // Only the initial prompt belongs in `instructions`, outside a + // provider-compacted input window. Later instructions must retain their + // position; developer-role items also work on Codex, which rejects + // system-role input items. + if message.role == Role::System && in_initial_system_prompt { for content in message.content { if let MessageContent::Text(text) = content && !text.trim().is_empty() @@ -321,6 +319,9 @@ pub fn into_open_ai_response( } continue; } + if !message.contents_empty() { + in_initial_system_prompt = false; + } append_message_to_response_items( message, index, @@ -698,7 +699,7 @@ fn flush_response_parts( role: match role { Role::User => crate::Role::User, Role::Assistant => crate::Role::Assistant, - Role::System => crate::Role::System, + Role::System => crate::Role::Developer, }, content: parts.clone(), phase: match role { @@ -4578,7 +4579,7 @@ mod tests { } #[test] - fn into_open_ai_response_hoists_system_messages_into_instructions() { + fn into_open_ai_response_keeps_system_updates_as_developer_items() { let request = LanguageModelRequest { messages: vec![ LanguageModelRequestMessage { @@ -4618,7 +4619,7 @@ mod tests { let serialized = serde_json::to_value(&response).unwrap(); assert_eq!( serialized["instructions"], - json!("You are a coding assistant.\n\nPrefer terse answers.") + json!("You are a coding assistant.") ); assert_eq!( serialized["input"], @@ -4629,6 +4630,13 @@ mod tests { "content": [ { "type": "input_text", "text": "Hello." } ] + }, + { + "type": "message", + "role": "developer", + "content": [ + { "type": "input_text", "text": "Prefer terse answers." } + ] } ]) ); @@ -4678,6 +4686,14 @@ mod tests { cache: false, reasoning_details: None, }, + LanguageModelRequestMessage { + role: Role::System, + content: vec![MessageContent::Text( + "Use the updated project context.".into(), + )], + cache: false, + reasoning_details: None, + }, ], ..Default::default() }; @@ -4710,6 +4726,13 @@ mod tests { "content": [ { "type": "input_text", "text": "Continue." } ] + }, + { + "type": "message", + "role": "developer", + "content": [ + { "type": "input_text", "text": "Use the updated project context." } + ] } ]) ); diff --git a/crates/open_ai/src/open_ai.rs b/crates/open_ai/src/open_ai.rs index 072cfb6f237..da7249a11e9 100644 --- a/crates/open_ai/src/open_ai.rs +++ b/crates/open_ai/src/open_ai.rs @@ -33,6 +33,7 @@ pub enum Role { User, Assistant, System, + Developer, Tool, } @@ -44,6 +45,7 @@ impl TryFrom for Role { "user" => Ok(Self::User), "assistant" => Ok(Self::Assistant), "system" => Ok(Self::System), + "developer" => Ok(Self::Developer), "tool" => Ok(Self::Tool), _ => anyhow::bail!("invalid role '{value}'"), } @@ -56,6 +58,7 @@ impl From for String { Role::User => "user".to_owned(), Role::Assistant => "assistant".to_owned(), Role::System => "system".to_owned(), + Role::Developer => "developer".to_owned(), Role::Tool => "tool".to_owned(), } }