Preserve mid-conversation system prompt updates (#66490)

Release Notes:

- N/A
This commit is contained in:
Bennet Bo Fenner
2026-11-16 13:07:38 +00:00
committed by GitHub
parent 99dd426cba
commit 89d3da6578
9 changed files with 171 additions and 18 deletions
+21
View File
@@ -77,6 +77,26 @@ pub fn supports_forced_tool_use(model_id: &str) -> bool {
)
}
/// Whether this model accepts system messages inside conversation history.
///
/// Older models retain the flattened system-prompt representation.
/// <https://platform.claude.com/docs/en/build-with-claude/mid-conversation-system-messages>
pub fn supports_mid_conversation_system_messages(model_id: &str) -> bool {
matches!(
model_id,
"claude-fable-5-1"
| "claude-mythos-5-1"
| "claude-fable-5"
| "claude-mythos-5"
| "claude-mythos-preview"
| "claude-opus-5-5"
| "claude-opus-5"
| "claude-opus-4-8"
| "claude-sonnet-5-5"
| "claude-haiku-5-5"
)
}
/// <https://platform.claude.com/docs/en/build-with-claude/compaction>
pub const COMPACTION_BETA_HEADER: &str = "compact-2026-01-12";
@@ -784,6 +804,7 @@ pub struct Message {
pub enum Role {
User,
Assistant,
System,
}
#[derive(Debug, Serialize, Deserialize)]
+94 -7
View File
@@ -310,6 +310,7 @@ pub fn into_anthropic(
let mut new_messages: Vec<Message> = Vec::new();
let mut system_message = String::new();
let mut any_message_wants_cache = false;
let supports_system_updates = crate::supports_mid_conversation_system_messages(&model);
for message in request.messages {
if message.contents_empty() {
@@ -358,10 +359,32 @@ pub fn into_anthropic(
});
}
Role::System => {
if !system_message.is_empty() {
system_message.push_str("\n\n");
if supports_system_updates && !new_messages.is_empty() {
anyhow::ensure!(
message
.content
.iter()
.all(|content| matches!(content, MessageContent::Text(_))),
"Anthropic system updates must contain only text"
);
new_messages.push(Message {
role: crate::Role::System,
content: vec![RequestContent::Text {
text: message.string_contents(),
cache_control: (cache_mode == AnthropicPromptCacheMode::Legacy
&& message.cache)
.then_some(CacheControl {
cache_type: CacheControlType::Ephemeral,
ttl: None,
}),
}],
});
} else {
if !system_message.is_empty() {
system_message.push_str("\n\n");
}
system_message.push_str(&message.string_contents());
}
system_message.push_str(&message.string_contents());
}
}
}
@@ -936,6 +959,12 @@ mod tests {
cache: true,
reasoning_details: None,
},
LanguageModelRequestMessage {
role: Role::System,
content: vec![MessageContent::Text("Prefer terse answers.\n".into())],
cache: true,
reasoning_details: None,
},
],
thread_id: None,
prompt_cache_key: None,
@@ -958,8 +987,8 @@ mod tests {
};
let anthropic_request = into_anthropic(
request,
"claude-3-5-sonnet".to_string(),
request.clone(),
"claude-opus-4-8".to_string(),
0.7,
4096,
AnthropicModelMode::Default,
@@ -974,8 +1003,12 @@ mod tests {
);
// No message content block should carry cache_control anymore; the
// conversation breakpoint is set via top-level automatic caching.
assert_eq!(anthropic_request.messages.len(), 1);
for block in &anthropic_request.messages[0].content {
assert_eq!(anthropic_request.messages.len(), 2);
for block in anthropic_request
.messages
.iter()
.flat_map(|message| &message.content)
{
let cache_control = match block {
RequestContent::Text { cache_control, .. }
| RequestContent::Thinking { cache_control, .. }
@@ -1003,6 +1036,15 @@ mod tests {
// System prompt is emitted in array form with a long-TTL breakpoint on
// the final text block.
let serialized = serde_json::to_value(&anthropic_request).unwrap();
assert_eq!(serialized["system"][0]["text"], "You are helpful.");
assert_eq!(
serialized["messages"][1],
serde_json::json!({
"role": "system",
"content": [{"type": "text", "text": "Prefer terse answers.\n"}]
}),
);
match anthropic_request.system {
Some(StringOrContents::Content(ref blocks)) => {
assert_eq!(blocks.len(), 1);
@@ -1029,6 +1071,51 @@ mod tests {
ttl: Some(CacheTtl::OneHour),
})
));
let mut later_user_message = request.clone();
later_user_message
.messages
.push(LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Another user message.".into())],
cache: true,
reasoning_details: None,
});
let later_user_message = into_anthropic(
later_user_message,
"claude-opus-4-8".to_string(),
0.7,
4096,
AnthropicModelMode::Default,
AnthropicPromptCacheMode::Automatic,
&ANTHROPIC_PROVIDER_ID,
)
.unwrap();
assert_eq!(
later_user_message
.messages
.iter()
.map(|message| &message.role)
.collect::<Vec<_>>(),
vec![&crate::Role::User, &crate::Role::System, &crate::Role::User],
);
let legacy_request = into_anthropic(
request,
"claude-3-5-sonnet".to_string(),
0.7,
4096,
AnthropicModelMode::Default,
AnthropicPromptCacheMode::Automatic,
&ANTHROPIC_PROVIDER_ID,
)
.unwrap();
let serialized = serde_json::to_value(&legacy_request).unwrap();
assert_eq!(
serialized["system"][0]["text"],
"You are helpful.\n\nPrefer terse answers.\n"
);
assert_eq!(legacy_request.messages.len(), 1);
}
#[test]
+5
View File
@@ -63,6 +63,11 @@ pub fn language_model(model: &CopilotChatModel) -> LanguageModel {
supports_streaming_tools: true,
supports_images: model.supports_vision(),
supports_thinking: model.can_think(),
supports_system_prompt_updates: if model.supports_messages() {
anthropic::supports_mid_conversation_system_messages(model.id())
} else {
model.supports_response()
},
supported_effort_levels: supported_effort_levels.into(),
tool_choice_support: LanguageModelToolChoiceSupport {
auto: supports_tools,
@@ -247,6 +247,7 @@ impl BatchingLlmClient {
role: match msg.role {
Role::User => "user".to_string(),
Role::Assistant => "assistant".to_string(),
Role::System => "system".to_string(),
},
content: message_content_to_string(&msg.content),
})
@@ -571,6 +572,7 @@ impl BatchingLlmClient {
role: match msg.role.as_str() {
"user" => Role::User,
"assistant" => Role::Assistant,
"system" => Role::System,
_ => Role::User,
},
content: vec![RequestContent::Text {
@@ -129,6 +129,8 @@ pub struct LanguageModel {
/// request. Only meaningful when `supports_thinking` is `true`.
pub supports_disabling_thinking: bool,
pub supports_fast_mode: bool,
/// Whether instruction updates can retain their positions in conversation history.
pub supports_system_prompt_updates: bool,
/// The effort levels that can be used when thinking.
pub supported_effort_levels: Arc<[LanguageModelEffortLevel]>,
/// Whether this model supports provider-side automatic context
@@ -220,6 +222,7 @@ impl LanguageModel {
supports_thinking: false,
supports_disabling_thinking: true,
supports_fast_mode: false,
supports_system_prompt_updates: false,
supported_effort_levels: Arc::default(),
supports_server_side_compaction: false,
supports_explicit_compaction: false,
@@ -353,6 +353,10 @@ impl From<&str> for MessageContent {
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
pub struct LanguageModelRequestMessage {
/// Leading system messages form the initial prompt; later ones update the conversation.
///
/// Supporting providers preserve later system messages at their position.
/// Other providers may flatten them into the initial instructions.
pub role: Role,
pub content: Vec<MessageContent>,
pub cache: bool,
@@ -529,6 +529,11 @@ pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel
supports_thinking: model.supports_thinking,
supports_disabling_thinking: model.supports_disabling_thinking,
supports_fast_mode: model.supports_fast_mode,
supports_system_prompt_updates: match model.provider {
Anthropic => anthropic::supports_mid_conversation_system_messages(&model.id.0),
OpenAi => true,
Google | XAi => false,
},
supports_server_side_compaction: model.supports_server_side_compaction,
supports_explicit_compaction,
supports_explicit_compaction_output_limit: model.provider == Anthropic
+34 -11
View File
@@ -303,15 +303,13 @@ pub fn into_open_ai_response(
let mut replayed_reasoning_item_indexes = HashMap::default();
let mut tool_use_kinds_by_id = HashMap::default();
let mut system_instructions = Vec::new();
let mut in_initial_system_prompt = true;
for (index, message) in messages.into_iter().enumerate() {
// System messages go to the top-level `instructions` field rather
// than the input item list. `instructions` is applied per request
// (the Responses API documents it as a system message inserted into
// the model's context), so the current system prompt survives when
// replaying provider compaction state replaces the accumulated input
// items below. This also matches the Codex backend, which rejects
// system-role input items outright.
if message.role == Role::System {
// Only the initial prompt belongs in `instructions`, outside a
// provider-compacted input window. Later instructions must retain their
// position; developer-role items also work on Codex, which rejects
// system-role input items.
if message.role == Role::System && in_initial_system_prompt {
for content in message.content {
if let MessageContent::Text(text) = content
&& !text.trim().is_empty()
@@ -321,6 +319,9 @@ pub fn into_open_ai_response(
}
continue;
}
if !message.contents_empty() {
in_initial_system_prompt = false;
}
append_message_to_response_items(
message,
index,
@@ -698,7 +699,7 @@ fn flush_response_parts(
role: match role {
Role::User => crate::Role::User,
Role::Assistant => crate::Role::Assistant,
Role::System => crate::Role::System,
Role::System => crate::Role::Developer,
},
content: parts.clone(),
phase: match role {
@@ -4578,7 +4579,7 @@ mod tests {
}
#[test]
fn into_open_ai_response_hoists_system_messages_into_instructions() {
fn into_open_ai_response_keeps_system_updates_as_developer_items() {
let request = LanguageModelRequest {
messages: vec![
LanguageModelRequestMessage {
@@ -4618,7 +4619,7 @@ mod tests {
let serialized = serde_json::to_value(&response).unwrap();
assert_eq!(
serialized["instructions"],
json!("You are a coding assistant.\n\nPrefer terse answers.")
json!("You are a coding assistant.")
);
assert_eq!(
serialized["input"],
@@ -4629,6 +4630,13 @@ mod tests {
"content": [
{ "type": "input_text", "text": "Hello." }
]
},
{
"type": "message",
"role": "developer",
"content": [
{ "type": "input_text", "text": "Prefer terse answers." }
]
}
])
);
@@ -4678,6 +4686,14 @@ mod tests {
cache: false,
reasoning_details: None,
},
LanguageModelRequestMessage {
role: Role::System,
content: vec![MessageContent::Text(
"Use the updated project context.".into(),
)],
cache: false,
reasoning_details: None,
},
],
..Default::default()
};
@@ -4710,6 +4726,13 @@ mod tests {
"content": [
{ "type": "input_text", "text": "Continue." }
]
},
{
"type": "message",
"role": "developer",
"content": [
{ "type": "input_text", "text": "Use the updated project context." }
]
}
])
);
+3
View File
@@ -33,6 +33,7 @@ pub enum Role {
User,
Assistant,
System,
Developer,
Tool,
}
@@ -44,6 +45,7 @@ impl TryFrom<String> for Role {
"user" => Ok(Self::User),
"assistant" => Ok(Self::Assistant),
"system" => Ok(Self::System),
"developer" => Ok(Self::Developer),
"tool" => Ok(Self::Tool),
_ => anyhow::bail!("invalid role '{value}'"),
}
@@ -56,6 +58,7 @@ impl From<Role> for String {
Role::User => "user".to_owned(),
Role::Assistant => "assistant".to_owned(),
Role::System => "system".to_owned(),
Role::Developer => "developer".to_owned(),
Role::Tool => "tool".to_owned(),
}
}