mirror of
https://github.com/zed-industries/zed.git
synced 2026-11-19 06:59:22 +01:00
language_models_cloud: Support keepalives during explicit compaction (#66335)
Add a keepalive status to the completion protocol for Zed-hosted models and let explicit OpenAI compaction negotiate Cloud’s existing status-message envelope. Terminal response statuses without a compaction result now fail immediately instead of waiting for the HTTP body to close. This is the client-side prerequisite for Cloud to send keepalives during compaction. Release Notes: - N/A
This commit is contained in:
@@ -234,6 +234,7 @@ pub enum CompletionRequestStatus {
|
||||
position: usize,
|
||||
},
|
||||
Started,
|
||||
KeepAlive,
|
||||
Failed {
|
||||
code: String,
|
||||
message: String,
|
||||
|
||||
@@ -92,7 +92,9 @@ impl LanguageModelCompletionEvent {
|
||||
Ok(Some(LanguageModelCompletionEvent::Queued { position }))
|
||||
}
|
||||
CompletionRequestStatus::Started => Ok(Some(LanguageModelCompletionEvent::Started)),
|
||||
CompletionRequestStatus::Unknown | CompletionRequestStatus::StreamEnded => Ok(None),
|
||||
CompletionRequestStatus::KeepAlive
|
||||
| CompletionRequestStatus::Unknown
|
||||
| CompletionRequestStatus::StreamEnded => Ok(None),
|
||||
CompletionRequestStatus::Failed {
|
||||
code,
|
||||
message,
|
||||
|
||||
@@ -209,6 +209,11 @@ pub struct PerformLlmCompletionResponse {
|
||||
pub includes_status_messages: bool,
|
||||
}
|
||||
|
||||
enum StatusMessages {
|
||||
Request,
|
||||
Omit,
|
||||
}
|
||||
|
||||
impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
pub async fn perform_llm_completion(
|
||||
http_client: &HttpClientWithUrl,
|
||||
@@ -219,7 +224,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
|
||||
Self::perform_llm_request(
|
||||
"/completions",
|
||||
true,
|
||||
StatusMessages::Request,
|
||||
http_client,
|
||||
token_provider,
|
||||
auth_context,
|
||||
@@ -238,7 +243,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
|
||||
Self::perform_llm_request(
|
||||
"/completions/compact",
|
||||
false,
|
||||
StatusMessages::Request,
|
||||
http_client,
|
||||
token_provider,
|
||||
auth_context,
|
||||
@@ -250,7 +255,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
|
||||
async fn perform_llm_request(
|
||||
path: &str,
|
||||
request_status_messages: bool,
|
||||
status_messages: StatusMessages,
|
||||
http_client: &HttpClientWithUrl,
|
||||
token_provider: &TP,
|
||||
auth_context: TP::AuthContext,
|
||||
@@ -277,7 +282,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
})
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Authorization", format!("Bearer {token}"));
|
||||
if request_status_messages {
|
||||
if matches!(status_messages, StatusMessages::Request) {
|
||||
request = request
|
||||
.header(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
|
||||
.header(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME, "true");
|
||||
@@ -289,7 +294,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
|
||||
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
let includes_status_messages = request_status_messages
|
||||
let includes_status_messages = matches!(status_messages, StatusMessages::Request)
|
||||
&& response
|
||||
.headers()
|
||||
.get(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME)
|
||||
@@ -859,6 +864,20 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
|
||||
.map_err(LanguageModelCompletionError::Other)?;
|
||||
return Ok(CompactionResult { context, usage });
|
||||
}
|
||||
CompletionEvent::Status(CompletionRequestStatus::StreamEnded) => break,
|
||||
CompletionEvent::Status(CompletionRequestStatus::Failed {
|
||||
code,
|
||||
message,
|
||||
retry_after,
|
||||
..
|
||||
}) => {
|
||||
return Err(LanguageModelCompletionError::from_cloud_failure(
|
||||
provider_name,
|
||||
code,
|
||||
message,
|
||||
retry_after.map(Duration::from_secs_f64),
|
||||
));
|
||||
}
|
||||
CompletionEvent::Status(_) => {}
|
||||
}
|
||||
}
|
||||
@@ -955,7 +974,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
|
||||
.run(async move {
|
||||
let mut response = Self::perform_llm_request(
|
||||
"/count_tokens",
|
||||
false,
|
||||
StatusMessages::Omit,
|
||||
&http_client,
|
||||
&*token_provider,
|
||||
auth_context,
|
||||
@@ -1372,6 +1391,7 @@ pub fn response_lines<T: DeserializeOwned>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::TryStreamExt as _;
|
||||
use http_client::FakeHttpClient;
|
||||
use http_client::http::{HeaderMap, StatusCode};
|
||||
use language_model::{
|
||||
@@ -1393,6 +1413,7 @@ mod tests {
|
||||
json!({"event": {"candidates": [{
|
||||
"content": {"role": "model", "parts": [{"text": "Hello"}]}
|
||||
}]}}),
|
||||
json!({"status": CompletionRequestStatus::KeepAlive}),
|
||||
json!({"event": {
|
||||
"candidates": [{
|
||||
"content": {"role": "model", "parts": [{"text": " world"}]},
|
||||
@@ -1435,12 +1456,15 @@ mod tests {
|
||||
cx: &mut gpui::TestAppContext,
|
||||
) {
|
||||
for stream_ended in [false, true] {
|
||||
let mut events = vec![json!({"event": {"candidates": [{
|
||||
"content": {"role": "model", "parts": [{
|
||||
"functionCall": {"name": "list_directory", "args": {"path": "."}}
|
||||
}]},
|
||||
"finishReason": "STOP"
|
||||
}]}})];
|
||||
let mut events = vec![
|
||||
json!({"event": {"candidates": [{
|
||||
"content": {"role": "model", "parts": [{
|
||||
"functionCall": {"name": "list_directory", "args": {"path": "."}}
|
||||
}]},
|
||||
"finishReason": "STOP"
|
||||
}]}}),
|
||||
json!({"status": CompletionRequestStatus::KeepAlive}),
|
||||
];
|
||||
if stream_ended {
|
||||
events.push(json!({"status": "stream_ended"}));
|
||||
}
|
||||
@@ -1510,118 +1534,222 @@ mod tests {
|
||||
async fn cloud_explicit_compaction_forwards_supported_request_fields(
|
||||
cx: &mut gpui::TestAppContext,
|
||||
) {
|
||||
let captured_request = Arc::new(Mutex::new(None));
|
||||
let captured_request_for_handler = captured_request.clone();
|
||||
let http_client = FakeHttpClient::create(move |request| {
|
||||
let captured_request = captured_request_for_handler.clone();
|
||||
async move {
|
||||
let method = request.method().clone();
|
||||
let uri = request.uri().to_string();
|
||||
let authorization = request
|
||||
.headers()
|
||||
.get("Authorization")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
let requested_status_messages = request
|
||||
.headers()
|
||||
.contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);
|
||||
let requested_stream_end = request
|
||||
.headers()
|
||||
.contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);
|
||||
let mut body = request.into_body();
|
||||
let mut body_text = String::new();
|
||||
body.read_to_string(&mut body_text).await?;
|
||||
*captured_request.lock().unwrap() = Some((
|
||||
method,
|
||||
uri,
|
||||
authorization,
|
||||
requested_status_messages,
|
||||
requested_stream_end,
|
||||
body_text,
|
||||
));
|
||||
|
||||
Ok(http_client::Response::builder()
|
||||
.status(200)
|
||||
.body(AsyncBody::from(format!(
|
||||
"{}\n",
|
||||
json!({
|
||||
"id": "resp_compact",
|
||||
"created_at": 1_700_000_000,
|
||||
"object": "response.compaction",
|
||||
"output": [{
|
||||
"type": "compaction",
|
||||
"id": "cmp_manual",
|
||||
"encrypted_content": "opaque-state"
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 100,
|
||||
"input_tokens_details": {"cached_tokens": 20},
|
||||
"output_tokens": 10,
|
||||
"output_tokens_details": {"reasoning_tokens": 5},
|
||||
"total_tokens": 110
|
||||
}
|
||||
})
|
||||
)))?)
|
||||
}
|
||||
});
|
||||
let language_model = language_model(&cloud_test_config());
|
||||
assert!(language_model.supports_explicit_compaction());
|
||||
assert!(!language_model.supports_explicit_compaction_output_limit());
|
||||
|
||||
for includes_status_messages in [false, true] {
|
||||
let captured_request = Arc::new(Mutex::new(None));
|
||||
let captured_request_for_handler = captured_request.clone();
|
||||
let http_client = FakeHttpClient::create(move |request| {
|
||||
let captured_request = captured_request_for_handler.clone();
|
||||
async move {
|
||||
let method = request.method().clone();
|
||||
let uri = request.uri().to_string();
|
||||
let authorization = request
|
||||
.headers()
|
||||
.get("Authorization")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
let requested_status_messages = request
|
||||
.headers()
|
||||
.contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);
|
||||
let requested_stream_end = request
|
||||
.headers()
|
||||
.contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);
|
||||
let mut body = request.into_body();
|
||||
let mut body_text = String::new();
|
||||
body.read_to_string(&mut body_text).await?;
|
||||
*captured_request.lock().unwrap() = Some((
|
||||
method,
|
||||
uri,
|
||||
authorization,
|
||||
requested_status_messages,
|
||||
requested_stream_end,
|
||||
body_text,
|
||||
));
|
||||
|
||||
let result = json!({
|
||||
"id": "resp_compact",
|
||||
"created_at": 1_700_000_000,
|
||||
"object": "response.compaction",
|
||||
"output": [{
|
||||
"type": "compaction",
|
||||
"id": "cmp_manual",
|
||||
"encrypted_content": "opaque-state"
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 100,
|
||||
"input_tokens_details": {"cached_tokens": 20},
|
||||
"output_tokens": 10,
|
||||
"output_tokens_details": {"reasoning_tokens": 5},
|
||||
"total_tokens": 110
|
||||
}
|
||||
});
|
||||
let mut response = http_client::Response::builder().status(200);
|
||||
let body = if includes_status_messages {
|
||||
response =
|
||||
response.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true");
|
||||
[
|
||||
json!({"status": "keep_alive"}),
|
||||
json!({"status": "future_transport_status"}),
|
||||
json!({"event": result}),
|
||||
json!({"status": "stream_ended"}),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|event| format!("{event}\n"))
|
||||
.collect::<String>()
|
||||
} else {
|
||||
format!("{result}\n")
|
||||
};
|
||||
Ok(response.body(AsyncBody::from(body))?)
|
||||
}
|
||||
});
|
||||
let (provider, model) = cloud_test_model(http_client);
|
||||
let request = compact_test_request();
|
||||
|
||||
let result = cx
|
||||
.update(|cx| provider.compact(&model, request, cx))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result.usage,
|
||||
language_model::TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 10,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 20,
|
||||
}
|
||||
);
|
||||
let language_model::CompactedContext::ProviderState(state) = result.context else {
|
||||
panic!("expected provider compaction state");
|
||||
};
|
||||
assert_eq!(
|
||||
open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID)
|
||||
.unwrap(),
|
||||
Some(vec![json!({
|
||||
"type": "compaction",
|
||||
"id": "cmp_manual",
|
||||
"encrypted_content": "opaque-state"
|
||||
})])
|
||||
);
|
||||
let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =
|
||||
captured_request.lock().unwrap().take().unwrap();
|
||||
assert_eq!(method, Method::POST);
|
||||
assert_eq!(uri, "http://test.example/completions/compact?");
|
||||
assert_eq!(authorization.as_deref(), Some("Bearer test-token"));
|
||||
assert!(requested_status_messages);
|
||||
assert!(requested_stream_end);
|
||||
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
|
||||
assert_eq!(body["thread_id"], "thread-123");
|
||||
assert_eq!(body["provider"], "open_ai");
|
||||
assert_eq!(body["model"], "gpt-5.4");
|
||||
assert_eq!(
|
||||
body["provider_request"],
|
||||
json!({
|
||||
"model": "gpt-5.4",
|
||||
"input": [{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": "Retain this context."
|
||||
}]
|
||||
}],
|
||||
"prompt_cache_key": "thread-123",
|
||||
"service_tier": "priority"
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn cloud_explicit_compaction_preserves_streamed_failure(cx: &mut gpui::TestAppContext) {
|
||||
let http_client = FakeHttpClient::create(|_| async move {
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
|
||||
.body(AsyncBody::from(format!(
|
||||
"{}\n{}\n",
|
||||
json!({"status": "keep_alive"}),
|
||||
json!({"status": {"failed": {
|
||||
"code": "upstream_http_429",
|
||||
"message": "Compaction rate limit exceeded.",
|
||||
"request_id": "00000000-0000-0000-0000-000000000000",
|
||||
"retry_after": 1.5
|
||||
}}}),
|
||||
)))?)
|
||||
});
|
||||
let (provider, model) = cloud_test_model(http_client);
|
||||
let request = compact_test_request();
|
||||
|
||||
let result = cx
|
||||
.update(|cx| provider.compact(&model, request, cx))
|
||||
let error = cx
|
||||
.update(|cx| provider.compact(&model, compact_test_request(), cx))
|
||||
.await
|
||||
.unwrap();
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(
|
||||
result.usage,
|
||||
language_model::TokenUsage {
|
||||
input_tokens: 80,
|
||||
output_tokens: 10,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 20,
|
||||
}
|
||||
);
|
||||
let language_model::CompactedContext::ProviderState(state) = result.context else {
|
||||
panic!("expected provider compaction state");
|
||||
let LanguageModelCompletionError::ProviderRejection {
|
||||
provider,
|
||||
status,
|
||||
code,
|
||||
message,
|
||||
retry_after,
|
||||
category,
|
||||
} = error
|
||||
else {
|
||||
panic!("expected the streamed provider rejection, got {error:?}");
|
||||
};
|
||||
assert_eq!(
|
||||
open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID).unwrap(),
|
||||
Some(vec![json!({
|
||||
"type": "compaction",
|
||||
"id": "cmp_manual",
|
||||
"encrypted_content": "opaque-state"
|
||||
})])
|
||||
);
|
||||
let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =
|
||||
captured_request.lock().unwrap().take().unwrap();
|
||||
assert_eq!(method, Method::POST);
|
||||
assert_eq!(uri, "http://test.example/completions/compact?");
|
||||
assert_eq!(authorization.as_deref(), Some("Bearer test-token"));
|
||||
assert!(!requested_status_messages);
|
||||
assert!(!requested_stream_end);
|
||||
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
|
||||
assert_eq!(body["thread_id"], "thread-123");
|
||||
assert_eq!(body["provider"], "open_ai");
|
||||
assert_eq!(body["model"], "gpt-5.4");
|
||||
assert_eq!(
|
||||
body["provider_request"],
|
||||
json!({
|
||||
"model": "gpt-5.4",
|
||||
"input": [{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": "Retain this context."
|
||||
}]
|
||||
}],
|
||||
"prompt_cache_key": "thread-123",
|
||||
"service_tier": "priority"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider, OPEN_AI_PROVIDER_NAME);
|
||||
assert_eq!(status, Some(StatusCode::TOO_MANY_REQUESTS));
|
||||
assert_eq!(code.as_deref(), Some("upstream_http_429"));
|
||||
assert_eq!(message, "Compaction rate limit exceeded.");
|
||||
assert_eq!(retry_after, Some(Duration::from_secs_f64(1.5)));
|
||||
assert_eq!(category, ProviderErrorCategory::RateLimit);
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
async fn cloud_explicit_compaction_rejects_stream_ending_without_result(
|
||||
cx: &mut gpui::TestAppContext,
|
||||
) {
|
||||
for stream_ended in [false, true] {
|
||||
let http_client = FakeHttpClient::create(move |_| async move {
|
||||
let mut body = format!("{}\n", json!({"status": "keep_alive"}));
|
||||
if stream_ended {
|
||||
body.push_str(&format!("{}\n", json!({"status": "stream_ended"})));
|
||||
}
|
||||
let body = stream::iter([Ok::<_, std::io::Error>(body.into_bytes())])
|
||||
.chain(stream::poll_fn(move |_| {
|
||||
if stream_ended {
|
||||
Poll::Pending
|
||||
} else {
|
||||
Poll::Ready(None)
|
||||
}
|
||||
}))
|
||||
.into_async_read();
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
|
||||
.body(AsyncBody::from_reader(body))?)
|
||||
});
|
||||
let (provider, model) = cloud_test_model(http_client);
|
||||
|
||||
let error = cx
|
||||
.update(|cx| provider.compact(&model, compact_test_request(), cx))
|
||||
.now_or_never()
|
||||
.expect(
|
||||
"compaction should stop at EOF or StreamEnded without waiting for more bytes",
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
LanguageModelCompletionError::StreamEndedUnexpectedly { provider }
|
||||
if provider == &OPEN_AI_PROVIDER_NAME
|
||||
),
|
||||
"expected a missing compaction result error, got {error:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[gpui::test]
|
||||
|
||||
Reference in New Issue
Block a user