diff --git a/crates/cloud_llm_client/src/cloud_llm_client.rs b/crates/cloud_llm_client/src/cloud_llm_client.rs index 2999d5638f7..3173463b8ff 100644 --- a/crates/cloud_llm_client/src/cloud_llm_client.rs +++ b/crates/cloud_llm_client/src/cloud_llm_client.rs @@ -234,6 +234,7 @@ pub enum CompletionRequestStatus { position: usize, }, Started, + KeepAlive, Failed { code: String, message: String, diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rs index 3e13f1dc51a..7f033a6c0cf 100644 --- a/crates/language_model_core/src/language_model_core.rs +++ b/crates/language_model_core/src/language_model_core.rs @@ -92,7 +92,9 @@ impl LanguageModelCompletionEvent { Ok(Some(LanguageModelCompletionEvent::Queued { position })) } CompletionRequestStatus::Started => Ok(Some(LanguageModelCompletionEvent::Started)), - CompletionRequestStatus::Unknown | CompletionRequestStatus::StreamEnded => Ok(None), + CompletionRequestStatus::KeepAlive + | CompletionRequestStatus::Unknown + | CompletionRequestStatus::StreamEnded => Ok(None), CompletionRequestStatus::Failed { code, message, diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs index 5c2c99bea46..af77b3c9954 100644 --- a/crates/language_models_cloud/src/language_models_cloud.rs +++ b/crates/language_models_cloud/src/language_models_cloud.rs @@ -209,6 +209,11 @@ pub struct PerformLlmCompletionResponse { pub includes_status_messages: bool, } +enum StatusMessages { + Request, + Omit, +} + impl CloudModelProvider { pub async fn perform_llm_completion( http_client: &HttpClientWithUrl, @@ -219,7 +224,7 @@ impl CloudModelProvider { ) -> Result { Self::perform_llm_request( "/completions", - true, + StatusMessages::Request, http_client, token_provider, auth_context, @@ -238,7 +243,7 @@ impl CloudModelProvider { ) -> Result { Self::perform_llm_request( "/completions/compact", - false, + StatusMessages::Request, http_client, token_provider, auth_context, @@ -250,7 +255,7 @@ impl CloudModelProvider { async fn perform_llm_request( path: &str, - request_status_messages: bool, + status_messages: StatusMessages, http_client: &HttpClientWithUrl, token_provider: &TP, auth_context: TP::AuthContext, @@ -277,7 +282,7 @@ impl CloudModelProvider { }) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {token}")); - if request_status_messages { + if matches!(status_messages, StatusMessages::Request) { request = request .header(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true") .header(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME, "true"); @@ -289,7 +294,7 @@ impl CloudModelProvider { let status = response.status(); if status.is_success() { - let includes_status_messages = request_status_messages + let includes_status_messages = matches!(status_messages, StatusMessages::Request) && response .headers() .get(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME) @@ -859,6 +864,20 @@ impl CloudModelProvider { .map_err(LanguageModelCompletionError::Other)?; return Ok(CompactionResult { context, usage }); } + CompletionEvent::Status(CompletionRequestStatus::StreamEnded) => break, + CompletionEvent::Status(CompletionRequestStatus::Failed { + code, + message, + retry_after, + .. + }) => { + return Err(LanguageModelCompletionError::from_cloud_failure( + provider_name, + code, + message, + retry_after.map(Duration::from_secs_f64), + )); + } CompletionEvent::Status(_) => {} } } @@ -955,7 +974,7 @@ impl CloudModelProvider { .run(async move { let mut response = Self::perform_llm_request( "/count_tokens", - false, + StatusMessages::Omit, &http_client, &*token_provider, auth_context, @@ -1372,6 +1391,7 @@ pub fn response_lines( #[cfg(test)] mod tests { use super::*; + use futures::TryStreamExt as _; use http_client::FakeHttpClient; use http_client::http::{HeaderMap, StatusCode}; use language_model::{ @@ -1393,6 +1413,7 @@ mod tests { json!({"event": {"candidates": [{ "content": {"role": "model", "parts": [{"text": "Hello"}]} }]}}), + json!({"status": CompletionRequestStatus::KeepAlive}), json!({"event": { "candidates": [{ "content": {"role": "model", "parts": [{"text": " world"}]}, @@ -1435,12 +1456,15 @@ mod tests { cx: &mut gpui::TestAppContext, ) { for stream_ended in [false, true] { - let mut events = vec![json!({"event": {"candidates": [{ - "content": {"role": "model", "parts": [{ - "functionCall": {"name": "list_directory", "args": {"path": "."}} - }]}, - "finishReason": "STOP" - }]}})]; + let mut events = vec![ + json!({"event": {"candidates": [{ + "content": {"role": "model", "parts": [{ + "functionCall": {"name": "list_directory", "args": {"path": "."}} + }]}, + "finishReason": "STOP" + }]}}), + json!({"status": CompletionRequestStatus::KeepAlive}), + ]; if stream_ended { events.push(json!({"status": "stream_ended"})); } @@ -1510,118 +1534,222 @@ mod tests { async fn cloud_explicit_compaction_forwards_supported_request_fields( cx: &mut gpui::TestAppContext, ) { - let captured_request = Arc::new(Mutex::new(None)); - let captured_request_for_handler = captured_request.clone(); - let http_client = FakeHttpClient::create(move |request| { - let captured_request = captured_request_for_handler.clone(); - async move { - let method = request.method().clone(); - let uri = request.uri().to_string(); - let authorization = request - .headers() - .get("Authorization") - .and_then(|value| value.to_str().ok()) - .map(str::to_string); - let requested_status_messages = request - .headers() - .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME); - let requested_stream_end = request - .headers() - .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME); - let mut body = request.into_body(); - let mut body_text = String::new(); - body.read_to_string(&mut body_text).await?; - *captured_request.lock().unwrap() = Some(( - method, - uri, - authorization, - requested_status_messages, - requested_stream_end, - body_text, - )); - - Ok(http_client::Response::builder() - .status(200) - .body(AsyncBody::from(format!( - "{}\n", - json!({ - "id": "resp_compact", - "created_at": 1_700_000_000, - "object": "response.compaction", - "output": [{ - "type": "compaction", - "id": "cmp_manual", - "encrypted_content": "opaque-state" - }], - "usage": { - "input_tokens": 100, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens": 10, - "output_tokens_details": {"reasoning_tokens": 5}, - "total_tokens": 110 - } - }) - )))?) - } - }); let language_model = language_model(&cloud_test_config()); assert!(language_model.supports_explicit_compaction()); assert!(!language_model.supports_explicit_compaction_output_limit()); + + for includes_status_messages in [false, true] { + let captured_request = Arc::new(Mutex::new(None)); + let captured_request_for_handler = captured_request.clone(); + let http_client = FakeHttpClient::create(move |request| { + let captured_request = captured_request_for_handler.clone(); + async move { + let method = request.method().clone(); + let uri = request.uri().to_string(); + let authorization = request + .headers() + .get("Authorization") + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let requested_status_messages = request + .headers() + .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME); + let requested_stream_end = request + .headers() + .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME); + let mut body = request.into_body(); + let mut body_text = String::new(); + body.read_to_string(&mut body_text).await?; + *captured_request.lock().unwrap() = Some(( + method, + uri, + authorization, + requested_status_messages, + requested_stream_end, + body_text, + )); + + let result = json!({ + "id": "resp_compact", + "created_at": 1_700_000_000, + "object": "response.compaction", + "output": [{ + "type": "compaction", + "id": "cmp_manual", + "encrypted_content": "opaque-state" + }], + "usage": { + "input_tokens": 100, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens": 10, + "output_tokens_details": {"reasoning_tokens": 5}, + "total_tokens": 110 + } + }); + let mut response = http_client::Response::builder().status(200); + let body = if includes_status_messages { + response = + response.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true"); + [ + json!({"status": "keep_alive"}), + json!({"status": "future_transport_status"}), + json!({"event": result}), + json!({"status": "stream_ended"}), + ] + .into_iter() + .map(|event| format!("{event}\n")) + .collect::() + } else { + format!("{result}\n") + }; + Ok(response.body(AsyncBody::from(body))?) + } + }); + let (provider, model) = cloud_test_model(http_client); + let request = compact_test_request(); + + let result = cx + .update(|cx| provider.compact(&model, request, cx)) + .await + .unwrap(); + + assert_eq!( + result.usage, + language_model::TokenUsage { + input_tokens: 80, + output_tokens: 10, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 20, + } + ); + let language_model::CompactedContext::ProviderState(state) = result.context else { + panic!("expected provider compaction state"); + }; + assert_eq!( + open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID) + .unwrap(), + Some(vec![json!({ + "type": "compaction", + "id": "cmp_manual", + "encrypted_content": "opaque-state" + })]) + ); + let (method, uri, authorization, requested_status_messages, requested_stream_end, body) = + captured_request.lock().unwrap().take().unwrap(); + assert_eq!(method, Method::POST); + assert_eq!(uri, "http://test.example/completions/compact?"); + assert_eq!(authorization.as_deref(), Some("Bearer test-token")); + assert!(requested_status_messages); + assert!(requested_stream_end); + let body = serde_json::from_str::(&body).unwrap(); + assert_eq!(body["thread_id"], "thread-123"); + assert_eq!(body["provider"], "open_ai"); + assert_eq!(body["model"], "gpt-5.4"); + assert_eq!( + body["provider_request"], + json!({ + "model": "gpt-5.4", + "input": [{ + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": "Retain this context." + }] + }], + "prompt_cache_key": "thread-123", + "service_tier": "priority" + }) + ); + } + } + + #[gpui::test] + async fn cloud_explicit_compaction_preserves_streamed_failure(cx: &mut gpui::TestAppContext) { + let http_client = FakeHttpClient::create(|_| async move { + Ok(Response::builder() + .status(200) + .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true") + .body(AsyncBody::from(format!( + "{}\n{}\n", + json!({"status": "keep_alive"}), + json!({"status": {"failed": { + "code": "upstream_http_429", + "message": "Compaction rate limit exceeded.", + "request_id": "00000000-0000-0000-0000-000000000000", + "retry_after": 1.5 + }}}), + )))?) + }); let (provider, model) = cloud_test_model(http_client); - let request = compact_test_request(); - let result = cx - .update(|cx| provider.compact(&model, request, cx)) + let error = cx + .update(|cx| provider.compact(&model, compact_test_request(), cx)) .await - .unwrap(); + .unwrap_err(); - assert_eq!( - result.usage, - language_model::TokenUsage { - input_tokens: 80, - output_tokens: 10, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 20, - } - ); - let language_model::CompactedContext::ProviderState(state) = result.context else { - panic!("expected provider compaction state"); + let LanguageModelCompletionError::ProviderRejection { + provider, + status, + code, + message, + retry_after, + category, + } = error + else { + panic!("expected the streamed provider rejection, got {error:?}"); }; - assert_eq!( - open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID).unwrap(), - Some(vec![json!({ - "type": "compaction", - "id": "cmp_manual", - "encrypted_content": "opaque-state" - })]) - ); - let (method, uri, authorization, requested_status_messages, requested_stream_end, body) = - captured_request.lock().unwrap().take().unwrap(); - assert_eq!(method, Method::POST); - assert_eq!(uri, "http://test.example/completions/compact?"); - assert_eq!(authorization.as_deref(), Some("Bearer test-token")); - assert!(!requested_status_messages); - assert!(!requested_stream_end); - let body = serde_json::from_str::(&body).unwrap(); - assert_eq!(body["thread_id"], "thread-123"); - assert_eq!(body["provider"], "open_ai"); - assert_eq!(body["model"], "gpt-5.4"); - assert_eq!( - body["provider_request"], - json!({ - "model": "gpt-5.4", - "input": [{ - "type": "message", - "role": "user", - "content": [{ - "type": "input_text", - "text": "Retain this context." - }] - }], - "prompt_cache_key": "thread-123", - "service_tier": "priority" - }) - ); + assert_eq!(provider, OPEN_AI_PROVIDER_NAME); + assert_eq!(status, Some(StatusCode::TOO_MANY_REQUESTS)); + assert_eq!(code.as_deref(), Some("upstream_http_429")); + assert_eq!(message, "Compaction rate limit exceeded."); + assert_eq!(retry_after, Some(Duration::from_secs_f64(1.5))); + assert_eq!(category, ProviderErrorCategory::RateLimit); + } + + #[gpui::test] + async fn cloud_explicit_compaction_rejects_stream_ending_without_result( + cx: &mut gpui::TestAppContext, + ) { + for stream_ended in [false, true] { + let http_client = FakeHttpClient::create(move |_| async move { + let mut body = format!("{}\n", json!({"status": "keep_alive"})); + if stream_ended { + body.push_str(&format!("{}\n", json!({"status": "stream_ended"}))); + } + let body = stream::iter([Ok::<_, std::io::Error>(body.into_bytes())]) + .chain(stream::poll_fn(move |_| { + if stream_ended { + Poll::Pending + } else { + Poll::Ready(None) + } + })) + .into_async_read(); + Ok(Response::builder() + .status(200) + .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true") + .body(AsyncBody::from_reader(body))?) + }); + let (provider, model) = cloud_test_model(http_client); + + let error = cx + .update(|cx| provider.compact(&model, compact_test_request(), cx)) + .now_or_never() + .expect( + "compaction should stop at EOF or StreamEnded without waiting for more bytes", + ) + .unwrap_err(); + + assert!( + matches!( + &error, + LanguageModelCompletionError::StreamEndedUnexpectedly { provider } + if provider == &OPEN_AI_PROVIDER_NAME + ), + "expected a missing compaction result error, got {error:?}" + ); + } } #[gpui::test]