diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index 0542747c0c0..def85704efa 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -164,8 +164,16 @@ impl OpenAiLanguageModelProvider { request_limiter: &RateLimiter, request: open_ai::Request, cx: &AsyncApp, - ) -> BoxFuture<'static, Result>>> - { + ) -> BoxFuture< + 'static, + Result< + futures::stream::BoxStream< + 'static, + Result, + >, + LanguageModelCompletionError, + >, + > { let http_client = self.http_client.clone(); let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| { diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs index b18388cc03c..971d5b4b1fa 100644 --- a/crates/language_models/src/provider/open_ai_compatible.rs +++ b/crates/language_models/src/provider/open_ai_compatible.rs @@ -137,7 +137,10 @@ impl OpenAiCompatibleLanguageModelProvider { ) -> BoxFuture< 'static, Result< - futures::stream::BoxStream<'static, Result>, + futures::stream::BoxStream< + 'static, + Result, + >, LanguageModelCompletionError, >, > { diff --git a/crates/language_models/src/provider/vercel_ai_gateway.rs b/crates/language_models/src/provider/vercel_ai_gateway.rs index 8e23f16a2a9..c08228e2041 100644 --- a/crates/language_models/src/provider/vercel_ai_gateway.rs +++ b/crates/language_models/src/provider/vercel_ai_gateway.rs @@ -233,7 +233,10 @@ impl VercelAiGatewayLanguageModelProvider { ) -> BoxFuture< 'static, Result< - futures::stream::BoxStream<'static, Result>, + futures::stream::BoxStream< + 'static, + Result, + >, LanguageModelCompletionError, >, > { diff --git a/crates/language_models/src/provider/x_ai.rs b/crates/language_models/src/provider/x_ai.rs index 167501c7e62..087d201aeeb 100644 --- a/crates/language_models/src/provider/x_ai.rs +++ b/crates/language_models/src/provider/x_ai.rs @@ -154,7 +154,10 @@ impl XAiLanguageModelProvider { ) -> BoxFuture< 'static, Result< - futures::stream::BoxStream<'static, Result>, + futures::stream::BoxStream< + 'static, + Result, + >, LanguageModelCompletionError, >, > { diff --git a/crates/open_ai/src/chat_completion_transport_tests.rs b/crates/open_ai/src/chat_completion_transport_tests.rs index 8ee79cfd725..d0475175304 100644 --- a/crates/open_ai/src/chat_completion_transport_tests.rs +++ b/crates/open_ai/src/chat_completion_transport_tests.rs @@ -1,4 +1,6 @@ use super::*; +use anyhow::anyhow; +use futures::TryStreamExt as _; use futures::executor::block_on; use http_client::{ FakeHttpClient, Response, @@ -151,10 +153,6 @@ fn streaming_transport_preserves_usage_limit_errors() { .expect("provider error event") .expect_err("usage limit should reject the completion") }); - let error = error - .downcast_ref::() - .expect("typed provider rejection"); - assert!(!error.is_transient()); assert!(matches!( error, @@ -165,12 +163,69 @@ fn streaming_transport_preserves_usage_limit_errors() { message, category: ProviderErrorCategory::PaymentRequired, .. - } if provider == &LanguageModelProviderName::from("Compatible Provider".to_string()) + } if provider == LanguageModelProviderName::from("Compatible Provider".to_string()) && code == "usage_limit_reached" && message == "The usage limit has been reached" )); } +#[test] +fn streaming_transport_preserves_decode_and_read_errors() { + let bodies = [ + (AsyncBody::from("data: not JSON\n\n"), "decode"), + (AsyncBody::from("data: {\"choices\":false}\n\n"), "decode"), + ( + AsyncBody::from_reader( + futures::stream::iter([Err::, _>(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "connection reset", + ))]) + .into_async_read(), + ), + "read", + ), + ]; + for (body, expected_kind) in bodies { + let body = Mutex::new(Some(body)); + let client = FakeHttpClient::create(move |_| { + let body = body.lock().expect("body lock").take().expect("one request"); + async move { Ok(Response::builder().status(200).body(body)?) } + }); + let request = serde_json::from_value(json!({ + "model": "custom/model", + "messages": [], + "stream": true + })) + .expect("valid chat completion request"); + let error = block_on(async { + stream_completion( + client.as_ref(), + "Compatible Provider", + "https://example.com/v1", + "secret", + request, + &CustomHeaders::default(), + ) + .await + .expect("streaming request") + .next() + .await + .expect("stream error event") + .expect_err("invalid response should fail") + }); + let kind = match error { + LanguageModelCompletionError::DeserializeResponse { .. } => "decode", + LanguageModelCompletionError::ApiReadResponseError { error, .. } + if error.kind() == std::io::ErrorKind::ConnectionReset => + { + "read" + } + error => panic!("unexpected stream error: {error:?}"), + }; + assert_eq!(kind, expected_kind); + } +} + #[test] fn transport_preserves_typed_send_and_deserialization_errors() { let client = FakeHttpClient::create(|_| async move { Err(anyhow!("network unavailable")) }); diff --git a/crates/open_ai/src/open_ai.rs b/crates/open_ai/src/open_ai.rs index 49d913878e7..7ce79a82f23 100644 --- a/crates/open_ai/src/open_ai.rs +++ b/crates/open_ai/src/open_ai.rs @@ -5,13 +5,14 @@ pub mod responses; #[cfg(test)] mod chat_completion_transport_tests; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result}; use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream}; use http_client::{ AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt, StatusCode, http::{HeaderMap, HeaderValue}, }; +use language_model_core::LanguageModelCompletionError; pub use language_model_core::ReasoningEffort; pub use language_model_core::chat_completion::{ ChoiceDelta, FunctionChunk, PromptTokensDetails, ResponseMessageDelta, ResponseStreamError, @@ -1140,7 +1141,10 @@ pub async fn stream_completion( api_key: &str, request: Request, extra_headers: &CustomHeaders, -) -> Result>, RequestError> { +) -> Result< + BoxStream<'static, Result>, + RequestError, +> { let events = stream_chat_completion( client, provider_name, @@ -1159,7 +1163,7 @@ pub async fn stream_completion( let value = match event { Ok(ChatCompletionStreamEvent::Data(value)) => value, Ok(ChatCompletionStreamEvent::Done) => return None, - Err(error) => return Some(Err(anyhow!(error))), + Err(error) => return Some(Err(error.into())), }; match serde_json::from_str::(value.get()) { Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)), @@ -1170,15 +1174,13 @@ pub async fn stream_completion( None, &error.message, ); - Some(Err(anyhow!( - language_model_core::LanguageModelCompletionError::from_provider_response( - provider_name, - None, - error.code.or(error.error_type), - error.message, - None, - category, - ) + Some(Err(LanguageModelCompletionError::from_provider_response( + provider_name, + None, + error.code.or(error.error_type), + error.message, + None, + category, ))) } Err(error) => { @@ -1188,7 +1190,10 @@ pub async fn stream_completion( error, value, ); - Some(Err(anyhow!(error))) + Some(Err(LanguageModelCompletionError::DeserializeResponse { + provider: provider_name, + error, + })) } } } diff --git a/crates/x_ai_subscribed/src/x_ai_subscribed.rs b/crates/x_ai_subscribed/src/x_ai_subscribed.rs index 2c7604daf63..894072c8563 100644 --- a/crates/x_ai_subscribed/src/x_ai_subscribed.rs +++ b/crates/x_ai_subscribed/src/x_ai_subscribed.rs @@ -428,7 +428,10 @@ fn stream_open_ai_completion( ) -> BoxFuture< 'static, Result< - futures::stream::BoxStream<'static, Result>, + futures::stream::BoxStream< + 'static, + Result, + >, LanguageModelCompletionError, >, > {