From 574df448ea9cb9c22df1a5e887a4c6755e7e09ce Mon Sep 17 00:00:00 2001 From: Bennet Bo Fenner Date: Thu, 22 Oct 2026 13:57:27 +0000 Subject: [PATCH] Preserve transport errors in Google and OpenAI Responses streams (#65940) Preserve typed transport errors in Google AI and OpenAI Responses completion streams instead of converting mid-stream read failures into generic, non-retryable model errors. - Classify streaming body-read failures separately from malformed JSON, preserving the provider name and original error. - Carry typed errors through the completion mappers and direct/OpenAI-compatible callers. Keep the OpenAI Responses mapper compatible with integrations that still supply generic errors. - Apply the same classification to OpenCode's Google-protocol stream. - Leave timeout durations and retry policies unchanged. Release Notes: - Fixed streaming connection failures being classified as non-retryable errors for Google AI and OpenAI Responses model requests. --- crates/google_ai/Cargo.toml | 3 + crates/google_ai/src/completion.rs | 32 +---- crates/google_ai/src/google_ai.rs | 89 ++++++++++++- crates/language_models/src/provider/google.rs | 5 +- .../language_models/src/provider/open_ai.rs | 11 +- .../src/provider/open_ai_compatible.rs | 11 +- crates/open_ai/src/completion.rs | 13 +- crates/open_ai/src/responses.rs | 120 +++++++++++++++--- crates/opencode/src/opencode.rs | 18 ++- 9 files changed, 231 insertions(+), 71 deletions(-) diff --git a/crates/google_ai/Cargo.toml b/crates/google_ai/Cargo.toml index bd95a8e5346..b27e225a1ba 100644 --- a/crates/google_ai/Cargo.toml +++ b/crates/google_ai/Cargo.toml @@ -20,3 +20,6 @@ language_model_core.workspace = true log.workspace = true serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +http_client = { workspace = true, features = ["test-support"] } diff --git a/crates/google_ai/src/completion.rs b/crates/google_ai/src/completion.rs index 6c87161c500..0bc68b828ce 100644 --- a/crates/google_ai/src/completion.rs +++ b/crates/google_ai/src/completion.rs @@ -1,12 +1,11 @@ use anyhow::Result; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{Stream, StreamExt, TryStreamExt, stream::BoxStream}; use language_model_core::{ GOOGLE_PROVIDER_NAME, LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelRequest, LanguageModelRequestToolInput, LanguageModelToolChoice, LanguageModelToolUse, LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role, StopReason, TokenUsage, }; -use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{self, AtomicU64}; @@ -268,13 +267,13 @@ impl GoogleEventMapper { pub fn map_stream( self, - events: Pin>>>, + events: BoxStream<'static, Result>, ) -> impl Stream> { futures::stream::try_unfold((self, events), |(mut mapper, mut events)| async move { match events.next().await { Some(event) => { - let event = event.map_err(LanguageModelCompletionError::from)?; + let event = event?; Ok(Some(( futures::stream::iter(mapper.map_event(event)), (mapper, events), @@ -564,31 +563,6 @@ mod tests { } } - #[test] - fn stream_read_error_is_not_followed_by_completion() { - let responses = vec![ - Ok(serde_json::from_value(json!({"candidates": [{ - "content": {"role": "model", "parts": [{"text": "Partial answer"}]} - }]})) - .unwrap()), - Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "read failed").into()), - ]; - let events = futures::executor::block_on( - GoogleEventMapper::new() - .map_stream(Box::pin(futures::stream::iter(responses))) - .collect::>(), - ); - assert!(matches!( - events.as_slice(), - [ - Ok(LanguageModelCompletionEvent::Text(text)), - Err(LanguageModelCompletionError::Other(error)), - ] if text == "Partial answer" - && error.downcast_ref::() - .is_some_and(|error| error.kind() == std::io::ErrorKind::ConnectionReset) - )); - } - fn text_request() -> LanguageModelRequest { LanguageModelRequest { messages: vec![LanguageModelRequestMessage { diff --git a/crates/google_ai/src/google_ai.rs b/crates/google_ai/src/google_ai.rs index 22bb331ceb0..f909f7a6a03 100644 --- a/crates/google_ai/src/google_ai.rs +++ b/crates/google_ai/src/google_ai.rs @@ -1,6 +1,6 @@ use std::mem; -use anyhow::{Result, anyhow, bail}; +use anyhow::{Result, bail}; use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream}; use http_client::{ AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt, @@ -198,7 +198,10 @@ pub async fn stream_generate_content( api_key: &str, mut request: GenerateContentRequest, extra_headers: &CustomHeaders, -) -> Result>, LanguageModelCompletionError> { +) -> Result< + BoxStream<'static, Result>, + LanguageModelCompletionError, +> { let api_key = api_key.trim(); validate_generate_content_request(&request)?; @@ -251,15 +254,21 @@ pub async fn stream_generate_content( } match serde_json::from_str(line) { Ok(response) => Some(Ok(response)), - Err(error) => Some(Err(anyhow!(format!( - "Error parsing JSON: {error:?}\n{line:?}" - )))), + Err(error) => { + Some(Err(LanguageModelCompletionError::DeserializeResponse { + provider: GOOGLE_PROVIDER_NAME, + error, + })) + } } } else { None } } - Err(error) => Some(Err(anyhow!(error))), + Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError { + provider: GOOGLE_PROVIDER_NAME, + error, + })), } }) .boxed()) @@ -956,7 +965,75 @@ struct ListModelEntry { #[cfg(test)] mod tests { use super::*; + use completion::GoogleEventMapper; + use futures::{TryStreamExt, stream}; + use http_client::{FakeHttpClient, Response}; + use language_model_core::LanguageModelCompletionEvent; use serde_json::json; + use std::io::{Error, ErrorKind}; + + #[test] + fn stream_body_failures_remain_retryable() { + for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] { + let error = completion_error(Some(kind)); + assert!(error.is_transient()); + assert!(matches!( + error, + LanguageModelCompletionError::ApiReadResponseError { provider, error } + if provider == GOOGLE_PROVIDER_NAME && error.kind() == kind + )); + } + } + + #[test] + fn malformed_stream_data_remains_non_retryable() { + let error = completion_error(None); + assert!(!error.is_transient()); + assert!(matches!( + error, + LanguageModelCompletionError::DeserializeResponse { provider, .. } + if provider == GOOGLE_PROVIDER_NAME + )); + } + + fn completion_error(read_error: Option) -> LanguageModelCompletionError { + futures::executor::block_on(async { + let client = FakeHttpClient::create(move |_| async move { + let chunks = vec![ + Ok(b"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Thinking\"}]}}]}\n\n".to_vec()), + match read_error { + Some(kind) => Err(Error::new(kind, "stream read failed")), + None => Ok(b"data: not json\n\n".to_vec()), + }, + ]; + Ok(Response::builder().body(AsyncBody::from_reader( + stream::iter(chunks).into_async_read(), + ))?) + }); + let request = serde_json::from_value(json!({ + "model": "models/gemini-3.8-flash", + "contents": [{"role": "user", "parts": [{"text": "Think"}]}] + })) + .unwrap(); + let responses = stream_generate_content( + client.as_ref(), + API_URL, + "test-key", + request, + &CustomHeaders::default(), + ) + .await + .unwrap(); + let mut events = GoogleEventMapper::new().map_stream(responses).boxed(); + assert!(matches!( + events.next().await, + Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Thinking" + )); + let error = events.next().await.unwrap().unwrap_err(); + assert!(events.next().await.is_none()); + error + }) + } #[test] fn test_lowest_thinking_level_per_model() { diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rs index 9d1eccb7ca1..e202f00e07b 100644 --- a/crates/language_models/src/provider/google.rs +++ b/crates/language_models/src/provider/google.rs @@ -246,7 +246,10 @@ impl GoogleLanguageModelProvider { ) -> BoxFuture< 'static, Result< - futures::stream::BoxStream<'static, Result>, + futures::stream::BoxStream< + 'static, + Result, + >, LanguageModelCompletionError, >, > { diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index c6d429d4a1c..0542747c0c0 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -201,8 +201,15 @@ impl OpenAiLanguageModelProvider { request_limiter: &RateLimiter, request: ResponseRequest, cx: &AsyncApp, - ) -> BoxFuture<'static, Result>>> - { + ) -> BoxFuture< + 'static, + Result< + futures::stream::BoxStream< + 'static, + Result, + >, + >, + > { let http_client = self.http_client.clone(); let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| { diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs index 6ffb6520d07..b18388cc03c 100644 --- a/crates/language_models/src/provider/open_ai_compatible.rs +++ b/crates/language_models/src/provider/open_ai_compatible.rs @@ -177,8 +177,15 @@ impl OpenAiCompatibleLanguageModelProvider { request_limiter: &RateLimiter, request: ResponseRequest, cx: &AsyncApp, - ) -> BoxFuture<'static, Result>>> - { + ) -> BoxFuture< + 'static, + Result< + futures::stream::BoxStream< + 'static, + Result, + >, + >, + > { let http_client = self.http_client.clone(); let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, _cx| { diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs index 69bb669ecba..e4d10fb5c7c 100644 --- a/crates/open_ai/src/completion.rs +++ b/crates/open_ai/src/completion.rs @@ -1,6 +1,6 @@ use anyhow::{Result, anyhow}; use collections::HashMap; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, stream::BoxStream}; use http_client::StatusCode; use language_model_core::{ CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent, @@ -11,7 +11,6 @@ use language_model_core::{ ProviderErrorCategory, Role, StopReason, TokenUsage, provider_name_for_id, util::{fix_streamed_json, parse_tool_arguments}, }; -use std::pin::Pin; use std::sync::Arc; use crate::responses::{ @@ -790,15 +789,15 @@ impl OpenAiResponseEventMapper { } } - pub fn map_stream( + pub fn map_stream + 'static>( mut self, - events: Pin>>>, + events: BoxStream<'static, Result>, ) -> impl Stream> { events.flat_map(move |event| { futures::stream::iter(match event { Ok(event) => self.map_event(event), - Err(error) => vec![Err(LanguageModelCompletionError::from(anyhow!(error)))], + Err(error) => vec![Err(error.into())], }) }) } @@ -1592,7 +1591,9 @@ mod tests { fn map_response_events(events: Vec) -> Vec { block_on(async { OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID) - .map_stream(Box::pin(futures::stream::iter(events.into_iter().map(Ok)))) + .map_stream(Box::pin(futures::stream::iter( + events.into_iter().map(Ok::<_, crate::RequestError>), + ))) .collect::>() .await .into_iter() diff --git a/crates/open_ai/src/responses.rs b/crates/open_ai/src/responses.rs index 4037b40b72f..cc2293f205c 100644 --- a/crates/open_ai/src/responses.rs +++ b/crates/open_ai/src/responses.rs @@ -883,7 +883,7 @@ pub async fn stream_response( api_key: &str, request: Request, extra_headers: &CustomHeaders, -) -> Result>, RequestError> { +) -> Result>, RequestError> { let uri = format!("{api_url}/responses"); let is_streaming = request.stream; let request = HttpRequest::builder() @@ -909,31 +909,41 @@ pub async fn stream_response( if response.status().is_success() { if is_streaming { let reader = BufReader::new(response.into_body()); + let provider_name = provider_name.to_owned(); Ok(reader .lines() - .filter_map(|line| async move { - match line { - Ok(line) => { - let line = line - .strip_prefix("data: ") - .or_else(|| line.strip_prefix("data:"))?; - if line == "[DONE]" || line.is_empty() { - None - } else { - match decode_stream_event(line) { - Ok(event) => Some(Ok(event)), - Err(error) => { - log::error!( - "Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`", - error, - line, - ); - Some(Err(anyhow!(error))) + .filter_map(move |line| { + let provider_name = provider_name.clone(); + async move { + match line { + Ok(line) => { + let line = line + .strip_prefix("data: ") + .or_else(|| line.strip_prefix("data:"))?; + if line == "[DONE]" || line.is_empty() { + None + } else { + match decode_stream_event(line) { + Ok(event) => Some(Ok(event)), + Err(error) => { + log::error!( + "Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`", + error, + line, + ); + Some(Err(RequestError::DeserializeResponse { + provider: provider_name, + error, + })) + } } } } + Err(error) => Some(Err(RequestError::ReadResponse { + provider: provider_name, + error, + })), } - Err(error) => Some(Err(anyhow!(error))), } }) .boxed()) @@ -1094,12 +1104,30 @@ fn decode_stream_event(line: &str) -> serde_json::Result { #[cfg(test)] mod tests { use super::*; + use crate::completion::OpenAiResponseEventMapper; use futures::executor::block_on; + use futures::{TryStreamExt, stream}; use http_client::FakeHttpClient; - use language_model_core::OPEN_AI_PROVIDER_ID; + use language_model_core::{ + LanguageModelCompletionError, LanguageModelCompletionEvent, OPEN_AI_PROVIDER_ID, + }; use serde_json::json; + use std::io::{Error, ErrorKind}; use std::sync::{Arc, Mutex}; + #[test] + fn stream_body_failures_remain_retryable() { + for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] { + let error = stream_completion_error(Some(kind)); + assert!(error.is_transient()); + assert!(matches!( + error, + LanguageModelCompletionError::ApiReadResponseError { provider, error } + if provider.0.as_ref() == "Custom Responses" && error.kind() == kind + )); + } + } + #[test] fn reasoning_content_accepts_unknown_parts() { let content = json!([ @@ -1211,6 +1239,56 @@ mod tests { } } + #[test] + fn malformed_stream_data_remains_non_retryable() { + let error = stream_completion_error(None); + assert!(!error.is_transient()); + assert!(matches!( + error, + LanguageModelCompletionError::DeserializeResponse { provider, .. } + if provider.0.as_ref() == "Custom Responses" + )); + } + + fn stream_completion_error(read_error: Option) -> LanguageModelCompletionError { + block_on(async { + let client = FakeHttpClient::create(move |_| async move { + let chunks = vec![ + Ok(b"data: {\"type\":\"response.output_text.delta\",\"item_id\":\"message\",\"output_index\":0,\"delta\":\"Partial answer\"}\n\n".to_vec()), + match read_error { + Some(kind) => Err(Error::new(kind, "stream read failed")), + None => Ok(b"data: not json\n\n".to_vec()), + }, + ]; + Ok( + http_client::Response::builder().body(AsyncBody::from_reader( + stream::iter(chunks).into_async_read(), + ))?, + ) + }); + let responses = stream_response( + client.as_ref(), + "Custom Responses", + "https://example.com/v1", + "test-key", + response_test_request(), + &CustomHeaders::default(), + ) + .await + .unwrap(); + let mut events = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID) + .map_stream(responses) + .boxed(); + assert!(matches!( + events.next().await, + Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Partial answer" + )); + let error = events.next().await.unwrap().unwrap_err(); + assert!(events.next().await.is_none()); + error + }) + } + #[test] fn count_input_tokens_preserves_structured_input_without_generation_fields() { let input = json!([{ diff --git a/crates/opencode/src/opencode.rs b/crates/opencode/src/opencode.rs index 0b995993a38..41219c30775 100644 --- a/crates/opencode/src/opencode.rs +++ b/crates/opencode/src/opencode.rs @@ -8,7 +8,9 @@ use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::B use http_client::{ AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt, }; -use language_model_core::ReasoningEffort; +use language_model_core::{ + LanguageModelCompletionError, LanguageModelProviderName, ReasoningEffort, +}; use serde::{Deserialize, Serialize}; pub mod catalog; @@ -138,7 +140,9 @@ pub async fn stream_generate_content( api_key: &str, request: google_ai::GenerateContentRequest, extra_headers: &CustomHeaders, -) -> Result>> { +) -> Result< + BoxStream<'static, Result>, +> { let api_key = api_key.trim(); let model_id = &request.model.model_id; let uri = format!("{api_url}/v1/models/{model_id}:streamGenerateContent?alt=sse"); @@ -161,14 +165,20 @@ pub async fn stream_generate_content( match serde_json::from_str(line) { Ok(response) => Some(Ok(response)), Err(error) => { - Some(Err(anyhow!("Error parsing JSON: {error:?}\n{line:?}"))) + Some(Err(LanguageModelCompletionError::DeserializeResponse { + provider: LanguageModelProviderName::new("OpenCode"), + error, + })) } } } else { None } } - Err(error) => Some(Err(anyhow!(error))), + Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError { + provider: LanguageModelProviderName::new("OpenCode"), + error, + })), } }) .boxed())