Preserve transport errors in Google and OpenAI Responses streams (#65940)

Preserve typed transport errors in Google AI and OpenAI Responses
completion streams instead of converting mid-stream read failures into
generic, non-retryable model errors.

- Classify streaming body-read failures separately from malformed JSON,
preserving the provider name and original error.
- Carry typed errors through the completion mappers and
direct/OpenAI-compatible callers. Keep the OpenAI Responses mapper
compatible with integrations that still supply generic errors.
- Apply the same classification to OpenCode's Google-protocol stream.
- Leave timeout durations and retry policies unchanged.

Release Notes:

- Fixed streaming connection failures being classified as non-retryable
errors for Google AI and OpenAI Responses model requests.
This commit is contained in:
Bennet Bo Fenner
2026-10-22 13:57:27 +00:00
committed by GitHub
parent 566f893ee3
commit 574df448ea
9 changed files with 231 additions and 71 deletions
+3
View File
@@ -20,3 +20,6 @@ language_model_core.workspace = true
log.workspace = true
serde.workspace = true
serde_json.workspace = true
[dev-dependencies]
http_client = { workspace = true, features = ["test-support"] }
+3 -29
View File
@@ -1,12 +1,11 @@
use anyhow::Result;
use futures::{Stream, StreamExt, TryStreamExt};
use futures::{Stream, StreamExt, TryStreamExt, stream::BoxStream};
use language_model_core::{
GOOGLE_PROVIDER_NAME, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelRequest, LanguageModelRequestToolInput, LanguageModelToolChoice,
LanguageModelToolUse, LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role,
StopReason, TokenUsage,
};
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{self, AtomicU64};
@@ -268,13 +267,13 @@ impl GoogleEventMapper {
pub fn map_stream(
self,
events: Pin<Box<dyn Send + Stream<Item = Result<GenerateContentResponse>>>>,
events: BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
{
futures::stream::try_unfold((self, events), |(mut mapper, mut events)| async move {
match events.next().await {
Some(event) => {
let event = event.map_err(LanguageModelCompletionError::from)?;
let event = event?;
Ok(Some((
futures::stream::iter(mapper.map_event(event)),
(mapper, events),
@@ -564,31 +563,6 @@ mod tests {
}
}
#[test]
fn stream_read_error_is_not_followed_by_completion() {
let responses = vec![
Ok(serde_json::from_value(json!({"candidates": [{
"content": {"role": "model", "parts": [{"text": "Partial answer"}]}
}]}))
.unwrap()),
Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "read failed").into()),
];
let events = futures::executor::block_on(
GoogleEventMapper::new()
.map_stream(Box::pin(futures::stream::iter(responses)))
.collect::<Vec<_>>(),
);
assert!(matches!(
events.as_slice(),
[
Ok(LanguageModelCompletionEvent::Text(text)),
Err(LanguageModelCompletionError::Other(error)),
] if text == "Partial answer"
&& error.downcast_ref::<std::io::Error>()
.is_some_and(|error| error.kind() == std::io::ErrorKind::ConnectionReset)
));
}
fn text_request() -> LanguageModelRequest {
LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
+83 -6
View File
@@ -1,6 +1,6 @@
use std::mem;
use anyhow::{Result, anyhow, bail};
use anyhow::{Result, bail};
use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
@@ -198,7 +198,10 @@ pub async fn stream_generate_content(
api_key: &str,
mut request: GenerateContentRequest,
extra_headers: &CustomHeaders,
) -> Result<BoxStream<'static, Result<GenerateContentResponse>>, LanguageModelCompletionError> {
) -> Result<
BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
LanguageModelCompletionError,
> {
let api_key = api_key.trim();
validate_generate_content_request(&request)?;
@@ -251,15 +254,21 @@ pub async fn stream_generate_content(
}
match serde_json::from_str(line) {
Ok(response) => Some(Ok(response)),
Err(error) => Some(Err(anyhow!(format!(
"Error parsing JSON: {error:?}\n{line:?}"
)))),
Err(error) => {
Some(Err(LanguageModelCompletionError::DeserializeResponse {
provider: GOOGLE_PROVIDER_NAME,
error,
}))
}
}
} else {
None
}
}
Err(error) => Some(Err(anyhow!(error))),
Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
provider: GOOGLE_PROVIDER_NAME,
error,
})),
}
})
.boxed())
@@ -956,7 +965,75 @@ struct ListModelEntry {
#[cfg(test)]
mod tests {
use super::*;
use completion::GoogleEventMapper;
use futures::{TryStreamExt, stream};
use http_client::{FakeHttpClient, Response};
use language_model_core::LanguageModelCompletionEvent;
use serde_json::json;
use std::io::{Error, ErrorKind};
#[test]
fn stream_body_failures_remain_retryable() {
for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
let error = completion_error(Some(kind));
assert!(error.is_transient());
assert!(matches!(
error,
LanguageModelCompletionError::ApiReadResponseError { provider, error }
if provider == GOOGLE_PROVIDER_NAME && error.kind() == kind
));
}
}
#[test]
fn malformed_stream_data_remains_non_retryable() {
let error = completion_error(None);
assert!(!error.is_transient());
assert!(matches!(
error,
LanguageModelCompletionError::DeserializeResponse { provider, .. }
if provider == GOOGLE_PROVIDER_NAME
));
}
fn completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
futures::executor::block_on(async {
let client = FakeHttpClient::create(move |_| async move {
let chunks = vec![
Ok(b"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Thinking\"}]}}]}\n\n".to_vec()),
match read_error {
Some(kind) => Err(Error::new(kind, "stream read failed")),
None => Ok(b"data: not json\n\n".to_vec()),
},
];
Ok(Response::builder().body(AsyncBody::from_reader(
stream::iter(chunks).into_async_read(),
))?)
});
let request = serde_json::from_value(json!({
"model": "models/gemini-3.8-flash",
"contents": [{"role": "user", "parts": [{"text": "Think"}]}]
}))
.unwrap();
let responses = stream_generate_content(
client.as_ref(),
API_URL,
"test-key",
request,
&CustomHeaders::default(),
)
.await
.unwrap();
let mut events = GoogleEventMapper::new().map_stream(responses).boxed();
assert!(matches!(
events.next().await,
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Thinking"
));
let error = events.next().await.unwrap().unwrap_err();
assert!(events.next().await.is_none());
error
})
}
#[test]
fn test_lowest_thinking_level_per_model() {
@@ -246,7 +246,10 @@ impl GoogleLanguageModelProvider {
) -> BoxFuture<
'static,
Result<
futures::stream::BoxStream<'static, Result<GenerateContentResponse>>,
futures::stream::BoxStream<
'static,
Result<GenerateContentResponse, LanguageModelCompletionError>,
>,
LanguageModelCompletionError,
>,
> {
@@ -201,8 +201,15 @@ impl OpenAiLanguageModelProvider {
request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
{
) -> BoxFuture<
'static,
Result<
futures::stream::BoxStream<
'static,
Result<ResponsesStreamEvent, open_ai::RequestError>,
>,
>,
> {
let http_client = self.http_client.clone();
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
@@ -177,8 +177,15 @@ impl OpenAiCompatibleLanguageModelProvider {
request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
{
) -> BoxFuture<
'static,
Result<
futures::stream::BoxStream<
'static,
Result<ResponsesStreamEvent, open_ai::RequestError>,
>,
>,
> {
let http_client = self.http_client.clone();
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, _cx| {
+7 -6
View File
@@ -1,6 +1,6 @@
use anyhow::{Result, anyhow};
use collections::HashMap;
use futures::{Stream, StreamExt};
use futures::{Stream, StreamExt, stream::BoxStream};
use http_client::StatusCode;
use language_model_core::{
CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent,
@@ -11,7 +11,6 @@ use language_model_core::{
ProviderErrorCategory, Role, StopReason, TokenUsage, provider_name_for_id,
util::{fix_streamed_json, parse_tool_arguments},
};
use std::pin::Pin;
use std::sync::Arc;
use crate::responses::{
@@ -790,15 +789,15 @@ impl OpenAiResponseEventMapper {
}
}
pub fn map_stream(
pub fn map_stream<E: Into<LanguageModelCompletionError> + 'static>(
mut self,
events: Pin<Box<dyn Send + Stream<Item = Result<ResponsesStreamEvent>>>>,
events: BoxStream<'static, Result<ResponsesStreamEvent, E>>,
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
{
events.flat_map(move |event| {
futures::stream::iter(match event {
Ok(event) => self.map_event(event),
Err(error) => vec![Err(LanguageModelCompletionError::from(anyhow!(error)))],
Err(error) => vec![Err(error.into())],
})
})
}
@@ -1592,7 +1591,9 @@ mod tests {
fn map_response_events(events: Vec<ResponsesStreamEvent>) -> Vec<LanguageModelCompletionEvent> {
block_on(async {
OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
.map_stream(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
.map_stream(Box::pin(futures::stream::iter(
events.into_iter().map(Ok::<_, crate::RequestError>),
)))
.collect::<Vec<_>>()
.await
.into_iter()
+99 -21
View File
@@ -883,7 +883,7 @@ pub async fn stream_response(
api_key: &str,
request: Request,
extra_headers: &CustomHeaders,
) -> Result<BoxStream<'static, Result<StreamEvent>>, RequestError> {
) -> Result<BoxStream<'static, Result<StreamEvent, RequestError>>, RequestError> {
let uri = format!("{api_url}/responses");
let is_streaming = request.stream;
let request = HttpRequest::builder()
@@ -909,31 +909,41 @@ pub async fn stream_response(
if response.status().is_success() {
if is_streaming {
let reader = BufReader::new(response.into_body());
let provider_name = provider_name.to_owned();
Ok(reader
.lines()
.filter_map(|line| async move {
match line {
Ok(line) => {
let line = line
.strip_prefix("data: ")
.or_else(|| line.strip_prefix("data:"))?;
if line == "[DONE]" || line.is_empty() {
None
} else {
match decode_stream_event(line) {
Ok(event) => Some(Ok(event)),
Err(error) => {
log::error!(
"Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
error,
line,
);
Some(Err(anyhow!(error)))
.filter_map(move |line| {
let provider_name = provider_name.clone();
async move {
match line {
Ok(line) => {
let line = line
.strip_prefix("data: ")
.or_else(|| line.strip_prefix("data:"))?;
if line == "[DONE]" || line.is_empty() {
None
} else {
match decode_stream_event(line) {
Ok(event) => Some(Ok(event)),
Err(error) => {
log::error!(
"Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
error,
line,
);
Some(Err(RequestError::DeserializeResponse {
provider: provider_name,
error,
}))
}
}
}
}
Err(error) => Some(Err(RequestError::ReadResponse {
provider: provider_name,
error,
})),
}
Err(error) => Some(Err(anyhow!(error))),
}
})
.boxed())
@@ -1094,12 +1104,30 @@ fn decode_stream_event(line: &str) -> serde_json::Result<StreamEvent> {
#[cfg(test)]
mod tests {
use super::*;
use crate::completion::OpenAiResponseEventMapper;
use futures::executor::block_on;
use futures::{TryStreamExt, stream};
use http_client::FakeHttpClient;
use language_model_core::OPEN_AI_PROVIDER_ID;
use language_model_core::{
LanguageModelCompletionError, LanguageModelCompletionEvent, OPEN_AI_PROVIDER_ID,
};
use serde_json::json;
use std::io::{Error, ErrorKind};
use std::sync::{Arc, Mutex};
#[test]
fn stream_body_failures_remain_retryable() {
for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
let error = stream_completion_error(Some(kind));
assert!(error.is_transient());
assert!(matches!(
error,
LanguageModelCompletionError::ApiReadResponseError { provider, error }
if provider.0.as_ref() == "Custom Responses" && error.kind() == kind
));
}
}
#[test]
fn reasoning_content_accepts_unknown_parts() {
let content = json!([
@@ -1211,6 +1239,56 @@ mod tests {
}
}
#[test]
fn malformed_stream_data_remains_non_retryable() {
let error = stream_completion_error(None);
assert!(!error.is_transient());
assert!(matches!(
error,
LanguageModelCompletionError::DeserializeResponse { provider, .. }
if provider.0.as_ref() == "Custom Responses"
));
}
fn stream_completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
block_on(async {
let client = FakeHttpClient::create(move |_| async move {
let chunks = vec![
Ok(b"data: {\"type\":\"response.output_text.delta\",\"item_id\":\"message\",\"output_index\":0,\"delta\":\"Partial answer\"}\n\n".to_vec()),
match read_error {
Some(kind) => Err(Error::new(kind, "stream read failed")),
None => Ok(b"data: not json\n\n".to_vec()),
},
];
Ok(
http_client::Response::builder().body(AsyncBody::from_reader(
stream::iter(chunks).into_async_read(),
))?,
)
});
let responses = stream_response(
client.as_ref(),
"Custom Responses",
"https://example.com/v1",
"test-key",
response_test_request(),
&CustomHeaders::default(),
)
.await
.unwrap();
let mut events = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
.map_stream(responses)
.boxed();
assert!(matches!(
events.next().await,
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Partial answer"
));
let error = events.next().await.unwrap().unwrap_err();
assert!(events.next().await.is_none());
error
})
}
#[test]
fn count_input_tokens_preserves_structured_input_without_generation_fields() {
let input = json!([{
+14 -4
View File
@@ -8,7 +8,9 @@ use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::B
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
};
use language_model_core::ReasoningEffort;
use language_model_core::{
LanguageModelCompletionError, LanguageModelProviderName, ReasoningEffort,
};
use serde::{Deserialize, Serialize};
pub mod catalog;
@@ -138,7 +140,9 @@ pub async fn stream_generate_content(
api_key: &str,
request: google_ai::GenerateContentRequest,
extra_headers: &CustomHeaders,
) -> Result<BoxStream<'static, Result<google_ai::GenerateContentResponse>>> {
) -> Result<
BoxStream<'static, Result<google_ai::GenerateContentResponse, LanguageModelCompletionError>>,
> {
let api_key = api_key.trim();
let model_id = &request.model.model_id;
let uri = format!("{api_url}/v1/models/{model_id}:streamGenerateContent?alt=sse");
@@ -161,14 +165,20 @@ pub async fn stream_generate_content(
match serde_json::from_str(line) {
Ok(response) => Some(Ok(response)),
Err(error) => {
Some(Err(anyhow!("Error parsing JSON: {error:?}\n{line:?}")))
Some(Err(LanguageModelCompletionError::DeserializeResponse {
provider: LanguageModelProviderName::new("OpenCode"),
error,
}))
}
}
} else {
None
}
}
Err(error) => Some(Err(anyhow!(error))),
Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
provider: LanguageModelProviderName::new("OpenCode"),
error,
})),
}
})
.boxed())