mirror of
https://github.com/zed-industries/zed.git
synced 2026-11-19 06:59:22 +01:00
Preserve transport errors in Google and OpenAI Responses streams (#65940)
Preserve typed transport errors in Google AI and OpenAI Responses completion streams instead of converting mid-stream read failures into generic, non-retryable model errors. - Classify streaming body-read failures separately from malformed JSON, preserving the provider name and original error. - Carry typed errors through the completion mappers and direct/OpenAI-compatible callers. Keep the OpenAI Responses mapper compatible with integrations that still supply generic errors. - Apply the same classification to OpenCode's Google-protocol stream. - Leave timeout durations and retry policies unchanged. Release Notes: - Fixed streaming connection failures being classified as non-retryable errors for Google AI and OpenAI Responses model requests.
This commit is contained in:
@@ -20,3 +20,6 @@ language_model_core.workspace = true
|
||||
log.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
http_client = { workspace = true, features = ["test-support"] }
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
use anyhow::Result;
|
||||
use futures::{Stream, StreamExt, TryStreamExt};
|
||||
use futures::{Stream, StreamExt, TryStreamExt, stream::BoxStream};
|
||||
use language_model_core::{
|
||||
GOOGLE_PROVIDER_NAME, LanguageModelCompletionError, LanguageModelCompletionEvent,
|
||||
LanguageModelRequest, LanguageModelRequestToolInput, LanguageModelToolChoice,
|
||||
LanguageModelToolUse, LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role,
|
||||
StopReason, TokenUsage,
|
||||
};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{self, AtomicU64};
|
||||
|
||||
@@ -268,13 +267,13 @@ impl GoogleEventMapper {
|
||||
|
||||
pub fn map_stream(
|
||||
self,
|
||||
events: Pin<Box<dyn Send + Stream<Item = Result<GenerateContentResponse>>>>,
|
||||
events: BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
|
||||
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
|
||||
{
|
||||
futures::stream::try_unfold((self, events), |(mut mapper, mut events)| async move {
|
||||
match events.next().await {
|
||||
Some(event) => {
|
||||
let event = event.map_err(LanguageModelCompletionError::from)?;
|
||||
let event = event?;
|
||||
Ok(Some((
|
||||
futures::stream::iter(mapper.map_event(event)),
|
||||
(mapper, events),
|
||||
@@ -564,31 +563,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_read_error_is_not_followed_by_completion() {
|
||||
let responses = vec![
|
||||
Ok(serde_json::from_value(json!({"candidates": [{
|
||||
"content": {"role": "model", "parts": [{"text": "Partial answer"}]}
|
||||
}]}))
|
||||
.unwrap()),
|
||||
Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "read failed").into()),
|
||||
];
|
||||
let events = futures::executor::block_on(
|
||||
GoogleEventMapper::new()
|
||||
.map_stream(Box::pin(futures::stream::iter(responses)))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
assert!(matches!(
|
||||
events.as_slice(),
|
||||
[
|
||||
Ok(LanguageModelCompletionEvent::Text(text)),
|
||||
Err(LanguageModelCompletionError::Other(error)),
|
||||
] if text == "Partial answer"
|
||||
&& error.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|error| error.kind() == std::io::ErrorKind::ConnectionReset)
|
||||
));
|
||||
}
|
||||
|
||||
fn text_request() -> LanguageModelRequest {
|
||||
LanguageModelRequest {
|
||||
messages: vec![LanguageModelRequestMessage {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::mem;
|
||||
|
||||
use anyhow::{Result, anyhow, bail};
|
||||
use anyhow::{Result, bail};
|
||||
use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
|
||||
use http_client::{
|
||||
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
|
||||
@@ -198,7 +198,10 @@ pub async fn stream_generate_content(
|
||||
api_key: &str,
|
||||
mut request: GenerateContentRequest,
|
||||
extra_headers: &CustomHeaders,
|
||||
) -> Result<BoxStream<'static, Result<GenerateContentResponse>>, LanguageModelCompletionError> {
|
||||
) -> Result<
|
||||
BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
|
||||
LanguageModelCompletionError,
|
||||
> {
|
||||
let api_key = api_key.trim();
|
||||
validate_generate_content_request(&request)?;
|
||||
|
||||
@@ -251,15 +254,21 @@ pub async fn stream_generate_content(
|
||||
}
|
||||
match serde_json::from_str(line) {
|
||||
Ok(response) => Some(Ok(response)),
|
||||
Err(error) => Some(Err(anyhow!(format!(
|
||||
"Error parsing JSON: {error:?}\n{line:?}"
|
||||
)))),
|
||||
Err(error) => {
|
||||
Some(Err(LanguageModelCompletionError::DeserializeResponse {
|
||||
provider: GOOGLE_PROVIDER_NAME,
|
||||
error,
|
||||
}))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
|
||||
provider: GOOGLE_PROVIDER_NAME,
|
||||
error,
|
||||
})),
|
||||
}
|
||||
})
|
||||
.boxed())
|
||||
@@ -956,7 +965,75 @@ struct ListModelEntry {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use completion::GoogleEventMapper;
|
||||
use futures::{TryStreamExt, stream};
|
||||
use http_client::{FakeHttpClient, Response};
|
||||
use language_model_core::LanguageModelCompletionEvent;
|
||||
use serde_json::json;
|
||||
use std::io::{Error, ErrorKind};
|
||||
|
||||
#[test]
|
||||
fn stream_body_failures_remain_retryable() {
|
||||
for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
|
||||
let error = completion_error(Some(kind));
|
||||
assert!(error.is_transient());
|
||||
assert!(matches!(
|
||||
error,
|
||||
LanguageModelCompletionError::ApiReadResponseError { provider, error }
|
||||
if provider == GOOGLE_PROVIDER_NAME && error.kind() == kind
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_stream_data_remains_non_retryable() {
|
||||
let error = completion_error(None);
|
||||
assert!(!error.is_transient());
|
||||
assert!(matches!(
|
||||
error,
|
||||
LanguageModelCompletionError::DeserializeResponse { provider, .. }
|
||||
if provider == GOOGLE_PROVIDER_NAME
|
||||
));
|
||||
}
|
||||
|
||||
fn completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
|
||||
futures::executor::block_on(async {
|
||||
let client = FakeHttpClient::create(move |_| async move {
|
||||
let chunks = vec![
|
||||
Ok(b"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Thinking\"}]}}]}\n\n".to_vec()),
|
||||
match read_error {
|
||||
Some(kind) => Err(Error::new(kind, "stream read failed")),
|
||||
None => Ok(b"data: not json\n\n".to_vec()),
|
||||
},
|
||||
];
|
||||
Ok(Response::builder().body(AsyncBody::from_reader(
|
||||
stream::iter(chunks).into_async_read(),
|
||||
))?)
|
||||
});
|
||||
let request = serde_json::from_value(json!({
|
||||
"model": "models/gemini-3.8-flash",
|
||||
"contents": [{"role": "user", "parts": [{"text": "Think"}]}]
|
||||
}))
|
||||
.unwrap();
|
||||
let responses = stream_generate_content(
|
||||
client.as_ref(),
|
||||
API_URL,
|
||||
"test-key",
|
||||
request,
|
||||
&CustomHeaders::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut events = GoogleEventMapper::new().map_stream(responses).boxed();
|
||||
assert!(matches!(
|
||||
events.next().await,
|
||||
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Thinking"
|
||||
));
|
||||
let error = events.next().await.unwrap().unwrap_err();
|
||||
assert!(events.next().await.is_none());
|
||||
error
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lowest_thinking_level_per_model() {
|
||||
|
||||
@@ -246,7 +246,10 @@ impl GoogleLanguageModelProvider {
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<
|
||||
futures::stream::BoxStream<'static, Result<GenerateContentResponse>>,
|
||||
futures::stream::BoxStream<
|
||||
'static,
|
||||
Result<GenerateContentResponse, LanguageModelCompletionError>,
|
||||
>,
|
||||
LanguageModelCompletionError,
|
||||
>,
|
||||
> {
|
||||
|
||||
@@ -201,8 +201,15 @@ impl OpenAiLanguageModelProvider {
|
||||
request_limiter: &RateLimiter,
|
||||
request: ResponseRequest,
|
||||
cx: &AsyncApp,
|
||||
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
|
||||
{
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<
|
||||
futures::stream::BoxStream<
|
||||
'static,
|
||||
Result<ResponsesStreamEvent, open_ai::RequestError>,
|
||||
>,
|
||||
>,
|
||||
> {
|
||||
let http_client = self.http_client.clone();
|
||||
|
||||
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
|
||||
|
||||
@@ -177,8 +177,15 @@ impl OpenAiCompatibleLanguageModelProvider {
|
||||
request_limiter: &RateLimiter,
|
||||
request: ResponseRequest,
|
||||
cx: &AsyncApp,
|
||||
) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
|
||||
{
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<
|
||||
futures::stream::BoxStream<
|
||||
'static,
|
||||
Result<ResponsesStreamEvent, open_ai::RequestError>,
|
||||
>,
|
||||
>,
|
||||
> {
|
||||
let http_client = self.http_client.clone();
|
||||
|
||||
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, _cx| {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use anyhow::{Result, anyhow};
|
||||
use collections::HashMap;
|
||||
use futures::{Stream, StreamExt};
|
||||
use futures::{Stream, StreamExt, stream::BoxStream};
|
||||
use http_client::StatusCode;
|
||||
use language_model_core::{
|
||||
CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent,
|
||||
@@ -11,7 +11,6 @@ use language_model_core::{
|
||||
ProviderErrorCategory, Role, StopReason, TokenUsage, provider_name_for_id,
|
||||
util::{fix_streamed_json, parse_tool_arguments},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::responses::{
|
||||
@@ -790,15 +789,15 @@ impl OpenAiResponseEventMapper {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn map_stream(
|
||||
pub fn map_stream<E: Into<LanguageModelCompletionError> + 'static>(
|
||||
mut self,
|
||||
events: Pin<Box<dyn Send + Stream<Item = Result<ResponsesStreamEvent>>>>,
|
||||
events: BoxStream<'static, Result<ResponsesStreamEvent, E>>,
|
||||
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
|
||||
{
|
||||
events.flat_map(move |event| {
|
||||
futures::stream::iter(match event {
|
||||
Ok(event) => self.map_event(event),
|
||||
Err(error) => vec![Err(LanguageModelCompletionError::from(anyhow!(error)))],
|
||||
Err(error) => vec![Err(error.into())],
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -1592,7 +1591,9 @@ mod tests {
|
||||
fn map_response_events(events: Vec<ResponsesStreamEvent>) -> Vec<LanguageModelCompletionEvent> {
|
||||
block_on(async {
|
||||
OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
|
||||
.map_stream(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
.map_stream(Box::pin(futures::stream::iter(
|
||||
events.into_iter().map(Ok::<_, crate::RequestError>),
|
||||
)))
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
|
||||
@@ -883,7 +883,7 @@ pub async fn stream_response(
|
||||
api_key: &str,
|
||||
request: Request,
|
||||
extra_headers: &CustomHeaders,
|
||||
) -> Result<BoxStream<'static, Result<StreamEvent>>, RequestError> {
|
||||
) -> Result<BoxStream<'static, Result<StreamEvent, RequestError>>, RequestError> {
|
||||
let uri = format!("{api_url}/responses");
|
||||
let is_streaming = request.stream;
|
||||
let request = HttpRequest::builder()
|
||||
@@ -909,31 +909,41 @@ pub async fn stream_response(
|
||||
if response.status().is_success() {
|
||||
if is_streaming {
|
||||
let reader = BufReader::new(response.into_body());
|
||||
let provider_name = provider_name.to_owned();
|
||||
Ok(reader
|
||||
.lines()
|
||||
.filter_map(|line| async move {
|
||||
match line {
|
||||
Ok(line) => {
|
||||
let line = line
|
||||
.strip_prefix("data: ")
|
||||
.or_else(|| line.strip_prefix("data:"))?;
|
||||
if line == "[DONE]" || line.is_empty() {
|
||||
None
|
||||
} else {
|
||||
match decode_stream_event(line) {
|
||||
Ok(event) => Some(Ok(event)),
|
||||
Err(error) => {
|
||||
log::error!(
|
||||
"Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
|
||||
error,
|
||||
line,
|
||||
);
|
||||
Some(Err(anyhow!(error)))
|
||||
.filter_map(move |line| {
|
||||
let provider_name = provider_name.clone();
|
||||
async move {
|
||||
match line {
|
||||
Ok(line) => {
|
||||
let line = line
|
||||
.strip_prefix("data: ")
|
||||
.or_else(|| line.strip_prefix("data:"))?;
|
||||
if line == "[DONE]" || line.is_empty() {
|
||||
None
|
||||
} else {
|
||||
match decode_stream_event(line) {
|
||||
Ok(event) => Some(Ok(event)),
|
||||
Err(error) => {
|
||||
log::error!(
|
||||
"Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
|
||||
error,
|
||||
line,
|
||||
);
|
||||
Some(Err(RequestError::DeserializeResponse {
|
||||
provider: provider_name,
|
||||
error,
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(RequestError::ReadResponse {
|
||||
provider: provider_name,
|
||||
error,
|
||||
})),
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
}
|
||||
})
|
||||
.boxed())
|
||||
@@ -1094,12 +1104,30 @@ fn decode_stream_event(line: &str) -> serde_json::Result<StreamEvent> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::completion::OpenAiResponseEventMapper;
|
||||
use futures::executor::block_on;
|
||||
use futures::{TryStreamExt, stream};
|
||||
use http_client::FakeHttpClient;
|
||||
use language_model_core::OPEN_AI_PROVIDER_ID;
|
||||
use language_model_core::{
|
||||
LanguageModelCompletionError, LanguageModelCompletionEvent, OPEN_AI_PROVIDER_ID,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[test]
|
||||
fn stream_body_failures_remain_retryable() {
|
||||
for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
|
||||
let error = stream_completion_error(Some(kind));
|
||||
assert!(error.is_transient());
|
||||
assert!(matches!(
|
||||
error,
|
||||
LanguageModelCompletionError::ApiReadResponseError { provider, error }
|
||||
if provider.0.as_ref() == "Custom Responses" && error.kind() == kind
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_content_accepts_unknown_parts() {
|
||||
let content = json!([
|
||||
@@ -1211,6 +1239,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_stream_data_remains_non_retryable() {
|
||||
let error = stream_completion_error(None);
|
||||
assert!(!error.is_transient());
|
||||
assert!(matches!(
|
||||
error,
|
||||
LanguageModelCompletionError::DeserializeResponse { provider, .. }
|
||||
if provider.0.as_ref() == "Custom Responses"
|
||||
));
|
||||
}
|
||||
|
||||
fn stream_completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
|
||||
block_on(async {
|
||||
let client = FakeHttpClient::create(move |_| async move {
|
||||
let chunks = vec![
|
||||
Ok(b"data: {\"type\":\"response.output_text.delta\",\"item_id\":\"message\",\"output_index\":0,\"delta\":\"Partial answer\"}\n\n".to_vec()),
|
||||
match read_error {
|
||||
Some(kind) => Err(Error::new(kind, "stream read failed")),
|
||||
None => Ok(b"data: not json\n\n".to_vec()),
|
||||
},
|
||||
];
|
||||
Ok(
|
||||
http_client::Response::builder().body(AsyncBody::from_reader(
|
||||
stream::iter(chunks).into_async_read(),
|
||||
))?,
|
||||
)
|
||||
});
|
||||
let responses = stream_response(
|
||||
client.as_ref(),
|
||||
"Custom Responses",
|
||||
"https://example.com/v1",
|
||||
"test-key",
|
||||
response_test_request(),
|
||||
&CustomHeaders::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut events = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
|
||||
.map_stream(responses)
|
||||
.boxed();
|
||||
assert!(matches!(
|
||||
events.next().await,
|
||||
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Partial answer"
|
||||
));
|
||||
let error = events.next().await.unwrap().unwrap_err();
|
||||
assert!(events.next().await.is_none());
|
||||
error
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_input_tokens_preserves_structured_input_without_generation_fields() {
|
||||
let input = json!([{
|
||||
|
||||
@@ -8,7 +8,9 @@ use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::B
|
||||
use http_client::{
|
||||
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
|
||||
};
|
||||
use language_model_core::ReasoningEffort;
|
||||
use language_model_core::{
|
||||
LanguageModelCompletionError, LanguageModelProviderName, ReasoningEffort,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod catalog;
|
||||
@@ -138,7 +140,9 @@ pub async fn stream_generate_content(
|
||||
api_key: &str,
|
||||
request: google_ai::GenerateContentRequest,
|
||||
extra_headers: &CustomHeaders,
|
||||
) -> Result<BoxStream<'static, Result<google_ai::GenerateContentResponse>>> {
|
||||
) -> Result<
|
||||
BoxStream<'static, Result<google_ai::GenerateContentResponse, LanguageModelCompletionError>>,
|
||||
> {
|
||||
let api_key = api_key.trim();
|
||||
let model_id = &request.model.model_id;
|
||||
let uri = format!("{api_url}/v1/models/{model_id}:streamGenerateContent?alt=sse");
|
||||
@@ -161,14 +165,20 @@ pub async fn stream_generate_content(
|
||||
match serde_json::from_str(line) {
|
||||
Ok(response) => Some(Ok(response)),
|
||||
Err(error) => {
|
||||
Some(Err(anyhow!("Error parsing JSON: {error:?}\n{line:?}")))
|
||||
Some(Err(LanguageModelCompletionError::DeserializeResponse {
|
||||
provider: LanguageModelProviderName::new("OpenCode"),
|
||||
error,
|
||||
}))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
|
||||
provider: LanguageModelProviderName::new("OpenCode"),
|
||||
error,
|
||||
})),
|
||||
}
|
||||
})
|
||||
.boxed())
|
||||
|
||||
Reference in New Issue
Block a user