Normalize image detail for the receiving model (#44249)

## Why

Switching models can send images from conversation history with `detail: original` to a model that does not support it.

## What changed

Normalize image details on request copies using the receiving model's capabilities. Downgrade `original` to `high` when unsupported, covering user messages and function and custom tool outputs. Preserve stored image details so switching back to a supporting model retains `original`, and continue stripping details for Responses Lite.

## Testing

Expand unit coverage across image detail values and model capabilities. Extend integration coverage for model switches within and between turns, text-only models, and resumed and forked threads, including assertions that persisted tool outputs remain unchanged.

GitOrigin-RevId: 2561651ea74f045e00f708ae53771a41dcd49c94
This commit is contained in:
Abhinav authored and copyberry committed 2026-09-09 17:56:06 +00:00
1 parent aa88a0333c
commit 6eecd04fc1
5 files changed
+199 -97

No files matched your search

+16 -7
View File
@@ -2,8 +2,11 @@ pub use codex_api::ResponseEvent;
use codex_protocol::error::Result;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_tools::ToolSpec;
use futures::Stream;
use serde_json::Value;
@@ -55,23 +58,21 @@ impl Default for Prompt {
impl Prompt {
pub(crate) fn get_formatted_input_for_request(
&self,
use_responses_lite: bool,
model_info: &ModelInfo,
) -> Vec<ResponseItem> {
let mut input = self.input.clone();
if use_responses_lite {
strip_image_details(&mut input);
}
normalize_image_details(&mut input, model_info);
input
}
}
fn strip_image_details(items: &mut [ResponseItem]) {
fn normalize_image_details(items: &mut [ResponseItem], model_info: &ModelInfo) {
for item in items {
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { detail, .. } = content_item {
*detail = None;
normalize_image_detail(detail, model_info);
}
}
}
@@ -82,7 +83,7 @@ fn strip_image_details(items: &mut [ResponseItem]) {
if let FunctionCallOutputContentItem::InputImage { detail, .. } =
content_item
{
*detail = None;
normalize_image_detail(detail, model_info);
}
}
}
@@ -106,6 +107,14 @@ fn strip_image_details(items: &mut [ResponseItem]) {
}
}
fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelInfo) {
if model_info.use_responses_lite {
*detail = None;
} else if *detail == Some(ImageDetail::Original) && !model_info.supports_image_detail_original {
*detail = Some(DEFAULT_IMAGE_DETAIL);
}
}
pub struct ResponseStream {
pub(crate) rx_event: mpsc::Receiver<Result<ResponseEvent>>,
/// Signals the mapper task that the consumer stopped polling before the