github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-25 15:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Normalize image detail for the receiving model (#44249)

Browse Source 
## Why

Switching models can send images from conversation history with `detail: original` to a model that does not support it.

## What changed

Normalize image details on request copies using the receiving model's capabilities. Downgrade `original` to `high` when unsupported, covering user messages and function and custom tool outputs. Preserve stored image details so switching back to a supporting model retains `original`, and continue stripping details for Responses Lite.

## Testing

Expand unit coverage across image detail values and model capabilities. Extend integration coverage for model switches within and between turns, text-only models, and resumed and forked threads, including assertions that persisted tool outputs remain unchanged.

GitOrigin-RevId: 2561651ea74f045e00f708ae53771a41dcd49c94
This commit is contained in:
Abhinav
2026-09-09 17:56:06 +00:00
committed by copyberry
parent aa88a0333c
commit 6eecd04fc1
5 changed files with 199 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/client.rs
+1 -1
View File
@@ -897,7 +897,7 @@ impl ModelClient {
service_tier: Option<String>,
responses_metadata: &CodexResponsesMetadata,
) -> Result<ResponsesApiRequest> {
let mut input = prompt.get_formatted_input_for_request(model_info.use_responses_lite);
let mut input = prompt.get_formatted_input_for_request(model_info);
let is_openai = self.state.provider.info().is_openai();
let (instructions, tools) = if model_info.use_responses_lite {
// These prompt-only items are rebuilt on every request. Hash their visible payloads
codex-rs/core/src/client_common.rs
+16 -7
View File
@@ -2,8 +2,11 @@ pub use codex_api::ResponseEvent;
use codex_protocol::error::Result;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_tools::ToolSpec;
use futures::Stream;
use serde_json::Value;
@@ -55,23 +58,21 @@ impl Default for Prompt {
impl Prompt {
pub(crate) fn get_formatted_input_for_request(
&self,
use_responses_lite: bool,
model_info: &ModelInfo,
) -> Vec<ResponseItem> {
let mut input = self.input.clone();
if use_responses_lite {
strip_image_details(&mut input);
}
normalize_image_details(&mut input, model_info);
input
}
}
fn strip_image_details(items: &mut [ResponseItem]) {
fn normalize_image_details(items: &mut [ResponseItem], model_info: &ModelInfo) {
for item in items {
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { detail, .. } = content_item {
*detail = None;
normalize_image_detail(detail, model_info);
}
}
}
@@ -82,7 +83,7 @@ fn strip_image_details(items: &mut [ResponseItem]) {
if let FunctionCallOutputContentItem::InputImage { detail, .. } =
content_item
{
*detail = None;
normalize_image_detail(detail, model_info);
}
}
}
@@ -106,6 +107,14 @@ fn strip_image_details(items: &mut [ResponseItem]) {
}
}
fn normalize_image_detail(detail: &mut Option<ImageDetail>, model_info: &ModelInfo) {
if model_info.use_responses_lite {
*detail = None;
} else if *detail == Some(ImageDetail::Original) && !model_info.supports_image_detail_original {
*detail = Some(DEFAULT_IMAGE_DETAIL);
}
}
pub struct ResponseStream {
pub(crate) rx_event: mpsc::Receiver<Result<ResponseEvent>>,
/// Signals the mapper task that the consumer stopped polling before the
codex-rs/core/src/client_common_tests.rs
+43 -55
View File
@@ -2,6 +2,7 @@ use codex_api::OpenAiVerbosity;
use codex_api::ResponsesApiRequest;
use codex_api::TextControls;
use codex_api::create_text_param_for_request;
use codex_models_manager::model_info::model_info_from_slug;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
@@ -15,16 +16,21 @@ fn empty_tools() -> Arc<RawValue> {
Arc::from(RawValue::from_string("[]".to_string()).expect("valid tool JSON"))
}
fn prompt_with_image_outputs() -> Prompt {
fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
Prompt {
input: vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Original),
}],
content: vec![
ContentItem::InputText {
text: "Describe this image.".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail,
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
},
@@ -36,7 +42,7 @@ fn prompt_with_image_outputs() -> Prompt {
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,function".to_string(),
detail: Some(ImageDetail::High),
detail,
},
]),
internal_chat_message_metadata_passthrough: None,
@@ -48,7 +54,7 @@ fn prompt_with_image_outputs() -> Prompt {
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,custom".to_string(),
detail: Some(ImageDetail::Auto),
detail,
},
]),
internal_chat_message_metadata_passthrough: None,
@@ -58,58 +64,40 @@ fn prompt_with_image_outputs() -> Prompt {
}
}
#[test]
fn responses_lite_request_copies_strip_image_details() {
let prompt = prompt_with_image_outputs();
#[test_case::test_case(Some(ImageDetail::Original), Some(ImageDetail::High); "original")]
#[test_case::test_case(Some(ImageDetail::High), Some(ImageDetail::High); "high")]
#[test_case::test_case(Some(ImageDetail::Auto), Some(ImageDetail::Auto); "auto")]
#[test_case::test_case(Some(ImageDetail::Low), Some(ImageDetail::Low); "low")]
#[test_case::test_case(None, None; "unspecified")]
fn request_copies_project_image_details_for_receiving_model(
detail: Option<ImageDetail>,
unsupported_detail: Option<ImageDetail>,
) {
let prompt = prompt_with_image_outputs(detail);
let original = prompt.input.clone();
let stripped = prompt.get_formatted_input_for_request(/*use_responses_lite*/ true);
let mut model_info = model_info_from_slug("gpt-5.4");
model_info.use_responses_lite = false;
model_info.supports_image_detail_original = true;
assert_eq!(
stripped,
vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: None,
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
},
ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("function-call".to_string()),
name: None,
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,function".to_string(),
detail: None,
},
]),
internal_chat_message_metadata_passthrough: None,
},
ResponseItem::CustomToolCallOutput {
id: None,
call_id: "custom-call".to_string(),
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,custom".to_string(),
detail: None,
},
]),
internal_chat_message_metadata_passthrough: None,
},
]
);
assert_eq!(prompt.input, original);
assert_eq!(
prompt.get_formatted_input_for_request(/*use_responses_lite*/ false),
prompt.get_formatted_input_for_request(&model_info),
original
);
model_info.supports_image_detail_original = false;
assert_eq!(
prompt.get_formatted_input_for_request(&model_info),
prompt_with_image_outputs(unsupported_detail).input
);
model_info.use_responses_lite = true;
for supports_original in [false, true] {
model_info.supports_image_detail_original = supports_original;
assert_eq!(
prompt.get_formatted_input_for_request(&model_info),
prompt_with_image_outputs(/*detail*/ None).input
);
}
assert_eq!(prompt.input, original);
}
#[test]