Refactor image content to use a shared ImageReference type (#45543)

## What changed

Represent images in `ContentItem` and `FunctionCallOutputContentItem` with `ImageReference::Inline`, flattened to preserve the existing `image_url` wire format. Update image producers and consumers and regenerate app-server schemas and SDK artifacts.

Preserve the Python SDK's `InputImageContentItem` and `InputImageFunctionCallOutputContentItem` class names during generation.

## Testing

Add a regression test for stable Python image class names and adapt existing image tests to the shared representation.

GitOrigin-RevId: c38a780ac3314c2ac2deb3afc1b93b94b6f93fec
This commit is contained in:
Krish Chainani authored and copyberry committed 2026-09-14 23:25:09 +00:00
1 parent 6ae5e71458
commit 5a66d460d3
79 files changed
+994 -312

No files matched your search

+10 -3
View File
@@ -40,6 +40,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
use codex_protocol::models::LocalShellStatus;
@@ -1078,7 +1079,9 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: legacy_image_url.to_string(),
image: ImageReference::Inline {
image_url: legacy_image_url.to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -1218,7 +1221,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: Some(ImageDetail::Original),
},
]),
@@ -1247,7 +1252,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: Some(ImageDetail::Original),
},
]),
+4 -1
View File
@@ -42,6 +42,7 @@ use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
@@ -5662,7 +5663,9 @@ async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transc
assert_eq!(
evidence.images(),
vec![ContentItem::InputImage {
image_url: SCREENSHOT.to_owned(),
image: ImageReference::Inline {
image_url: SCREENSHOT.to_owned()
},
detail: Some(ImageDetail::Low),
}]
);
+13 -5
View File
@@ -10,6 +10,7 @@ use codex_protocol::config_types::Settings;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
@@ -69,7 +70,10 @@ fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
fn extract_image_url(item: &ResponseItem) -> Option<String> {
match item {
ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span {
ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
_ => None,
}),
_ => None,
@@ -178,7 +182,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu
),
},
ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -269,7 +273,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -341,7 +345,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
historical_content.insert(
/*index*/ 0,
ContentItem::InputImage {
image_url: original_image_url.clone(),
image: ImageReference::Inline {
image_url: original_image_url.clone(),
},
detail: Some(ImageDetail::High),
},
);
@@ -480,7 +486,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
ResponseInputItem::Message {
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: original_image_url,
image: ImageReference::Inline {
image_url: original_image_url,
},
detail: Some(ImageDetail::High),
}],
phase: None,
+4 -1
View File
@@ -52,6 +52,7 @@ use codex_protocol::mcp_policy::McpServerRequirement;
use codex_protocol::mcp_policy::PluginMcpRequirements;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::PermissionProfileSnapshot;
use codex_protocol::models::ResponseItem;
@@ -1549,7 +1550,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
};
assert!(
content.contains(&ContentItem::InputImage {
image_url: OPENAI_PNG.to_string(),
image: ImageReference::Inline {
image_url: OPENAI_PNG.to_string()
},
detail: Some(ImageDetail::Original),
}),
"interrupted input must use the current model's unified image budget"