mirror of
https://github.com/openai/codex.git
synced 2026-11-14 14:49:31 +00:00
Refactor image content to use a shared ImageReference type (#45543)
## What changed Represent images in `ContentItem` and `FunctionCallOutputContentItem` with `ImageReference::Inline`, flattened to preserve the existing `image_url` wire format. Update image producers and consumers and regenerate app-server schemas and SDK artifacts. Preserve the Python SDK's `InputImageContentItem` and `InputImageFunctionCallOutputContentItem` class names during generation. ## Testing Add a regression test for stable Python image class names and adapt existing image tests to the shared representation. GitOrigin-RevId: c38a780ac3314c2ac2deb3afc1b93b94b6f93fec
This commit is contained in:
1 parent
6ae5e71458
commit
5a66d460d3
79 files changed
+994
-312
No files matched your search
@@ -40,6 +40,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::LocalShellAction;
|
||||
use codex_protocol::models::LocalShellExecAction;
|
||||
use codex_protocol::models::LocalShellStatus;
|
||||
@@ -1078,7 +1079,9 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: legacy_image_url.to_string(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: legacy_image_url.to_string(),
|
||||
},
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
}],
|
||||
phase: None,
|
||||
@@ -1218,7 +1221,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
|
||||
namespace: None,
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.to_string(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: image_url.to_string(),
|
||||
},
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
]),
|
||||
@@ -1247,7 +1252,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
|
||||
name: None,
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.to_string(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: image_url.to_string(),
|
||||
},
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
]),
|
||||
|
||||
@@ -42,6 +42,7 @@ use codex_protocol::dynamic_tools::DynamicToolResponse;
|
||||
use codex_protocol::dynamic_tools::DynamicToolSpec;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
@@ -5662,7 +5663,9 @@ async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transc
|
||||
assert_eq!(
|
||||
evidence.images(),
|
||||
vec![ContentItem::InputImage {
|
||||
image_url: SCREENSHOT.to_owned(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: SCREENSHOT.to_owned()
|
||||
},
|
||||
detail: Some(ImageDetail::Low),
|
||||
}]
|
||||
);
|
||||
|
||||
@@ -10,6 +10,7 @@ use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -69,7 +70,10 @@ fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
|
||||
fn extract_image_url(item: &ResponseItem) -> Option<String> {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span {
|
||||
ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
|
||||
ContentItem::InputImage {
|
||||
image: ImageReference::Inline { image_url },
|
||||
..
|
||||
} => Some(image_url.clone()),
|
||||
_ => None,
|
||||
}),
|
||||
_ => None,
|
||||
@@ -178,7 +182,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu
|
||||
),
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image_url,
|
||||
image: ImageReference::Inline { image_url },
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
ContentItem::InputText {
|
||||
@@ -269,7 +273,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputImage {
|
||||
image_url,
|
||||
image: ImageReference::Inline { image_url },
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
ContentItem::InputText {
|
||||
@@ -341,7 +345,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
|
||||
historical_content.insert(
|
||||
/*index*/ 0,
|
||||
ContentItem::InputImage {
|
||||
image_url: original_image_url.clone(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: original_image_url.clone(),
|
||||
},
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
);
|
||||
@@ -480,7 +486,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: original_image_url,
|
||||
image: ImageReference::Inline {
|
||||
image_url: original_image_url,
|
||||
},
|
||||
detail: Some(ImageDetail::High),
|
||||
}],
|
||||
phase: None,
|
||||
|
||||
@@ -52,6 +52,7 @@ use codex_protocol::mcp_policy::McpServerRequirement;
|
||||
use codex_protocol::mcp_policy::PluginMcpRequirements;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::PermissionProfileSnapshot;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -1549,7 +1550,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
|
||||
};
|
||||
assert!(
|
||||
content.contains(&ContentItem::InputImage {
|
||||
image_url: OPENAI_PNG.to_string(),
|
||||
image: ImageReference::Inline {
|
||||
image_url: OPENAI_PNG.to_string()
|
||||
},
|
||||
detail: Some(ImageDetail::Original),
|
||||
}),
|
||||
"interrupted input must use the current model's unified image budget"
|
||||
|
||||
Reference in new issue
Block a user