Preserve image detail in app-server inputs (#20693)

## Summary

- Add optional image detail to user image inputs across core, app-server
v2, thread history/event mapping, and the generated app-server
schemas/types.
- Preserve requested detail when serializing Responses image inputs:
omitted detail stays on the existing `high` default, while explicit
`original` keeps local images on the original-resolution path.
- Support `high`/`original` consistently for tool image outputs,
including MCP `codex/imageDetail`, code-mode image helpers, and
`view_image`.
This commit is contained in:
Curtis 'Fjord' Hawthorne authored and GitHub committed 2026-05-15 15:04:04 -07:00
1 parent 249d50aafc
commit 8543e39885
81 files changed
+1302 -156

No files matched your search

+1 -1
View File
@@ -4727,7 +4727,7 @@ async fn backtrack_resubmit_preserves_data_image_urls_in_user_turn() {
assert!(items.iter().any(|item| {
matches!(
item,
UserInput::Image { url } if url == &data_image_url
UserInput::Image { url, .. } if url == &data_image_url
)
}));
}
@@ -164,12 +164,14 @@ impl ChatWidget {
for image_url in &remote_image_urls {
items.push(UserInput::Image {
url: image_url.clone(),
detail: None,
});
}
for image in &local_images {
items.push(UserInput::LocalImage {
path: image.path.clone(),
detail: None,
});
}
@@ -58,7 +58,8 @@ async fn submission_preserves_text_elements_and_local_images() {
assert_eq!(
items[0],
UserInput::LocalImage {
path: local_images[0].clone()
path: local_images[0].clone(),
detail: None,
}
);
assert_eq!(
@@ -259,12 +260,14 @@ async fn submission_with_remote_and_local_images_keeps_local_placeholder_numberi
items[0],
UserInput::Image {
url: remote_url.clone(),
detail: None,
}
);
assert_eq!(
items[1],
UserInput::LocalImage {
path: local_images[0].clone(),
detail: None,
}
);
assert_eq!(
@@ -342,6 +345,7 @@ async fn enter_with_only_remote_images_submits_user_turn() {
items,
vec![UserInput::Image {
url: remote_url.clone(),
detail: None,
}]
);
assert_eq!(summary, None);
@@ -1157,9 +1161,11 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() {
},
UserInput::Image {
url: "https://example.com/remote.png".to_string(),
detail: None,
},
UserInput::LocalImage {
path: local_image.clone(),
detail: None,
},
UserInput::Skill {
name: "demo".to_string(),
@@ -1232,6 +1238,7 @@ async fn committed_user_message_with_hidden_prompt_context_renders_local_images(
},
UserInput::LocalImage {
path: local_image.clone(),
detail: None,
},
],
);
@@ -119,6 +119,7 @@ async fn replayed_user_message_preserves_text_elements_and_local_images() {
},
AppServerUserInput::LocalImage {
path: local_images[0].clone(),
detail: None,
},
],
ReplayKind::ResumeInitialMessages,
@@ -188,6 +189,7 @@ async fn replayed_user_message_preserves_remote_image_urls() {
},
AppServerUserInput::Image {
url: remote_image_urls[0].clone(),
detail: None,
},
],
ReplayKind::ResumeInitialMessages,
@@ -441,6 +443,7 @@ async fn replayed_user_message_with_only_remote_images_renders_history_cell() {
"user-1",
vec![AppServerUserInput::Image {
url: remote_image_urls[0].clone(),
detail: None,
}],
ReplayKind::ResumeInitialMessages,
);
@@ -496,6 +499,7 @@ async fn replayed_user_message_with_only_local_images_renders_history_cell() {
"user-1",
vec![AppServerUserInput::LocalImage {
path: local_images[0].clone(),
detail: None,
}],
ReplayKind::ResumeInitialMessages,
);
@@ -597,6 +597,7 @@ async fn item_completed_pops_pending_steer_with_local_image_and_text_elements()
vec![
UserInput::Image {
url: "data:image/png;base64,placeholder".to_string(),
detail: None,
},
UserInput::Text {
text,
+2 -2
View File
@@ -593,8 +593,8 @@ impl ChatWidget {
)
}),
),
UserInput::Image { url } => remote_image_urls.push(url.clone()),
UserInput::LocalImage { path } => local_images.push(path.clone()),
UserInput::Image { url, .. } => remote_image_urls.push(url.clone()),
UserInput::LocalImage { path, .. } => local_images.push(path.clone()),
UserInput::Skill { .. } | UserInput::Mention { .. } => {}
}
}
+2
View File
@@ -269,6 +269,7 @@ mod tests {
let mut items = vec![
UserInput::LocalImage {
path: PathBuf::from("/tmp/screenshot.png"),
detail: None,
},
UserInput::Text {
text,
@@ -288,6 +289,7 @@ mod tests {
vec![
UserInput::LocalImage {
path: PathBuf::from("/tmp/screenshot.png"),
detail: None,
},
UserInput::Text {
text: format!("{expected_prefix}Ask $figma"),