mirror of
https://github.com/openai/codex.git
synced 2026-11-13 22:47:20 +00:00
Label attached images so agent can understand in-message labels (#8950)
Agent wouldn't "see" attached images and would instead try to use the view_file tool: <img width="1516" height="504" alt="image" src="https://github.com/user-attachments/assets/68a705bb-f962-4fc1-9087-e932a6859b12" /> In this PR, we wrap image content items in XML tags with the name of each image (now just a numbered name like `[Image #1]`), so that the model can understand inline image references (based on name). We also put the image content items above the user message which the model seems to prefer (maybe it's more used to definitions being before references). We also tweak the view_file tool description which seemed to help a bit Results on a simple eval set of images: Before <img width="980" height="310" alt="image" src="https://github.com/user-attachments/assets/ba838651-2565-4684-a12e-81a36641bf86" /> After <img width="918" height="322" alt="image" src="https://github.com/user-attachments/assets/10a81951-7ee6-415e-a27e-e7a3fd0aee6f" /> ```json [ { "id": "single_describe", "prompt": "Describe the attached image in one sentence.", "images": ["image_a.png"] }, { "id": "single_color", "prompt": "What is the dominant color in the image? Answer with a single color word.", "images": ["image_b.png"] }, { "id": "orientation_check", "prompt": "Is the image portrait or landscape? Answer in one sentence.", "images": ["image_c.png"] }, { "id": "detail_request", "prompt": "Look closely at the image and call out any small details you notice.", "images": ["image_d.png"] }, { "id": "two_images_compare", "prompt": "I attached two images. Are they the same or different? Briefly explain.", "images": ["image_a.png", "image_b.png"] }, { "id": "two_images_captions", "prompt": "Provide a short caption for each image (Image 1, Image 2).", "images": ["image_c.png", "image_d.png"] }, { "id": "multi_image_rank", "prompt": "Rank the attached images from most colorful to least colorful.", "images": ["image_a.png", "image_b.png", "image_c.png"] }, { "id": "multi_image_choice", "prompt": "Which image looks more vibrant? Answer with 'Image 1' or 'Image 2'.", "images": ["image_b.png", "image_d.png"] } ] ```
This commit is contained in:
1 parent
cf515142b0
commit
6709ad8975
16 files changed
+800
-608
No files matched your search
@@ -1452,12 +1452,13 @@ impl ChatWidget {
|
||||
{
|
||||
match paste_image_to_temp_png() {
|
||||
Ok((path, info)) => {
|
||||
self.attach_image(
|
||||
path,
|
||||
tracing::debug!(
|
||||
"pasted image size={}x{} format={}",
|
||||
info.width,
|
||||
info.height,
|
||||
info.encoded_format.label(),
|
||||
info.encoded_format.label()
|
||||
);
|
||||
self.attach_image(path);
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!("failed to paste image: {err}");
|
||||
@@ -1510,18 +1511,9 @@ impl ChatWidget {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn attach_image(
|
||||
&mut self,
|
||||
path: PathBuf,
|
||||
width: u32,
|
||||
height: u32,
|
||||
format_label: &str,
|
||||
) {
|
||||
tracing::info!(
|
||||
"attach_image path={path:?} width={width} height={height} format={format_label}",
|
||||
);
|
||||
self.bottom_pane
|
||||
.attach_image(path, width, height, format_label);
|
||||
pub(crate) fn attach_image(&mut self, path: PathBuf) {
|
||||
tracing::info!("attach_image path={path:?}");
|
||||
self.bottom_pane.attach_image(path);
|
||||
self.request_redraw();
|
||||
}
|
||||
|
||||
@@ -1818,14 +1810,14 @@ impl ChatWidget {
|
||||
return;
|
||||
}
|
||||
|
||||
if !text.is_empty() {
|
||||
items.push(UserInput::Text { text: text.clone() });
|
||||
}
|
||||
|
||||
for path in image_paths {
|
||||
items.push(UserInput::LocalImage { path });
|
||||
}
|
||||
|
||||
if !text.is_empty() {
|
||||
items.push(UserInput::Text { text: text.clone() });
|
||||
}
|
||||
|
||||
if let Some(skills) = self.bottom_pane.skills() {
|
||||
let skill_mentions = find_skill_mentions(&text, skills);
|
||||
for skill in skill_mentions {
|
||||
|
||||
Reference in new issue
Block a user