mirror of
https://github.com/openai/codex.git
synced 2026-11-21 15:10:25 +00:00
Route image edits through referenced file paths (#26486)
## Why Image edits should use the exact images selected by the model instead of inferring edit inputs from conversation history. ## What changed - Replaced the image tool's `action` argument with optional `referenced_image_paths`. - Treats omitted or empty references as generation and populated references as editing. - Reads referenced absolute image paths and packages them as image data URLs for the edit request. - Removed the previous history-selection and image-count heuristics. - Updated direct and code-mode tool instructions and calls. - Added an app-server integration test covering an attached image routed to the image edit endpoint. ## Validation - Tested end-to-end on local `just codex` with copy pasted image, attached image, etc. - `just test -p codex-image-generation-extension` - `just test -p codex-app-server standalone_image_edit_uses_attached_model_visible_image` - `just fix -p codex-image-generation-extension` - `just bazel-lock-check`
This commit is contained in:
1 parent
85fd52f7e4
commit
123cf62a48
6 files changed
+471
-326
No files matched your search
@@ -19,10 +19,9 @@ use pretty_assertions::assert_eq;
|
||||
|
||||
use super::GeneratedImageOutput;
|
||||
use super::ImageRequest;
|
||||
use super::ImagegenAction;
|
||||
use super::ImagegenArgs;
|
||||
use super::imagegen_tool_spec;
|
||||
use super::request_for_action;
|
||||
use super::request_for_args;
|
||||
use crate::IMAGE_GEN_NAMESPACE;
|
||||
use crate::IMAGEGEN_TOOL_NAME;
|
||||
|
||||
@@ -39,10 +38,17 @@ fn uses_reserved_image_gen_namespace() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_uses_fixed_request_defaults() {
|
||||
fn omitted_references_generate_with_fixed_defaults() {
|
||||
assert_eq!(
|
||||
request_for_action(&args(ImagegenAction::Generate, "paint a moonlit lake"), &[])
|
||||
.expect("generation request should build"),
|
||||
request_for_args(
|
||||
&ImagegenArgs {
|
||||
prompt: "paint a moonlit lake".to_string(),
|
||||
referenced_image_paths: None,
|
||||
num_last_images_to_include: None,
|
||||
},
|
||||
&[]
|
||||
)
|
||||
.expect("generation request should build"),
|
||||
ImageRequest::Generate(ImageGenerationRequest {
|
||||
prompt: "paint a moonlit lake".to_string(),
|
||||
background: Some(ImageBackground::Auto),
|
||||
@@ -54,6 +60,144 @@ fn generate_uses_fixed_request_defaults() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recent_image_fallback_selects_newest_images_in_chronological_order() {
|
||||
let history = vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
input_image("user-1"),
|
||||
input_image("user-2"),
|
||||
ContentItem::InputText {
|
||||
text: "edit these".to_string(),
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::FunctionCall {
|
||||
id: None,
|
||||
name: "mcp_image".to_string(),
|
||||
namespace: None,
|
||||
arguments: "{}".to_string(),
|
||||
call_id: "mcp-call".to_string(),
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "mcp-call".to_string(),
|
||||
output: image_output("mcp"),
|
||||
},
|
||||
ResponseItem::CustomToolCall {
|
||||
id: None,
|
||||
status: Some("completed".to_string()),
|
||||
call_id: "code-mode-call".to_string(),
|
||||
name: "exec".to_string(),
|
||||
input: String::new(),
|
||||
},
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id: "code-mode-call".to_string(),
|
||||
name: Some("exec".to_string()),
|
||||
output: image_output("code-mode"),
|
||||
},
|
||||
ResponseItem::ImageGenerationCall {
|
||||
id: "generated-call".to_string(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: None,
|
||||
result: "generated".to_string(),
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "orphan-call".to_string(),
|
||||
output: image_output("orphan"),
|
||||
},
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
request_for_args(
|
||||
&ImagegenArgs {
|
||||
prompt: "change the lighting".to_string(),
|
||||
referenced_image_paths: None,
|
||||
num_last_images_to_include: Some(4),
|
||||
},
|
||||
&history,
|
||||
)
|
||||
.expect("history-backed edit request should build"),
|
||||
ImageRequest::Edit(expected_edit_request(
|
||||
"change the lighting",
|
||||
&["user-2", "mcp", "code-mode", "generated"],
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conflicting_image_selectors_return_tool_error() {
|
||||
let error = request_for_args(
|
||||
&ImagegenArgs {
|
||||
prompt: "change the lighting".to_string(),
|
||||
referenced_image_paths: Some(vec![
|
||||
"/tmp/image.png"
|
||||
.try_into()
|
||||
.expect("test path should be absolute"),
|
||||
]),
|
||||
num_last_images_to_include: Some(1),
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.expect_err("conflicting selectors should fail");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"provide only one of `referenced_image_paths` or `num_last_images_to_include`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_many_referenced_image_paths_return_tool_error() {
|
||||
let error = request_for_args(
|
||||
&ImagegenArgs {
|
||||
prompt: "change the lighting".to_string(),
|
||||
referenced_image_paths: Some(
|
||||
(0..6)
|
||||
.map(|index| {
|
||||
format!("/tmp/image-{index}.png")
|
||||
.try_into()
|
||||
.expect("test path should be absolute")
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
num_last_images_to_include: None,
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.expect_err("too many paths should fail before reading files");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"`referenced_image_paths` must contain at most 5 paths"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recent_image_fallback_requires_requested_count() {
|
||||
let error = request_for_args(
|
||||
&ImagegenArgs {
|
||||
prompt: "change the lighting".to_string(),
|
||||
referenced_image_paths: None,
|
||||
num_last_images_to_include: Some(2),
|
||||
},
|
||||
&[ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![input_image("only-image")],
|
||||
phase: None,
|
||||
}],
|
||||
)
|
||||
.expect_err("history-backed edit should require the requested image count");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"requested the last 2 conversation images, but only 1 were available"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_output_returns_image_input_and_output_hint() {
|
||||
let output_hint =
|
||||
@@ -128,195 +272,26 @@ fn generated_output_omits_oversized_output_hint() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_matches_context_selector_for_generated_images_after_latest_user_anchor() {
|
||||
let history = vec![
|
||||
generated_item("g1"),
|
||||
generated_item("g2"),
|
||||
generated_item("g3"),
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,u1".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,u2".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
},
|
||||
generated_item("g4"),
|
||||
generated_item("g5"),
|
||||
generated_item("g6"),
|
||||
generated_item("g7"),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
edit_request("change the lighting", &history),
|
||||
expected_edit_request(
|
||||
"change the lighting",
|
||||
&[
|
||||
"data:image/png;base64,u1",
|
||||
"data:image/png;base64,u2",
|
||||
"data:image/png;base64,g5",
|
||||
"data:image/png;base64,g6",
|
||||
"data:image/png;base64,g7",
|
||||
]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_preserves_a_generated_image_when_user_anchor_fills_the_limit() {
|
||||
let history = vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: ["a", "b", "c", "d", "e"]
|
||||
.into_iter()
|
||||
.map(|image| ContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{image}"),
|
||||
detail: None,
|
||||
})
|
||||
.collect(),
|
||||
phase: None,
|
||||
},
|
||||
generated_item("generated"),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
edit_request("edit the last generated image", &history),
|
||||
expected_edit_request(
|
||||
"edit the last generated image",
|
||||
&[
|
||||
"data:image/png;base64,b",
|
||||
"data:image/png;base64,c",
|
||||
"data:image/png;base64,d",
|
||||
"data:image/png;base64,e",
|
||||
"data:image/png;base64,generated",
|
||||
]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_uses_latest_user_upload_before_a_text_only_follow_up() {
|
||||
let history = vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,user".to_string(),
|
||||
detail: None,
|
||||
}],
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: "edit this image".to_string(),
|
||||
}],
|
||||
phase: None,
|
||||
},
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
edit_request("change the lighting", &history),
|
||||
expected_edit_request("change the lighting", &["data:image/png;base64,user"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_reuses_images_from_prior_standalone_imagegen_calls() {
|
||||
let history = vec![
|
||||
ResponseItem::FunctionCall {
|
||||
id: None,
|
||||
name: IMAGEGEN_TOOL_NAME.to_string(),
|
||||
namespace: Some(IMAGE_GEN_NAMESPACE.to_string()),
|
||||
arguments: "{}".to_string(),
|
||||
call_id: "imagegen-1".to_string(),
|
||||
},
|
||||
generated_function_output("imagegen-1", "standalone"),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
edit_request("change the lighting", &history),
|
||||
expected_edit_request("change the lighting", &["data:image/png;base64,standalone"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_keeps_newest_standalone_generated_images_when_over_limit() {
|
||||
let history = (1..=6)
|
||||
.flat_map(|index| {
|
||||
let call_id = format!("imagegen-{index}");
|
||||
vec![
|
||||
ResponseItem::FunctionCall {
|
||||
id: None,
|
||||
name: IMAGEGEN_TOOL_NAME.to_string(),
|
||||
namespace: Some(IMAGE_GEN_NAMESPACE.to_string()),
|
||||
arguments: "{}".to_string(),
|
||||
call_id: call_id.clone(),
|
||||
},
|
||||
generated_function_output(&call_id, &index.to_string()),
|
||||
]
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
edit_request("change the lighting", &history),
|
||||
expected_edit_request(
|
||||
"change the lighting",
|
||||
&[
|
||||
"data:image/png;base64,2",
|
||||
"data:image/png;base64,3",
|
||||
"data:image/png;base64,4",
|
||||
"data:image/png;base64,5",
|
||||
"data:image/png;base64,6",
|
||||
]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_without_image_history_returns_tool_error() {
|
||||
let error = request_for_action(&args(ImagegenAction::Edit, "change the lighting"), &[])
|
||||
.expect_err("edit should require image context");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"image edit requested without any usable image in conversation history"
|
||||
);
|
||||
}
|
||||
|
||||
fn args(action: ImagegenAction, prompt: &str) -> ImagegenArgs {
|
||||
ImagegenArgs {
|
||||
prompt: prompt.to_string(),
|
||||
action,
|
||||
fn input_image(image: &str) -> ContentItem {
|
||||
ContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{image}"),
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn edit_request(prompt: &str, history: &[ResponseItem]) -> ImageEditRequest {
|
||||
let ImageRequest::Edit(request) =
|
||||
request_for_action(&args(ImagegenAction::Edit, prompt), history)
|
||||
.expect("edit request should build")
|
||||
else {
|
||||
panic!("expected edit request");
|
||||
};
|
||||
request
|
||||
fn image_output(image: &str) -> FunctionCallOutputPayload {
|
||||
FunctionCallOutputPayload::from_content_items(vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{image}"),
|
||||
detail: None,
|
||||
}])
|
||||
}
|
||||
|
||||
fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {
|
||||
ImageEditRequest {
|
||||
images: images
|
||||
.iter()
|
||||
.map(|image_url| ImageUrl {
|
||||
image_url: (*image_url).to_string(),
|
||||
.map(|image| ImageUrl {
|
||||
image_url: format!("data:image/png;base64,{image}"),
|
||||
})
|
||||
.collect(),
|
||||
prompt: prompt.to_string(),
|
||||
@@ -328,33 +303,6 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn generated_item(result: &str) -> ResponseItem {
|
||||
ResponseItem::ImageGenerationCall {
|
||||
id: format!("id-{result}"),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: None,
|
||||
result: result.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn generated_function_output(call_id: &str, result: &str) -> ResponseItem {
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: call_id.to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{result}"),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "generated image save hint".to_string(),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn function_payload() -> ToolPayload {
|
||||
ToolPayload::Function {
|
||||
arguments: "{}".to_string(),
|
||||
|
||||
Reference in new issue
Block a user