diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index b3e5e7d2aa..849a94f65d 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -28,6 +28,7 @@ use core_test_support::skip_if_remote; use pretty_assertions::assert_eq; use serde_json::json; use tempfile::TempDir; +use test_case::test_case; use tokio::time::timeout; use wiremock::Mock; use wiremock::MockServer; @@ -59,8 +60,14 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); #[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); +#[test_case(Some(true), "transparent"; "transparent")] +#[test_case(Some(false), "opaque"; "opaque")] +#[test_case(None, "opaque"; "omitted_defaults_to_opaque")] #[tokio::test] -async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> { +async fn standalone_image_generation_returns_saved_path_hint_to_model( + requested_transparent_background: Option, + expected_background: &str, +) -> Result<()> { let call_id = "image-run-1"; let server = responses::start_mock_server().await; Mock::given(method("POST")) @@ -80,6 +87,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul .expect(1) .mount(&server) .await; + let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background); let response_mock = responses::mount_sse_sequence( &server, @@ -90,10 +98,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul call_id, "image_gen", "imagegen", - &json!({ - "prompt": "paint a blue whale", - }) - .to_string(), + &arguments.to_string(), ), responses::ev_completed("resp-1"), ]), @@ -158,6 +163,16 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul .into_iter() .find(|request| request.url.path() == "/api/codex/images/generations") .context("image generation request should be sent")?; + assert_eq!( + image_request.body_json::()?, + json!({ + "prompt": "paint a blue whale", + "background": expected_background, + "model": "gpt-image-2", + "quality": "auto", + "size": "auto", + }) + ); assert_eq!( image_request .headers @@ -231,7 +246,11 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() -> call_id, "image_gen", "imagegen", - &json!({"prompt": "a blue whale on a transparent background"}).to_string(), + &json!({ + "prompt": "a blue whale on a transparent background", + "transparent_background": true, + }) + .to_string(), ), responses::ev_completed("resp-1"), ]), @@ -390,6 +409,79 @@ async fn automatic_image_background_preserves_unknown_transparency() -> Result<( Ok(()) } +#[test_case(json!(null); "null")] +#[test_case(json!("true"); "string")] +#[test_case(json!(1); "number")] +#[tokio::test] +async fn invalid_transparent_background_does_not_dispatch( + transparent_background: serde_json::Value, +) -> Result<()> { + let call_id = "invalid-background"; + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + call_id, + "image_gen", + "imagegen", + &json!({ + "prompt": "paint a blue whale", + "transparent_background": transparent_background, + }) + .to_string(), + ), + responses::ev_completed("resp-1"), + ]), + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "The background argument was invalid."), + responses::ev_completed("resp-2"), + ]), + ], + ) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("access-chatgpt"), + AuthCredentialsStoreMode::File, + )?; + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .with_env_overrides(&[("OPENAI_API_KEY", None)]) + .build_initialized_with_timeout(DEFAULT_READ_TIMEOUT) + .await?; + start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + let output = requests[1].function_call_output(call_id); + assert!( + output["output"] + .as_str() + .is_some_and(|text| text.contains("expected a boolean")), + "the model should receive the argument validation error: {output}" + ); + assert!( + server + .received_requests() + .await + .context("failed to fetch received requests")? + .iter() + .all(|request| !request.url.path().starts_with("/api/codex/images/")) + ); + Ok(()) +} + #[tokio::test] async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> { let call_id = "image-run-failed"; @@ -624,8 +716,14 @@ async fn image_generation_usage_limit_preserves_correlated_failure_metadata() -> Ok(()) } +#[test_case(Some(true), "transparent"; "transparent")] +#[test_case(Some(false), "opaque"; "opaque")] +#[test_case(None, "opaque"; "omitted_defaults_to_opaque")] #[tokio::test] -async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> { +async fn standalone_image_edit_uses_attached_model_visible_image( + requested_transparent_background: Option, + expected_background: &str, +) -> Result<()> { skip_if_remote!( Ok(()), "remote executors use different imagegen storage approaches, so host-local image paths are unavailable" @@ -634,11 +732,10 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> let (edit_request, _) = run_image_edit_test(|codex_home| { let image_path = codex_home.join("attached.png"); std::fs::write(&image_path, TINY_PNG_BYTES)?; + let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background); + arguments["referenced_image_paths"] = json!([image_path.display().to_string()]); Ok(( - json!({ - "prompt": "add a red hat", - "referenced_image_paths": [image_path.display().to_string()], - }), + arguments, vec![ V2UserInput::Text { text: "Edit the attached image".to_string(), @@ -652,21 +749,35 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> )) }) .await?; - assert_eq!(edit_request["prompt"], "add a red hat"); - assert_eq!(edit_request["images"][0]["image_url"], TINY_PNG_DATA_URL); + assert_eq!( + edit_request, + json!({ + "prompt": "add a red hat", + "images": [{"image_url": TINY_PNG_DATA_URL}], + "background": expected_background, + "model": "gpt-image-2", + "quality": "auto", + "size": "auto", + }) + ); Ok(()) } +#[test_case(Some(true), "transparent"; "transparent")] +#[test_case(Some(false), "opaque"; "opaque")] +#[test_case(None, "opaque"; "omitted_defaults_to_opaque")] #[tokio::test] -async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -> Result<()> { +async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image( + requested_transparent_background: Option, + expected_background: &str, +) -> Result<()> { let image_url = TINY_PNG_DATA_URL; let (edit_request, completed_image) = run_image_edit_test(|_| { + let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background); + arguments["num_last_images_to_include"] = json!(1); Ok(( - json!({ - "prompt": "add a red hat", - "num_last_images_to_include": 1, - }), + arguments, vec![ V2UserInput::Text { text: "Edit the attached image".to_string(), @@ -682,8 +793,17 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() - )) }) .await?; - assert_eq!(edit_request["prompt"], "add a red hat"); - assert_eq!(edit_request["images"][0]["image_url"], image_url); + assert_eq!( + edit_request, + json!({ + "prompt": "add a red hat", + "images": [{"image_url": image_url}], + "background": expected_background, + "model": "gpt-image-2", + "quality": "auto", + "size": "auto", + }) + ); assert_eq!(completed_image.transparent_background, Some(true)); Ok(()) @@ -735,11 +855,18 @@ async fn standalone_image_generation_is_exposed_in_code_mode_only() -> Result<() } #[cfg(not(windows))] +#[test_case(Some(true), "transparent"; "transparent")] +#[test_case(Some(false), "opaque"; "opaque")] +#[test_case(None, "opaque"; "omitted_defaults_to_opaque")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result<()> { +async fn standalone_image_generation_is_callable_from_code_mode_only( + requested_transparent_background: Option, + expected_background: &str, +) -> Result<()> { let call_id = "code-mode-image-run-1"; let server = responses::start_mock_server().await; mount_image_response_with_background(&server, "opaque").await; + let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background); let response_mock = responses::mount_sse_sequence( &server, @@ -749,12 +876,12 @@ async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result responses::ev_custom_tool_call( call_id, "exec", - r#" -const result = await tools.image_gen__imagegen({ - prompt: "paint a blue whale", -}); + &format!( + r#" +const result = await tools.image_gen__imagegen({arguments}); generatedImage(result); -"#, +"# + ), ), responses::ev_completed("resp-1"), ]), @@ -808,10 +935,35 @@ generatedImage(result); .is_some_and(|text| text.contains("Generated images are saved")) ); assert_eq!(output["output"].as_array().map(Vec::len), Some(3)); + let image_request = server + .received_requests() + .await + .context("failed to fetch received requests")? + .into_iter() + .find(|request| request.url.path() == "/api/codex/images/generations") + .context("image generation request should be sent")?; + assert_eq!( + image_request.body_json::()?, + json!({ + "prompt": "paint a blue whale", + "background": expected_background, + "model": "gpt-image-2", + "quality": "auto", + "size": "auto", + }) + ); Ok(()) } +fn imagegen_arguments(prompt: &str, transparent_background: Option) -> serde_json::Value { + let mut arguments = json!({"prompt": prompt}); + if let Some(transparent_background) = transparent_background { + arguments["transparent_background"] = json!(transparent_background); + } + arguments +} + async fn start_image_generation_turn( mcp: &mut TestAppServer, thread_start_params: ThreadStartParams, diff --git a/codex-rs/ext/image-generation/imagegen_description.md b/codex-rs/ext/image-generation/imagegen_description.md index 0667fb494d..0e793f2a8d 100644 --- a/codex-rs/ext/image-generation/imagegen_description.md +++ b/codex-rs/ext/image-generation/imagegen_description.md @@ -6,6 +6,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi Guidelines: - imagegen needs a few minutes to finish. In code-mode, use the first-line @exec directive to give the initial call 120 seconds and the same yield for any waits that follow. Once it finishes, return the image with generatedImage(result). - Avoid printing the full result or its base64 image data with `text()` or `notify()`; print only small metadata when needed. +- Set `transparent_background` to true when the request calls for a transparent background, including background removal or a cutout; set it to false otherwise. For edits, preserve existing transparency unless the user asks to change it. - Omit both `referenced_image_paths` and `num_last_images_to_include` when generating a brand new image. - For edits, use `referenced_image_paths` when every target image has a local file path. - If you have not seen a local image yet, use `view_image` to inspect it before editing. diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs index 791244445e..0e6b9b278c 100644 --- a/codex-rs/ext/image-generation/src/tests.rs +++ b/codex-rs/ext/image-generation/src/tests.rs @@ -62,6 +62,7 @@ async fn omitted_references_generate_with_fixed_defaults() { request_for_call_args( &ImagegenArgs { prompt: "paint a moonlit lake".to_string(), + transparent_background: false, referenced_image_paths: None, num_last_images_to_include: None, }, @@ -72,7 +73,7 @@ async fn omitted_references_generate_with_fixed_defaults() { .expect("generation request should build"), ImageRequest::Generate(ImageGenerationRequest { prompt: "paint a moonlit lake".to_string(), - background: Some(ImageBackground::Auto), + background: Some(ImageBackground::Opaque), model: "gpt-image-2".to_string(), n: None, quality: Some(ImageQuality::Auto), @@ -151,6 +152,7 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() { request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: None, num_last_images_to_include: Some(5), }, @@ -172,6 +174,7 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() { let error = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: None, num_last_images_to_include: Some(1), }, @@ -208,6 +211,7 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_wind let error = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: None, num_last_images_to_include: Some(1), }, @@ -252,6 +256,7 @@ async fn conflicting_image_selectors_return_tool_error() { let error = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: Some(vec![ "/tmp/image.png" .try_into() @@ -276,6 +281,7 @@ async fn too_many_referenced_image_paths_return_tool_error() { let error = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: Some( (0..6) .map(|index| { @@ -304,6 +310,7 @@ async fn recent_image_fallback_requires_requested_count() { let error = request_for_call_args( &ImagegenArgs { prompt: "change the lighting".to_string(), + transparent_background: false, referenced_image_paths: None, num_last_images_to_include: Some(2), }, @@ -430,7 +437,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest { }) .collect(), prompt: prompt.to_string(), - background: Some(ImageBackground::Auto), + background: Some(ImageBackground::Opaque), model: "gpt-image-2".to_string(), n: None, quality: Some(ImageQuality::Auto), diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 689708823d..fb54cb3595 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -89,6 +89,9 @@ impl ImageGenerationTool { #[serde(deny_unknown_fields)] struct ImagegenArgs { prompt: String, + /// Whether the output should have a transparent background. Defaults to false. + #[serde(default)] + transparent_background: bool, #[schemars(length(max = 5))] referenced_image_paths: Option>, #[schemars(range(min = 1, max = 5))] @@ -120,7 +123,7 @@ impl<'call> ToolExecutor> for ImageGenerationTool { ToolName::namespaced(IMAGE_GEN_NAMESPACE, IMAGEGEN_TOOL_NAME) } - /// Advertises the model contract: a rewritten prompt and optional edit references. + /// Advertises a rewritten prompt, background choice, and optional edit references. fn spec(&self) -> ToolSpec { imagegen_tool_spec() } @@ -427,6 +430,11 @@ async fn request_for_call_args( history: &[ResponseItem], environments: &[ToolEnvironment<'_>], ) -> Result { + let background = if args.transparent_background { + ImageBackground::Transparent + } else { + ImageBackground::Opaque + }; let paths = args.referenced_image_paths.as_deref().unwrap_or_default(); if paths.len() > MAX_EDIT_IMAGES { return Err(FunctionCallError::RespondToModel(format!( @@ -437,7 +445,7 @@ async fn request_for_call_args( (true, None) => { return Ok(ImageRequest::Generate(ImageGenerationRequest { prompt: args.prompt.clone(), - background: Some(ImageBackground::Auto), + background: Some(background), model: IMAGE_MODEL.to_string(), n: None, quality: Some(ImageQuality::Auto), @@ -481,7 +489,7 @@ async fn request_for_call_args( Ok(ImageRequest::Edit(ImageEditRequest { images, prompt: args.prompt.clone(), - background: Some(ImageBackground::Auto), + background: Some(background), model: IMAGE_MODEL.to_string(), n: None, quality: Some(ImageQuality::Auto),