Add explicit background control to image generation (#47484)

## What changed

Add the boolean `transparent_background` argument to `image_gen.imagegen` for generation and editing. Map `true` to `transparent` and `false` or omission to `opaque`, replacing the previous `auto` background setting.

Update tool guidance to request transparency for background removal and cutouts, and to preserve existing transparency during edits unless the user asks to change it.

## Testing

Add integration coverage for transparent, opaque, and omitted background arguments across generation, editing, and code-mode generation. Verify that `null`, string, and numeric values return a validation error without dispatching an image request.

GitOrigin-RevId: b10ac200a619c72cc1787da41b49d65d0cde05dd

codex-rs/app-server/tests/suite/v2/imagegen_extension.rs

@@ -28,6 +28,7 @@ use core_test_support::skip_if_remote;
 use pretty_assertions::assert_eq;
 use serde_json::json;
 use tempfile::TempDir;
+use test_case::test_case;
 use tokio::time::timeout;
 use wiremock::Mock;
 use wiremock::MockServer;
@@ -59,8 +60,14 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60);
 #[cfg(not(any(target_os = "macos", windows)))]
 const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
 
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
 #[tokio::test]
-async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> {
+async fn standalone_image_generation_returns_saved_path_hint_to_model(
+    requested_transparent_background: Option<bool>,
+    expected_background: &str,
+) -> Result<()> {
     let call_id = "image-run-1";
     let server = responses::start_mock_server().await;
     Mock::given(method("POST"))
@@ -80,6 +87,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
         .expect(1)
         .mount(&server)
         .await;
+    let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);
 
     let response_mock = responses::mount_sse_sequence(
         &server,
@@ -90,10 +98,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
                     call_id,
                     "image_gen",
                     "imagegen",
-                    &json!({
-                        "prompt": "paint a blue whale",
-                    })
-                    .to_string(),
+                    &arguments.to_string(),
                 ),
                 responses::ev_completed("resp-1"),
             ]),
@@ -158,6 +163,16 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
         .into_iter()
         .find(|request| request.url.path() == "/api/codex/images/generations")
         .context("image generation request should be sent")?;
+    assert_eq!(
+        image_request.body_json::<serde_json::Value>()?,
+        json!({
+            "prompt": "paint a blue whale",
+            "background": expected_background,
+            "model": "gpt-image-2",
+            "quality": "auto",
+            "size": "auto",
+        })
+    );
     assert_eq!(
         image_request
             .headers
@@ -231,7 +246,11 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() ->
                     call_id,
                     "image_gen",
                     "imagegen",
-                    &json!({"prompt": "a blue whale on a transparent background"}).to_string(),
+                    &json!({
+                        "prompt": "a blue whale on a transparent background",
+                        "transparent_background": true,
+                    })
+                    .to_string(),
                 ),
                 responses::ev_completed("resp-1"),
             ]),
@@ -390,6 +409,79 @@ async fn automatic_image_background_preserves_unknown_transparency() -> Result<(
     Ok(())
 }
 
+#[test_case(json!(null); "null")]
+#[test_case(json!("true"); "string")]
+#[test_case(json!(1); "number")]
+#[tokio::test]
+async fn invalid_transparent_background_does_not_dispatch(
+    transparent_background: serde_json::Value,
+) -> Result<()> {
+    let call_id = "invalid-background";
+    let server = responses::start_mock_server().await;
+    let response_mock = responses::mount_sse_sequence(
+        &server,
+        vec![
+            responses::sse(vec![
+                responses::ev_response_created("resp-1"),
+                responses::ev_function_call_with_namespace(
+                    call_id,
+                    "image_gen",
+                    "imagegen",
+                    &json!({
+                        "prompt": "paint a blue whale",
+                        "transparent_background": transparent_background,
+                    })
+                    .to_string(),
+                ),
+                responses::ev_completed("resp-1"),
+            ]),
+            responses::sse(vec![
+                responses::ev_assistant_message("msg-1", "The background argument was invalid."),
+                responses::ev_completed("resp-2"),
+            ]),
+        ],
+    )
+    .await;
+
+    let codex_home = TempDir::new()?;
+    create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?;
+    write_chatgpt_auth(
+        codex_home.path(),
+        ChatGptAuthFixture::new("access-chatgpt"),
+        AuthCredentialsStoreMode::File,
+    )?;
+    let mut mcp = TestAppServer::builder()
+        .with_codex_home(codex_home.path())
+        .with_env_overrides(&[("OPENAI_API_KEY", None)])
+        .build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)
+        .await?;
+    start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?;
+    timeout(
+        DEFAULT_READ_TIMEOUT,
+        mcp.read_stream_until_notification_message("turn/completed"),
+    )
+    .await??;
+
+    let requests = response_mock.requests();
+    assert_eq!(requests.len(), 2);
+    let output = requests[1].function_call_output(call_id);
+    assert!(
+        output["output"]
+            .as_str()
+            .is_some_and(|text| text.contains("expected a boolean")),
+        "the model should receive the argument validation error: {output}"
+    );
+    assert!(
+        server
+            .received_requests()
+            .await
+            .context("failed to fetch received requests")?
+            .iter()
+            .all(|request| !request.url.path().starts_with("/api/codex/images/"))
+    );
+    Ok(())
+}
+
 #[tokio::test]
 async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> {
     let call_id = "image-run-failed";
@@ -624,8 +716,14 @@ async fn image_generation_usage_limit_preserves_correlated_failure_metadata() ->
     Ok(())
 }
 
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
 #[tokio::test]
-async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> {
+async fn standalone_image_edit_uses_attached_model_visible_image(
+    requested_transparent_background: Option<bool>,
+    expected_background: &str,
+) -> Result<()> {
     skip_if_remote!(
         Ok(()),
         "remote executors use different imagegen storage approaches, so host-local image paths are unavailable"
@@ -634,11 +732,10 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
     let (edit_request, _) = run_image_edit_test(|codex_home| {
         let image_path = codex_home.join("attached.png");
         std::fs::write(&image_path, TINY_PNG_BYTES)?;
+        let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);
+        arguments["referenced_image_paths"] = json!([image_path.display().to_string()]);
         Ok((
-            json!({
-                "prompt": "add a red hat",
-                "referenced_image_paths": [image_path.display().to_string()],
-            }),
+            arguments,
             vec![
                 V2UserInput::Text {
                     text: "Edit the attached image".to_string(),
@@ -652,21 +749,35 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
         ))
     })
     .await?;
-    assert_eq!(edit_request["prompt"], "add a red hat");
-    assert_eq!(edit_request["images"][0]["image_url"], TINY_PNG_DATA_URL);
+    assert_eq!(
+        edit_request,
+        json!({
+            "prompt": "add a red hat",
+            "images": [{"image_url": TINY_PNG_DATA_URL}],
+            "background": expected_background,
+            "model": "gpt-image-2",
+            "quality": "auto",
+            "size": "auto",
+        })
+    );
 
     Ok(())
 }
 
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
 #[tokio::test]
-async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -> Result<()> {
+async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image(
+    requested_transparent_background: Option<bool>,
+    expected_background: &str,
+) -> Result<()> {
     let image_url = TINY_PNG_DATA_URL;
     let (edit_request, completed_image) = run_image_edit_test(|_| {
+        let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);
+        arguments["num_last_images_to_include"] = json!(1);
         Ok((
-            json!({
-                "prompt": "add a red hat",
-                "num_last_images_to_include": 1,
-            }),
+            arguments,
             vec![
                 V2UserInput::Text {
                     text: "Edit the attached image".to_string(),
@@ -682,8 +793,17 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -
         ))
     })
     .await?;
-    assert_eq!(edit_request["prompt"], "add a red hat");
-    assert_eq!(edit_request["images"][0]["image_url"], image_url);
+    assert_eq!(
+        edit_request,
+        json!({
+            "prompt": "add a red hat",
+            "images": [{"image_url": image_url}],
+            "background": expected_background,
+            "model": "gpt-image-2",
+            "quality": "auto",
+            "size": "auto",
+        })
+    );
     assert_eq!(completed_image.transparent_background, Some(true));
 
     Ok(())
@@ -735,11 +855,18 @@ async fn standalone_image_generation_is_exposed_in_code_mode_only() -> Result<()
 }
 
 #[cfg(not(windows))]
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result<()> {
+async fn standalone_image_generation_is_callable_from_code_mode_only(
+    requested_transparent_background: Option<bool>,
+    expected_background: &str,
+) -> Result<()> {
     let call_id = "code-mode-image-run-1";
     let server = responses::start_mock_server().await;
     mount_image_response_with_background(&server, "opaque").await;
+    let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);
 
     let response_mock = responses::mount_sse_sequence(
         &server,
@@ -749,12 +876,12 @@ async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result
                 responses::ev_custom_tool_call(
                     call_id,
                     "exec",
-                    r#"
-const result = await tools.image_gen__imagegen({
-  prompt: "paint a blue whale",
-});
+                    &format!(
+                        r#"
+const result = await tools.image_gen__imagegen({arguments});
 generatedImage(result);
-"#,
+"#
+                    ),
                 ),
                 responses::ev_completed("resp-1"),
             ]),
@@ -808,10 +935,35 @@ generatedImage(result);
             .is_some_and(|text| text.contains("Generated images are saved"))
     );
     assert_eq!(output["output"].as_array().map(Vec::len), Some(3));
+    let image_request = server
+        .received_requests()
+        .await
+        .context("failed to fetch received requests")?
+        .into_iter()
+        .find(|request| request.url.path() == "/api/codex/images/generations")
+        .context("image generation request should be sent")?;
+    assert_eq!(
+        image_request.body_json::<serde_json::Value>()?,
+        json!({
+            "prompt": "paint a blue whale",
+            "background": expected_background,
+            "model": "gpt-image-2",
+            "quality": "auto",
+            "size": "auto",
+        })
+    );
 
     Ok(())
 }
 
+fn imagegen_arguments(prompt: &str, transparent_background: Option<bool>) -> serde_json::Value {
+    let mut arguments = json!({"prompt": prompt});
+    if let Some(transparent_background) = transparent_background {
+        arguments["transparent_background"] = json!(transparent_background);
+    }
+    arguments
+}
+
 async fn start_image_generation_turn(
     mcp: &mut TestAppServer,
     thread_start_params: ThreadStartParams,

codex-rs/ext/image-generation/imagegen_description.md

@@ -6,6 +6,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
 Guidelines:
 - imagegen needs a few minutes to finish. In code-mode, use the first-line @exec directive to give the initial call 120 seconds and the same yield for any waits that follow. Once it finishes, return the image with generatedImage(result).
 - Avoid printing the full result or its base64 image data with `text()` or `notify()`; print only small metadata when needed.
+- Set `transparent_background` to true when the request calls for a transparent background, including background removal or a cutout; set it to false otherwise. For edits, preserve existing transparency unless the user asks to change it.
 - Omit both `referenced_image_paths` and `num_last_images_to_include` when generating a brand new image.
 - For edits, use `referenced_image_paths` when every target image has a local file path.
 - If you have not seen a local image yet, use `view_image` to inspect it before editing.

codex-rs/ext/image-generation/src/tests.rs

@@ -62,6 +62,7 @@ async fn omitted_references_generate_with_fixed_defaults() {
         request_for_call_args(
             &ImagegenArgs {
                 prompt: "paint a moonlit lake".to_string(),
+                transparent_background: false,
                 referenced_image_paths: None,
                 num_last_images_to_include: None,
             },
@@ -72,7 +73,7 @@ async fn omitted_references_generate_with_fixed_defaults() {
         .expect("generation request should build"),
         ImageRequest::Generate(ImageGenerationRequest {
             prompt: "paint a moonlit lake".to_string(),
-            background: Some(ImageBackground::Auto),
+            background: Some(ImageBackground::Opaque),
             model: "gpt-image-2".to_string(),
             n: None,
             quality: Some(ImageQuality::Auto),
@@ -151,6 +152,7 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {
         request_for_call_args(
             &ImagegenArgs {
                 prompt: "change the lighting".to_string(),
+                transparent_background: false,
                 referenced_image_paths: None,
                 num_last_images_to_include: Some(5),
             },
@@ -172,6 +174,7 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
     let error = request_for_call_args(
         &ImagegenArgs {
             prompt: "change the lighting".to_string(),
+            transparent_background: false,
             referenced_image_paths: None,
             num_last_images_to_include: Some(1),
         },
@@ -208,6 +211,7 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_wind
     let error = request_for_call_args(
         &ImagegenArgs {
             prompt: "change the lighting".to_string(),
+            transparent_background: false,
             referenced_image_paths: None,
             num_last_images_to_include: Some(1),
         },
@@ -252,6 +256,7 @@ async fn conflicting_image_selectors_return_tool_error() {
     let error = request_for_call_args(
         &ImagegenArgs {
             prompt: "change the lighting".to_string(),
+            transparent_background: false,
             referenced_image_paths: Some(vec![
                 "/tmp/image.png"
                     .try_into()
@@ -276,6 +281,7 @@ async fn too_many_referenced_image_paths_return_tool_error() {
     let error = request_for_call_args(
         &ImagegenArgs {
             prompt: "change the lighting".to_string(),
+            transparent_background: false,
             referenced_image_paths: Some(
                 (0..6)
                     .map(|index| {
@@ -304,6 +310,7 @@ async fn recent_image_fallback_requires_requested_count() {
     let error = request_for_call_args(
         &ImagegenArgs {
             prompt: "change the lighting".to_string(),
+            transparent_background: false,
             referenced_image_paths: None,
             num_last_images_to_include: Some(2),
         },
@@ -430,7 +437,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {
             })
             .collect(),
         prompt: prompt.to_string(),
-        background: Some(ImageBackground::Auto),
+        background: Some(ImageBackground::Opaque),
         model: "gpt-image-2".to_string(),
         n: None,
         quality: Some(ImageQuality::Auto),

codex-rs/ext/image-generation/src/tool.rs

@@ -89,6 +89,9 @@ impl ImageGenerationTool {
 #[serde(deny_unknown_fields)]
 struct ImagegenArgs {
     prompt: String,
+    /// Whether the output should have a transparent background. Defaults to false.
+    #[serde(default)]
+    transparent_background: bool,
     #[schemars(length(max = 5))]
     referenced_image_paths: Option<Vec<AbsolutePathBuf>>,
     #[schemars(range(min = 1, max = 5))]
@@ -120,7 +123,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for ImageGenerationTool {
         ToolName::namespaced(IMAGE_GEN_NAMESPACE, IMAGEGEN_TOOL_NAME)
     }
 
-    /// Advertises the model contract: a rewritten prompt and optional edit references.
+    /// Advertises a rewritten prompt, background choice, and optional edit references.
     fn spec(&self) -> ToolSpec {
         imagegen_tool_spec()
     }
@@ -427,6 +430,11 @@ async fn request_for_call_args(
     history: &[ResponseItem],
     environments: &[ToolEnvironment<'_>],
 ) -> Result<ImageRequest, FunctionCallError> {
+    let background = if args.transparent_background {
+        ImageBackground::Transparent
+    } else {
+        ImageBackground::Opaque
+    };
     let paths = args.referenced_image_paths.as_deref().unwrap_or_default();
     if paths.len() > MAX_EDIT_IMAGES {
         return Err(FunctionCallError::RespondToModel(format!(
@@ -437,7 +445,7 @@ async fn request_for_call_args(
         (true, None) => {
             return Ok(ImageRequest::Generate(ImageGenerationRequest {
                 prompt: args.prompt.clone(),
-                background: Some(ImageBackground::Auto),
+                background: Some(background),
                 model: IMAGE_MODEL.to_string(),
                 n: None,
                 quality: Some(ImageQuality::Auto),
@@ -481,7 +489,7 @@ async fn request_for_call_args(
     Ok(ImageRequest::Edit(ImageEditRequest {
         images,
         prompt: args.prompt.clone(),
-        background: Some(ImageBackground::Auto),
+        background: Some(background),
         model: IMAGE_MODEL.to_string(),
         n: None,
         quality: Some(ImageQuality::Auto),