Add a unified image budget (#37206)

## What changed

- Add the gated `unified_image_budget` feature for models that support original image detail or Responses Lite.
- Apply one 6,000-pixel, 10,000-patch preprocessing limit regardless of legacy image detail hints.
- Hide detail controls from `view_image` and code mode while continuing to accept existing hints for compatibility.
- Preserve detail-based resizing and tool contracts for unsupported models.

## Testing

- Cover unified resizing limits, Responses Lite, legacy detail hints, unsupported models, and `view_image` integration in code mode.

GitOrigin-RevId: cb07bff1669a96599fdfd076b3d9ec80f2b6fff7

codex-rs/core/tests/suite/code_mode.rs

@@ -82,6 +82,7 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 use std::time::Instant;
+use test_case::test_case;
 use wiremock::Mock;
 use wiremock::MockServer;
 use wiremock::ResponseTemplate;
@@ -3635,6 +3636,114 @@ async fn code_mode_resizes_explicit_original_image() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let image = ImageBuffer::from_pixel(
+        /*width*/ 2304,
+        /*height*/ 864,
+        Rgba([20, 40, 60, 255]),
+    );
+    let mut encoded = Cursor::new(Vec::new());
+    DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
+    let image_data = BASE64_STANDARD.encode(encoded.into_inner());
+    let code = format!(
+        r#"
+const data = {};
+const imageUrl = `data:image/png;base64,${{data}}`;
+image(imageUrl);
+image(imageUrl, "auto");
+image(imageUrl, "high");
+image({{ image_url: imageUrl, detail: "low" }});
+image({{
+  type: "image",
+  data,
+  mimeType: "image/png",
+  _meta: {{ "codex/imageDetail": "original" }}
+}});
+"#,
+        serde_json::to_string(&image_data)?
+    );
+
+    let server = responses::start_mock_server().await;
+    let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
+        &server,
+        "emit images with legacy detail arguments and MCP metadata",
+        &code,
+        "gpt-5.4",
+        |config| {
+            let _ = config.features.enable(Feature::UnifiedImageBudget);
+        },
+    )
+    .await?;
+
+    let request = second_mock.single_request();
+    let items = custom_tool_output_items(&request, "call-1");
+    assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}");
+    for item in &items[1..] {
+        assert_eq!(item["type"], "input_image");
+        assert_eq!(item["detail"], "original");
+        let image_url = item["image_url"]
+            .as_str()
+            .expect("image helper should emit a data URL");
+        let (_, payload) = image_url
+            .split_once(',')
+            .expect("image data URL should have a base64 payload");
+        let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?;
+        assert_eq!(image.dimensions(), (2304, 864));
+    }
+
+    let body = request.body_json();
+    let exec_description = body["tools"]
+        .as_array()
+        .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
+        .and_then(|tool| tool["description"].as_str())
+        .expect("the model request should contain the code-mode exec tool");
+    assert!(
+        exec_description
+            .contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`")
+    );
+    assert!(!exec_description.contains("codex/imageDetail"));
+    assert!(!exec_description.contains("detail?:"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model()
+-> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let server = responses::start_mock_server().await;
+    let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
+        &server,
+        "emit an image on a legacy model",
+        r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#,
+        "gpt-5.2",
+        |config| {
+            let _ = config.features.enable(Feature::UnifiedImageBudget);
+        },
+    )
+    .await?;
+
+    let request = second_mock.single_request();
+    let items = custom_tool_output_items(&request, "call-1");
+    assert_eq!(items.len(), 2);
+    assert_eq!(items[1]["detail"], "high");
+
+    let body = request.body_json();
+    let exec_description = body["tools"]
+        .as_array()
+        .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
+        .and_then(|tool| tool["description"].as_str())
+        .expect("the model request should contain the code-mode exec tool");
+    assert!(exec_description.contains("codex/imageDetail"));
+    assert!(exec_description.contains("detail?:"));
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn code_mode_image_helper_rejects_remote_url() -> Result<()> {
     skip_if_no_network!(Ok(()));
@@ -3718,8 +3827,12 @@ image(s.trim(), "original");
     Ok(())
 }
 
+#[test_case(false; "legacy detail")]
+#[test_case(true; "unified image budget")]
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
+async fn code_mode_can_use_view_image_result_with_image_helper(
+    unified_image_budget: bool,
+) -> Result<()> {
     skip_if_no_network!(Ok(()));
 
     let server = responses::start_mock_server().await;
@@ -3727,6 +3840,9 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
         .with_model("gpt-5.4")
         .with_config(move |config| {
             let _ = config.features.enable(Feature::CodeMode);
+            if unified_image_budget {
+                let _ = config.features.enable(Feature::UnifiedImageBudget);
+            }
         });
     let test = builder.build(&server).await?;
 
@@ -3737,9 +3853,18 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
     fs::write(&image_path, image_bytes)?;
 
     let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;
+    let expected_output_keys = if unified_image_budget {
+        r#"["image_url"]"#
+    } else {
+        r#"["detail","image_url"]"#
+    };
     let code = format!(
         r#"
 const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }});
+const keys = Object.keys(out).sort();
+if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{
+  throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`);
+}}
 image(out);
 "#
     );

codex-rs/core/tests/suite/view_image.rs

@@ -73,6 +73,13 @@ enum ResizeNoticeExpectation {
     Enabled,
 }
 
+#[derive(Clone, Copy, Eq, PartialEq)]
+enum ImageBudgetPolicy {
+    DetailBased,
+    Unified,
+    UnifiedResponsesLiteWithoutOriginalSupport,
+}
+
 fn disabled_user_turn(test: &TestCodex, items: Vec<UserInput>, model: String) -> Op {
     let (sandbox_policy, permission_profile) =
         turn_permission_fields(PermissionProfile::Disabled, test.config.cwd.as_path());
@@ -204,17 +211,27 @@ async fn write_workspace_png(
 async fn assert_user_turn_local_image_resizes_to(
     original_dimensions: (u32, u32),
     expected_dimensions: (u32, u32),
+    image_budget_policy: ImageBudgetPolicy,
     resize_notice_expectation: ResizeNoticeExpectation,
 ) -> anyhow::Result<()> {
     let server = start_mock_server().await;
 
-    let builder = test_codex();
-    let mut builder = match resize_notice_expectation {
-        ResizeNoticeExpectation::Disabled => builder,
-        ResizeNoticeExpectation::Enabled => builder.with_config(|config| {
-            let _ = config.features.enable(Feature::ImageResizeNotice);
-        }),
+    let builder = match image_budget_policy {
+        ImageBudgetPolicy::DetailBased | ImageBudgetPolicy::Unified => test_codex(),
+        ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport => test_codex()
+            .with_model_info_override("gpt-5.4", |model_info| {
+                model_info.supports_image_detail_original = false;
+                model_info.use_responses_lite = true;
+            }),
     };
+    let mut builder = builder.with_config(move |config| {
+        if image_budget_policy != ImageBudgetPolicy::DetailBased {
+            let _ = config.features.enable(Feature::UnifiedImageBudget);
+        }
+        if matches!(resize_notice_expectation, ResizeNoticeExpectation::Enabled) {
+            let _ = config.features.enable(Feature::ImageResizeNotice);
+        }
+    });
     let test = builder.build_with_auto_env(&server).await?;
     let TestCodex {
         codex,
@@ -329,6 +346,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
     assert_user_turn_local_image_resizes_to(
         (2304, 864),
         (2048, 768),
+        ImageBudgetPolicy::DetailBased,
         ResizeNoticeExpectation::Disabled,
     )
     .await
@@ -341,6 +359,7 @@ async fn user_turn_with_vertical_local_image_resizes_to_square_bounds() -> anyho
     assert_user_turn_local_image_resizes_to(
         (1024, 4096),
         (512, 2048),
+        ImageBudgetPolicy::DetailBased,
         ResizeNoticeExpectation::Disabled,
     )
     .await
@@ -353,11 +372,47 @@ async fn user_turn_local_image_applies_patch_budget_and_reports_resize() -> anyh
     assert_user_turn_local_image_resizes_to(
         (2048, 2048),
         (1600, 1600),
+        ImageBudgetPolicy::DetailBased,
         ResizeNoticeExpectation::Enabled,
     )
     .await
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn user_turn_unified_image_budget_enforces_dimension_and_patch_limits() -> anyhow::Result<()>
+{
+    skip_if_no_network!(Ok(()));
+
+    for (source_dimensions, expected_dimensions, resize_notice_expectation) in [
+        ((6401, 100), (6000, 94), ResizeNoticeExpectation::Disabled),
+        ((3201, 3201), (3200, 3200), ResizeNoticeExpectation::Enabled),
+    ] {
+        assert_user_turn_local_image_resizes_to(
+            source_dimensions,
+            expected_dimensions,
+            ImageBudgetPolicy::Unified,
+            resize_notice_expectation,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn user_turn_unified_image_budget_supports_responses_lite_without_original_detail()
+-> anyhow::Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    assert_user_turn_local_image_resizes_to(
+        (2304, 864),
+        (2304, 864),
+        ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport,
+        ResizeNoticeExpectation::Disabled,
+    )
+    .await
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
     skip_if_no_network!(Ok(()));
@@ -869,6 +924,82 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn view_image_unified_budget_hides_detail_but_accepts_legacy_hints() -> anyhow::Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let server = start_mock_server().await;
+    let mut builder = test_codex().with_model("gpt-5.4").with_config(|config| {
+        let _ = config.features.enable(Feature::UnifiedImageBudget);
+    });
+    let test = builder.build_with_auto_env(&server).await?;
+    let rel_path = "assets/unified-example.png";
+    write_workspace_png(
+        &test,
+        rel_path,
+        /*width*/ 2304,
+        /*height*/ 864,
+        [0u8, 80, 255, 255],
+    )
+    .await?;
+
+    let call_id = "view-image-unified";
+    let first_mock = responses::mount_sse_once(
+        &server,
+        sse(vec![
+            ev_response_created("resp-1"),
+            ev_function_call(
+                call_id,
+                "view_image",
+                &serde_json::json!({ "path": rel_path, "detail": "high" }).to_string(),
+            ),
+            ev_completed("resp-1"),
+        ]),
+    )
+    .await;
+    let second_mock = responses::mount_sse_once(
+        &server,
+        sse(vec![
+            ev_assistant_message("msg-1", "done"),
+            ev_completed("resp-2"),
+        ]),
+    )
+    .await;
+
+    test.submit_turn("show the screenshot").await?;
+
+    let first_request = first_mock.single_request().body_json();
+    let view_image_tool = first_request["tools"]
+        .as_array()
+        .and_then(|tools| tools.iter().find(|tool| tool["name"] == "view_image"))
+        .context("view_image tool should be available")?;
+    assert!(
+        view_image_tool["parameters"]["properties"]
+            .get("detail")
+            .is_none(),
+        "the unified image budget should not advertise detail"
+    );
+
+    let request = second_mock.single_request();
+    let output = request.function_call_output(call_id);
+    let output_items = output["output"]
+        .as_array()
+        .context("view_image should return image content")?;
+    assert_eq!(output_items.len(), 1);
+    assert_eq!(output_items[0]["detail"], "original");
+
+    let image_url = output_items[0]["image_url"]
+        .as_str()
+        .context("view_image output should include image_url")?;
+    let (_, payload) = image_url
+        .split_once(',')
+        .context("view_image image_url should include a base64 payload")?;
+    let image = load_from_memory(&BASE64_STANDARD.decode(payload)?)?;
+    assert_eq!(image.dimensions(), (2304, 864));
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyhow::Result<()> {
     skip_if_no_network!(Ok(()));
@@ -1041,8 +1172,27 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
 async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> {
     skip_if_no_network!(Ok(()));
 
+    assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::DetailBased).await
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn view_image_unified_budget_stays_disabled_for_unsupported_model() -> anyhow::Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::Unified).await
+}
+
+async fn assert_view_image_tool_resizes_without_original_support(
+    image_budget_policy: ImageBudgetPolicy,
+) -> anyhow::Result<()> {
     let server = start_mock_server().await;
-    let mut builder = test_codex().with_model("gpt-5.2");
+    let mut builder = test_codex()
+        .with_model("gpt-5.2")
+        .with_config(move |config| {
+            if image_budget_policy == ImageBudgetPolicy::Unified {
+                let _ = config.features.enable(Feature::UnifiedImageBudget);
+            }
+        });
     let test = builder.build_with_auto_env(&server).await?;
     let TestCodex {
         codex,