github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-20 00:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a unified image budget (#37206)

Browse Source 
## What changed

- Add the gated `unified_image_budget` feature for models that support original image detail or Responses Lite.
- Apply one 6,000-pixel, 10,000-patch preprocessing limit regardless of legacy image detail hints.
- Hide detail controls from `view_image` and code mode while continuing to accept existing hints for compatibility.
- Preserve detail-based resizing and tool contracts for unsupported models.

## Testing

- Cover unified resizing limits, Responses Lite, legacy detail hints, unsupported models, and `view_image` integration in code mode.

GitOrigin-RevId: cb07bff1669a96599fdfd076b3d9ec80f2b6fff7
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-08-06 03:15:08 +00:00
committed by copyberry
parent bc8b25ea02
commit 0a0ebb8535
13 changed files with 466 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/code-mode-protocol/src/description.rs
+23
View File
@@ -9,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME;
const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1;
const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some deferred nested tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`.
To find one, filter `ALL_TOOLS` by `name` and `description`."#;
const LEGACY_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument."#;
const UNIFIED_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`."#;
const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls
- Evaluates the provided JavaScript code in a fresh V8 isolate as an async module.
- All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`.
@@ -249,18 +251,31 @@ pub fn is_code_mode_nested_tool(tool_name: &str) -> bool {
tool_name != crate::PUBLIC_TOOL_NAME && tool_name != crate::WAIT_TOOL_NAME
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ImageDetailVisibility {
Visible,
Hidden,
}
pub fn build_exec_tool_description(
enabled_tools: &[ToolDefinition],
deferred_tools: &[ToolDefinition],
namespace_descriptions: &BTreeMap<String, ToolNamespaceDescription>,
default_exec_yield_time_ms: u64,
code_mode_only: bool,
image_detail_visibility: ImageDetailVisibility,
) -> String {
let mut sections = Vec::new();
sections.push(EXEC_DESCRIPTION_TEMPLATE.replace(
"Defaults to 10000 ms.",
&format!("Defaults to {default_exec_yield_time_ms} ms."),
));
if image_detail_visibility == ImageDetailVisibility::Hidden {
sections[0] = sections[0].replace(
LEGACY_IMAGE_HELPER_DESCRIPTION,
UNIFIED_IMAGE_HELPER_DESCRIPTION,
);
}
if !deferred_tools.is_empty() {
sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string());
}
@@ -714,6 +729,7 @@ fn render_json_schema_literal(value: &JsonValue) -> String {
#[cfg(test)]
mod tests {
use super::CodeModeToolKind;
use super::ImageDetailVisibility;
use super::ParsedExecSource;
use super::ToolDefinition;
use super::ToolNamespaceDescription;
@@ -874,6 +890,7 @@ mod tests {
&BTreeMap::new(),
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
);
assert!(description.contains(
"### `foo`
@@ -890,6 +907,7 @@ bar"
&BTreeMap::new(),
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ false,
ImageDetailVisibility::Visible,
);
assert!(description.contains("`audio(audioUrlOrItem:"));
assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`"));
@@ -944,6 +962,7 @@ bar"
&namespace_descriptions,
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
);
assert_eq!(description.matches("## mcp__sample").count(), 1);
assert!(description.contains("## mcp__sample\nShared namespace guidance."));
@@ -985,6 +1004,7 @@ bar"
&namespace_descriptions,
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
);
assert!(!description.contains("## mcp__sample"));
@@ -1085,6 +1105,7 @@ bar"
&BTreeMap::new(),
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
);
assert_eq!(
@@ -1121,6 +1142,7 @@ bar"
&BTreeMap::new(),
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
);
assert!(description.contains("Some deferred nested tools may be omitted"));
@@ -1143,6 +1165,7 @@ bar"
&BTreeMap::new(),
crate::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ false,
ImageDetailVisibility::Visible,
);
assert!(description.contains("Some deferred nested tools may be omitted"));
codex-rs/code-mode-protocol/src/lib.rs
+1
View File
@@ -7,6 +7,7 @@ mod session;
pub use description::CODE_MODE_PRAGMA_PREFIX;
pub use description::CodeModeToolKind;
pub use description::EnabledToolMetadata;
pub use description::ImageDetailVisibility;
pub use description::ToolDefinition;
pub use description::ToolNamespaceDescription;
pub use description::augment_tool_definition;
codex-rs/core/config.schema.json
+6
View File
@@ -756,6 +756,9 @@
"unified_exec_zsh_fork": {
"type": "boolean"
},
"unified_image_budget": {
"type": "boolean"
},
"use_agent_identity": {
"type": "boolean"
},
@@ -5387,6 +5390,9 @@
"unified_exec_zsh_fork": {
"type": "boolean"
},
"unified_image_budget": {
"type": "boolean"
},
"use_agent_identity": {
"type": "boolean"
},
codex-rs/core/src/image_preparation.rs
+42 -10
View File
@@ -1,13 +1,17 @@
use crate::config::ManagedFeatures;
use crate::context::ContextualUserFragment;
use crate::context::ImageResizeNotice;
use crate::context::ImageResizeNoticeSource;
use crate::context::ResizedImage;
use crate::original_image_detail::can_request_original_image_detail;
use codex_analytics::ImageDetailSetting;
use codex_analytics::ImagePreparationMetadata;
use codex_features::Feature;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_utils_image::ImageProcessingError;
use codex_utils_image::PromptImageMode;
use codex_utils_image::PromptImageResizeLimits;
@@ -26,11 +30,25 @@ const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 2048,
max_patches: 2_500,
};
const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
const UNIFIED_IMAGE_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 6000,
max_patches: 10_000,
};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ImagePreparationMode {
DetailBased,
UnifiedBudget,
}
pub(crate) fn unified_image_budget_enabled(
features: &ManagedFeatures,
model_info: &ModelInfo,
) -> bool {
features.enabled(Feature::UnifiedImageBudget)
&& (model_info.use_responses_lite || can_request_original_image_detail(model_info))
}
#[derive(Clone, Copy, Debug)]
struct ImageOrigin<'a> {
message_role: Option<&'a str>,
@@ -76,6 +94,7 @@ impl ImagePreparationError {
pub(crate) fn prepare_response_items(
items: &mut Vec<ResponseItem>,
mode: ImagePreparationMode,
resize_notice_mode: ImageResizeNoticeMode,
) -> Vec<ImagePreparationMetadata> {
let mut metadata = Vec::new();
@@ -95,6 +114,7 @@ pub(crate) fn prepare_response_items(
ImageResizeNoticeMode::Disabled
},
&mut metadata,
mode,
);
(!resized_images.is_empty()).then(|| {
ImageResizeNotice::new(ImageResizeNoticeSource::UserMessage, resized_images)
@@ -114,6 +134,7 @@ pub(crate) fn prepare_response_items(
},
resize_notice_mode,
&mut metadata,
mode,
);
(!resized_images.is_empty()).then(|| {
ImageResizeNotice::new(ImageResizeNoticeSource::ToolOutput, resized_images)
@@ -148,6 +169,7 @@ fn prepare_message_content(
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
) -> Vec<ResizedImage> {
let image_count = items
.iter()
@@ -158,7 +180,7 @@ fn prepare_message_content(
for item in items {
if let ContentItem::InputImage { image_url, detail } = item {
image_number += 1;
match prepare_image(image_url, *detail, origin, metadata) {
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -187,6 +209,7 @@ fn prepare_tool_output_content(
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
) -> Vec<ResizedImage> {
let image_count = items
.iter()
@@ -197,7 +220,7 @@ fn prepare_tool_output_content(
for item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item {
image_number += 1;
match prepare_image(image_url, *detail, origin, metadata) {
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -235,9 +258,10 @@ fn is_data_url(image_url: &str) -> bool {
fn prepare_image(
image_url: &mut String,
detail: Option<ImageDetail>,
detail: &mut Option<ImageDetail>,
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
) -> Result<Option<PreparedImageResize>, ImagePreparationError> {
if is_remote_image_url(image_url) {
return Err(ImagePreparationError::RemoteUrlUnsupported);
@@ -246,12 +270,15 @@ fn prepare_image(
return Ok(None);
}
let (effective_detail, limits) = match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => {
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
}
Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS),
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
let (effective_detail, limits) = match mode {
ImagePreparationMode::UnifiedBudget => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),
ImagePreparationMode::DetailBased => match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => {
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
}
Some(ImageDetail::Original) => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
},
};
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
metadata.push(ImagePreparationMetadata {
@@ -271,6 +298,11 @@ fn prepare_image(
prepared_height: image.height,
});
*image_url = image.into_data_url();
if mode == ImagePreparationMode::UnifiedBudget {
// Preserve accurate context-window accounting while older transports still require an
// image detail field. Responses Lite removes this compatibility hint before sending.
*detail = Some(ImageDetail::Original);
}
Ok(resize)
}
codex-rs/core/src/image_preparation_tests.rs
+25 -5
View File
@@ -51,7 +51,11 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
internal_chat_message_metadata_passthrough: None,
}];
prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);
prepare_response_items(
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
);
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
@@ -105,7 +109,11 @@ fn detail_policies_apply_the_expected_budgets() {
internal_chat_message_metadata_passthrough: None,
}];
let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);
let metadata = prepare_response_items(
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
);
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
@@ -144,7 +152,11 @@ fn preparation_reports_tool_output_item_id() {
]),
internal_chat_message_metadata_passthrough: None,
}];
let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);
let metadata = prepare_response_items(
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
);
assert_eq!(
metadata,
@@ -202,7 +214,11 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
},
];
prepare_response_items(&mut items, ImageResizeNoticeMode::Enabled);
prepare_response_items(
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Enabled,
);
let expected_user_notice = concat!(
"<image_resize_notice>\n",
"Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
@@ -325,7 +341,11 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
internal_chat_message_metadata_passthrough: None,
}];
prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);
prepare_response_items(
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
);
assert_eq!(
items,
codex-rs/core/src/session/mod.rs
+26 -10
View File
@@ -31,8 +31,10 @@ use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::exec_policy::BANNED_PREFIX_SUGGESTIONS;
use crate::exec_policy::ExecPolicyManager;
use crate::exec_policy::default_policy_path;
use crate::image_preparation::ImagePreparationMode;
use crate::image_preparation::ImageResizeNoticeMode;
use crate::image_preparation::prepare_response_items as prepare_image_response_items;
use crate::image_preparation::unified_image_budget_enabled;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::session::step_context::StepContext;
@@ -1445,7 +1447,11 @@ impl Session {
// will be processed again if the rollout is reconstructed in a future session.
// Never backfill resize notices during replay; only newly recorded items may
// emit them, so the historical model prefix remains unchanged.
let _ = prepare_image_response_items(&mut history, ImageResizeNoticeMode::Disabled);
let _ = prepare_image_response_items(
&mut history,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
);
prepare_audio_response_items(&mut history);
{
let mut state = self.state.lock().await;
@@ -2911,17 +2917,27 @@ impl Session {
items: &'a [ResponseItem],
) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {
let mut items = items.to_vec();
let image_preparation_mode = if unified_image_budget_enabled(
&turn_context.config.features,
&turn_context.model_info,
) {
ImagePreparationMode::UnifiedBudget
} else {
ImagePreparationMode::DetailBased
};
let image_resize_notice_mode = if turn_context
.config
.features
.enabled(Feature::ImageResizeNotice)
{
ImageResizeNoticeMode::Enabled
} else {
ImageResizeNoticeMode::Disabled
};
let image_preparations = prepare_image_response_items(
&mut items,
if turn_context
.config
.features
.enabled(Feature::ImageResizeNotice)
{
ImageResizeNoticeMode::Enabled
} else {
ImageResizeNoticeMode::Disabled
},
image_preparation_mode,
image_resize_notice_mode,
);
prepare_audio_response_items(&mut items);
// Most response items get their passthrough turn ID at the durable history boundary.
codex-rs/core/src/tools/code_mode/execute_spec.rs
+5
View File
@@ -1,3 +1,4 @@
use codex_code_mode::ImageDetailVisibility;
use codex_code_mode::ToolDefinition as CodeModeToolDefinition;
use codex_tools::FreeformTool;
use codex_tools::FreeformToolFormat;
@@ -10,6 +11,7 @@ pub(crate) fn create_code_mode_tool(
namespace_descriptions: &BTreeMap<String, codex_code_mode::ToolNamespaceDescription>,
default_exec_yield_time_ms: u64,
code_mode_only: bool,
image_detail_visibility: ImageDetailVisibility,
) -> ToolSpec {
const CODE_MODE_FREEFORM_GRAMMAR: &str = r#"
start: pragma_source | plain_source
@@ -29,6 +31,7 @@ SOURCE: /[\s\S]+/
namespace_descriptions,
default_exec_yield_time_ms,
code_mode_only,
image_detail_visibility,
),
defer_loading: None,
format: FreeformToolFormat {
@@ -63,6 +66,7 @@ mod tests {
&BTreeMap::new(),
codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
),
ToolSpec::Freeform(FreeformTool {
name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(),
@@ -72,6 +76,7 @@ mod tests {
&BTreeMap::new(),
codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
ImageDetailVisibility::Visible,
),
defer_loading: None,
format: FreeformToolFormat {
codex-rs/core/src/tools/handlers/view_image.rs
+16 -8
View File
@@ -34,6 +34,7 @@ impl Default for ViewImageHandler {
Self {
options: ViewImageToolOptions {
can_request_original_image_detail: false,
unified_image_budget: false,
include_environment_id: false,
},
}
@@ -120,8 +121,7 @@ impl ViewImageHandler {
environment_id,
detail,
} = parse_arguments(&arguments)?;
// `high` is the explicit spelling of the default resized path.
// Other string values remain invalid rather than being silently reinterpreted.
// Keep accepting previously supported detail hints after they disappear from the schema.
let detail = match detail.as_deref() {
None => None,
Some("high") => Some(ViewImageDetail::High),
@@ -175,8 +175,8 @@ impl ViewImageHandler {
})?;
let can_request_original_detail = can_request_original_image_detail(&turn.model_info);
let use_original_detail =
can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));
let use_original_detail = self.options.unified_image_budget
|| can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));
let image_detail = if use_original_detail {
ImageDetail::Original
} else {
@@ -196,6 +196,7 @@ impl ViewImageHandler {
Ok(boxed_tool_output(ViewImageOutput {
image_url,
image_detail,
unified_image_budget: self.options.unified_image_budget,
}))
}
}
@@ -205,6 +206,7 @@ impl CoreToolRuntime for ViewImageHandler {}
pub struct ViewImageOutput {
image_url: String,
image_detail: ImageDetail,
unified_image_budget: bool,
}
impl ToolOutput for ViewImageOutput {
@@ -234,10 +236,14 @@ impl ToolOutput for ViewImageOutput {
}
fn code_mode_result(&self, _payload: &ToolPayload) -> serde_json::Value {
serde_json::json!({
"image_url": self.image_url,
"detail": self.image_detail
})
if self.unified_image_budget {
serde_json::json!({ "image_url": self.image_url })
} else {
serde_json::json!({
"image_url": self.image_url,
"detail": self.image_detail
})
}
}
}
@@ -283,6 +289,7 @@ mod tests {
let output = ViewImageOutput {
image_url: "data:image/png;base64,AAA".to_string(),
image_detail: DEFAULT_IMAGE_DETAIL,
unified_image_budget: false,
};
assert_eq!(output.log_preview(), "<image data URL omitted: 25 bytes>");
@@ -293,6 +300,7 @@ mod tests {
let output = ViewImageOutput {
image_url: "data:image/png;base64,AAA".to_string(),
image_detail: DEFAULT_IMAGE_DETAIL,
unified_image_budget: false,
};
let result = output.code_mode_result(&ToolPayload::Function {
codex-rs/core/src/tools/handlers/view_image_spec.rs
+16 -11
View File
@@ -9,6 +9,7 @@ use std::collections::BTreeMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ViewImageToolOptions {
pub can_request_original_image_detail: bool,
pub unified_image_budget: bool,
pub include_environment_id: bool,
}
@@ -17,7 +18,7 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec {
"path".to_string(),
JsonSchema::string(Some("Local filesystem path to an image file.".to_string())),
)]);
if options.can_request_original_image_detail {
if options.can_request_original_image_detail && !options.unified_image_budget {
properties.insert(
"detail".to_string(),
JsonSchema::string_enum(
@@ -45,25 +46,29 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec {
strict: false,
defer_loading: None,
parameters: JsonSchema::object(properties, Some(vec!["path".to_string()]), Some(false.into())),
output_schema: Some(view_image_output_schema()),
output_schema: Some(view_image_output_schema(options)),
})
}
fn view_image_output_schema() -> Value {
json!({
fn view_image_output_schema(options: ViewImageToolOptions) -> Value {
let mut schema = json!({
"type": "object",
"properties": {
"image_url": {
"type": "string",
"description": "Data URL for the loaded image."
},
"detail": {
"type": "string",
"enum": ["high", "original"],
"description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved."
}
},
"required": ["image_url", "detail"],
"required": ["image_url"],
"additionalProperties": false
})
});
if !options.unified_image_budget {
schema["properties"]["detail"] = json!({
"type": "string",
"enum": ["high", "original"],
"description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved."
});
schema["required"] = json!(["image_url", "detail"]);
}
schema
}
codex-rs/core/src/tools/spec_plan.rs
+15
View File
@@ -1,6 +1,7 @@
use crate::agent::exceeds_thread_spawn_depth_limit;
use crate::agent::next_thread_spawn_depth;
use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::image_preparation::unified_image_budget_enabled;
use crate::mcp_tool_exposure::append_mcp_tools;
use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
@@ -730,6 +731,12 @@ fn register_code_mode_executors(
&namespace_descriptions,
default_exec_yield_time_ms,
tool_mode == ToolMode::CodeModeOnly,
if unified_image_budget_enabled(&turn_context.config.features, &turn_context.model_info)
{
codex_code_mode::ImageDetailVisibility::Hidden
} else {
codex_code_mode::ImageDetailVisibility::Visible
},
),
code_mode_nested_tool_specs,
);
@@ -838,6 +845,10 @@ fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolR
can_request_original_image_detail: can_request_original_image_detail(
&turn_context.model_info,
),
unified_image_budget: unified_image_budget_enabled(
&turn_context.config.features,
&turn_context.model_info,
),
include_environment_id,
}));
}
@@ -1034,6 +1045,10 @@ fn add_core_utility_tools(context: &CoreToolPlanContext<'_>, registry: &mut Tool
can_request_original_image_detail: can_request_original_image_detail(
&turn_context.model_info,
),
unified_image_budget: unified_image_budget_enabled(
&turn_context.config.features,
&turn_context.model_info,
),
include_environment_id,
}));
}
codex-rs/core/tests/suite/code_mode.rs
+126 -1
View File
@@ -82,6 +82,7 @@ use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use std::time::Instant;
use test_case::test_case;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
@@ -3635,6 +3636,114 @@ async fn code_mode_resizes_explicit_original_image() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> {
skip_if_no_network!(Ok(()));
let image = ImageBuffer::from_pixel(
/*width*/ 2304,
/*height*/ 864,
Rgba([20, 40, 60, 255]),
);
let mut encoded = Cursor::new(Vec::new());
DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;
let image_data = BASE64_STANDARD.encode(encoded.into_inner());
let code = format!(
r#"
const data = {};
const imageUrl = `data:image/png;base64,${{data}}`;
image(imageUrl);
image(imageUrl, "auto");
image(imageUrl, "high");
image({{ image_url: imageUrl, detail: "low" }});
image({{
type: "image",
data,
mimeType: "image/png",
_meta: {{ "codex/imageDetail": "original" }}
}});
"#,
serde_json::to_string(&image_data)?
);
let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"emit images with legacy detail arguments and MCP metadata",
&code,
"gpt-5.4",
|config| {
let _ = config.features.enable(Feature::UnifiedImageBudget);
},
)
.await?;
let request = second_mock.single_request();
let items = custom_tool_output_items(&request, "call-1");
assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}");
for item in &items[1..] {
assert_eq!(item["type"], "input_image");
assert_eq!(item["detail"], "original");
let image_url = item["image_url"]
.as_str()
.expect("image helper should emit a data URL");
let (_, payload) = image_url
.split_once(',')
.expect("image data URL should have a base64 payload");
let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?;
assert_eq!(image.dimensions(), (2304, 864));
}
let body = request.body_json();
let exec_description = body["tools"]
.as_array()
.and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
.and_then(|tool| tool["description"].as_str())
.expect("the model request should contain the code-mode exec tool");
assert!(
exec_description
.contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`")
);
assert!(!exec_description.contains("codex/imageDetail"));
assert!(!exec_description.contains("detail?:"));
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model()
-> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"emit an image on a legacy model",
r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#,
"gpt-5.2",
|config| {
let _ = config.features.enable(Feature::UnifiedImageBudget);
},
)
.await?;
let request = second_mock.single_request();
let items = custom_tool_output_items(&request, "call-1");
assert_eq!(items.len(), 2);
assert_eq!(items[1]["detail"], "high");
let body = request.body_json();
let exec_description = body["tools"]
.as_array()
.and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))
.and_then(|tool| tool["description"].as_str())
.expect("the model request should contain the code-mode exec tool");
assert!(exec_description.contains("codex/imageDetail"));
assert!(exec_description.contains("detail?:"));
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_image_helper_rejects_remote_url() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -3718,8 +3827,12 @@ image(s.trim(), "original");
Ok(())
}
#[test_case(false; "legacy detail")]
#[test_case(true; "unified image budget")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
async fn code_mode_can_use_view_image_result_with_image_helper(
unified_image_budget: bool,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
@@ -3727,6 +3840,9 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
.with_model("gpt-5.4")
.with_config(move |config| {
let _ = config.features.enable(Feature::CodeMode);
if unified_image_budget {
let _ = config.features.enable(Feature::UnifiedImageBudget);
}
});
let test = builder.build(&server).await?;
@@ -3737,9 +3853,18 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {
fs::write(&image_path, image_bytes)?;
let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;
let expected_output_keys = if unified_image_budget {
r#"["image_url"]"#
} else {
r#"["detail","image_url"]"#
};
let code = format!(
r#"
const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }});
const keys = Object.keys(out).sort();
if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{
throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`);
}}
image(out);
"#
);
codex-rs/core/tests/suite/view_image.rs
+157 -7
View File
@@ -73,6 +73,13 @@ enum ResizeNoticeExpectation {
Enabled,
}
#[derive(Clone, Copy, Eq, PartialEq)]
enum ImageBudgetPolicy {
DetailBased,
Unified,
UnifiedResponsesLiteWithoutOriginalSupport,
}
fn disabled_user_turn(test: &TestCodex, items: Vec<UserInput>, model: String) -> Op {
let (sandbox_policy, permission_profile) =
turn_permission_fields(PermissionProfile::Disabled, test.config.cwd.as_path());
@@ -204,17 +211,27 @@ async fn write_workspace_png(
async fn assert_user_turn_local_image_resizes_to(
original_dimensions: (u32, u32),
expected_dimensions: (u32, u32),
image_budget_policy: ImageBudgetPolicy,
resize_notice_expectation: ResizeNoticeExpectation,
) -> anyhow::Result<()> {
let server = start_mock_server().await;
let builder = test_codex();
let mut builder = match resize_notice_expectation {
ResizeNoticeExpectation::Disabled => builder,
ResizeNoticeExpectation::Enabled => builder.with_config(|config| {
let _ = config.features.enable(Feature::ImageResizeNotice);
}),
let builder = match image_budget_policy {
ImageBudgetPolicy::DetailBased | ImageBudgetPolicy::Unified => test_codex(),
ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport => test_codex()
.with_model_info_override("gpt-5.4", |model_info| {
model_info.supports_image_detail_original = false;
model_info.use_responses_lite = true;
}),
};
let mut builder = builder.with_config(move |config| {
if image_budget_policy != ImageBudgetPolicy::DetailBased {
let _ = config.features.enable(Feature::UnifiedImageBudget);
}
if matches!(resize_notice_expectation, ResizeNoticeExpectation::Enabled) {
let _ = config.features.enable(Feature::ImageResizeNotice);
}
});
let test = builder.build_with_auto_env(&server).await?;
let TestCodex {
codex,
@@ -329,6 +346,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
assert_user_turn_local_image_resizes_to(
(2304, 864),
(2048, 768),
ImageBudgetPolicy::DetailBased,
ResizeNoticeExpectation::Disabled,
)
.await
@@ -341,6 +359,7 @@ async fn user_turn_with_vertical_local_image_resizes_to_square_bounds() -> anyho
assert_user_turn_local_image_resizes_to(
(1024, 4096),
(512, 2048),
ImageBudgetPolicy::DetailBased,
ResizeNoticeExpectation::Disabled,
)
.await
@@ -353,11 +372,47 @@ async fn user_turn_local_image_applies_patch_budget_and_reports_resize() -> anyh
assert_user_turn_local_image_resizes_to(
(2048, 2048),
(1600, 1600),
ImageBudgetPolicy::DetailBased,
ResizeNoticeExpectation::Enabled,
)
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn user_turn_unified_image_budget_enforces_dimension_and_patch_limits() -> anyhow::Result<()>
{
skip_if_no_network!(Ok(()));
for (source_dimensions, expected_dimensions, resize_notice_expectation) in [
((6401, 100), (6000, 94), ResizeNoticeExpectation::Disabled),
((3201, 3201), (3200, 3200), ResizeNoticeExpectation::Enabled),
] {
assert_user_turn_local_image_resizes_to(
source_dimensions,
expected_dimensions,
ImageBudgetPolicy::Unified,
resize_notice_expectation,
)
.await?;
}
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn user_turn_unified_image_budget_supports_responses_lite_without_original_detail()
-> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
assert_user_turn_local_image_resizes_to(
(2304, 864),
(2304, 864),
ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport,
ResizeNoticeExpectation::Disabled,
)
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
@@ -869,6 +924,82 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_unified_budget_hides_detail_but_accepts_legacy_hints() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let mut builder = test_codex().with_model("gpt-5.4").with_config(|config| {
let _ = config.features.enable(Feature::UnifiedImageBudget);
});
let test = builder.build_with_auto_env(&server).await?;
let rel_path = "assets/unified-example.png";
write_workspace_png(
&test,
rel_path,
/*width*/ 2304,
/*height*/ 864,
[0u8, 80, 255, 255],
)
.await?;
let call_id = "view-image-unified";
let first_mock = responses::mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-1"),
ev_function_call(
call_id,
"view_image",
&serde_json::json!({ "path": rel_path, "detail": "high" }).to_string(),
),
ev_completed("resp-1"),
]),
)
.await;
let second_mock = responses::mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-1", "done"),
ev_completed("resp-2"),
]),
)
.await;
test.submit_turn("show the screenshot").await?;
let first_request = first_mock.single_request().body_json();
let view_image_tool = first_request["tools"]
.as_array()
.and_then(|tools| tools.iter().find(|tool| tool["name"] == "view_image"))
.context("view_image tool should be available")?;
assert!(
view_image_tool["parameters"]["properties"]
.get("detail")
.is_none(),
"the unified image budget should not advertise detail"
);
let request = second_mock.single_request();
let output = request.function_call_output(call_id);
let output_items = output["output"]
.as_array()
.context("view_image should return image content")?;
assert_eq!(output_items.len(), 1);
assert_eq!(output_items[0]["detail"], "original");
let image_url = output_items[0]["image_url"]
.as_str()
.context("view_image output should include image_url")?;
let (_, payload) = image_url
.split_once(',')
.context("view_image image_url should include a base64 payload")?;
let image = load_from_memory(&BASE64_STANDARD.decode(payload)?)?;
assert_eq!(image.dimensions(), (2304, 864));
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
@@ -1041,8 +1172,27 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::DetailBased).await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_unified_budget_stays_disabled_for_unsupported_model() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::Unified).await
}
async fn assert_view_image_tool_resizes_without_original_support(
image_budget_policy: ImageBudgetPolicy,
) -> anyhow::Result<()> {
let server = start_mock_server().await;
let mut builder = test_codex().with_model("gpt-5.2");
let mut builder = test_codex()
.with_model("gpt-5.2")
.with_config(move |config| {
if image_budget_policy == ImageBudgetPolicy::Unified {
let _ = config.features.enable(Feature::UnifiedImageBudget);
}
});
let test = builder.build_with_auto_env(&server).await?;
let TestCodex {
codex,
codex-rs/features/src/lib.rs
+8
View File
@@ -224,6 +224,8 @@ pub enum Feature {
ImageGeneration,
/// Tell the model when a prompt image was resized and include its dimensions.
ImageResizeNotice,
/// Apply one shared pixel and token budget to every image, regardless of legacy detail hints.
UnifiedImageBudget,
/// Removed compatibility flag for always-on centralized image preparation.
ResizeAllImages,
/// Removed compatibility flag for always-on response item IDs.
@@ -1276,6 +1278,12 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::UnifiedImageBudget,
key: "unified_image_budget",
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::ResizeAllImages,
key: "resize_all_images",