Centralize prompt image detail modes (#42624)

## What changed

- Add `PromptImageMode::HIGH_DETAIL` and `PromptImageMode::ORIGINAL_DETAIL`
  constants with the standard resize limits.
- Use the shared modes during core image preparation instead of defining the
  limits locally.
- Cover the dimension and patch budgets for both detail modes in the image
  utility tests.

GitOrigin-RevId: 27fdc77719f23d2e8f1060886576b3be843a8491
This commit is contained in:
Krish Chainani authored and copyberry committed 2026-09-03 22:22:48 +00:00
1 parent 781c183c3b
commit 280ae8b9fc
3 files changed
+44 -29

No files matched your search

+11 -15
View File
@@ -19,7 +19,6 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelInfo;
use codex_utils_image::ImageProcessingError; use codex_utils_image::ImageProcessingError;
use codex_utils_image::PromptImageMode; use codex_utils_image::PromptImageMode;
use codex_utils_image::PromptImageResizeLimits;
use codex_utils_image::load_data_url_for_prompt; use codex_utils_image::load_data_url_for_prompt;
use tracing::warn; use tracing::warn;
@@ -31,15 +30,6 @@ const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because
const REMOTE_IMAGE_URL_PLACEHOLDER: &str = const REMOTE_IMAGE_URL_PLACEHOLDER: &str =
"image content omitted because remote image URLs are not supported"; "image content omitted because remote image URLs are not supported";
const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 2048,
max_patches: 2_500,
};
const UNIFIED_IMAGE_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
max_dimension: 6000,
max_patches: 10_000,
};
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ImagePreparationMode { pub(crate) enum ImagePreparationMode {
DetailBased, DetailBased,
@@ -288,17 +278,23 @@ fn prepare_image(
return Ok(None); return Ok(None);
} }
let (effective_detail, limits) = match mode { let (effective_detail, image_mode) = match mode {
ImagePreparationMode::UnifiedBudget => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS), ImagePreparationMode::UnifiedBudget => (
ImageDetailSetting::Original,
PromptImageMode::ORIGINAL_DETAIL,
),
ImagePreparationMode::DetailBased => match detail { ImagePreparationMode::DetailBased => match detail {
None | Some(ImageDetail::Auto | ImageDetail::High) => { None | Some(ImageDetail::Auto | ImageDetail::High) => {
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS) (ImageDetailSetting::High, PromptImageMode::HIGH_DETAIL)
} }
Some(ImageDetail::Original) => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS), Some(ImageDetail::Original) => (
ImageDetailSetting::Original,
PromptImageMode::ORIGINAL_DETAIL,
),
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail), Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
}, },
}; };
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?; let image = load_data_url_for_prompt(image_url, image_mode)?;
metadata.push(ImagePreparationMetadata { metadata.push(ImagePreparationMetadata {
message_role: origin.message_role.map(str::to_string), message_role: origin.message_role.map(str::to_string),
item_id: origin.item_id.map(str::to_string), item_id: origin.item_id.map(str::to_string),
+20 -14
View File
@@ -276,21 +276,21 @@ fn data_url_processing_rejects_malformed_input() {
} }
} }
/// The shared detail modes apply the policies used by external byte-oriented callers.
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
async fn resize_with_limits_respects_dimension_and_patch_budgets() { async fn detail_modes_apply_expected_budgets() {
let image = ImageBuffer::from_pixel(2048, 2048, Rgba([200u8, 10, 10, 255])); for (mode, input_dimensions, expected_dimensions) in [
(PromptImageMode::HIGH_DETAIL, (2048, 2048), (1600, 1600)),
(PromptImageMode::ORIGINAL_DETAIL, (6401, 100), (6000, 94)),
] {
let image = ImageBuffer::from_pixel(
input_dimensions.0,
input_dimensions.1,
Rgba([200u8, 10, 10, 255]),
);
let original_bytes = image_bytes(&image, ImageFormat::Png); let original_bytes = image_bytes(&image, ImageFormat::Png);
let limits = PromptImageResizeLimits { let processed = load_for_prompt_bytes(Path::new("in-memory-image"), original_bytes, mode)
max_dimension: 2048, .expect("process image with detail mode");
max_patches: 2_500,
};
let processed = load_for_prompt_bytes(
Path::new("in-memory-image"),
original_bytes,
PromptImageMode::ResizeWithLimits(limits),
)
.expect("process image with explicit limits");
assert_eq!( assert_eq!(
( (
@@ -299,9 +299,15 @@ async fn resize_with_limits_respects_dimension_and_patch_budgets() {
processed.width, processed.width,
processed.height, processed.height,
), ),
(2048, 2048, 1600, 1600) (
input_dimensions.0,
input_dimensions.1,
expected_dimensions.0,
expected_dimensions.1,
)
); );
} }
}
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
async fn fails_cleanly_for_invalid_images() { async fn fails_cleanly_for_invalid_images() {
+13
View File
@@ -70,6 +70,19 @@ pub struct PromptImageResizeLimits {
pub max_patches: usize, pub max_patches: usize,
} }
impl PromptImageMode {
/// Resize policy for high-detail prompt images.
pub const HIGH_DETAIL: Self = Self::ResizeWithLimits(PromptImageResizeLimits {
max_dimension: 2048,
max_patches: 2_500,
});
/// Resize policy for original-detail prompt images.
pub const ORIGINAL_DETAIL: Self = Self::ResizeWithLimits(PromptImageResizeLimits {
max_dimension: 6000,
max_patches: 10_000,
});
}
struct ImageMetadata { struct ImageMetadata {
icc_profile: Option<Vec<u8>>, icc_profile: Option<Vec<u8>>,
exif: Option<Vec<u8>>, exif: Option<Vec<u8>>,