mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
Centralize prompt image detail modes (#42624)
## What changed - Add `PromptImageMode::HIGH_DETAIL` and `PromptImageMode::ORIGINAL_DETAIL` constants with the standard resize limits. - Use the shared modes during core image preparation instead of defining the limits locally. - Cover the dimension and patch budgets for both detail modes in the image utility tests. GitOrigin-RevId: 27fdc77719f23d2e8f1060886576b3be843a8491
This commit is contained in:
1 parent
781c183c3b
commit
280ae8b9fc
3 files changed
+44
-29
No files matched your search
@@ -19,7 +19,6 @@ use codex_protocol::models::ResponseItem;
|
|||||||
use codex_protocol::openai_models::ModelInfo;
|
use codex_protocol::openai_models::ModelInfo;
|
||||||
use codex_utils_image::ImageProcessingError;
|
use codex_utils_image::ImageProcessingError;
|
||||||
use codex_utils_image::PromptImageMode;
|
use codex_utils_image::PromptImageMode;
|
||||||
use codex_utils_image::PromptImageResizeLimits;
|
|
||||||
use codex_utils_image::load_data_url_for_prompt;
|
use codex_utils_image::load_data_url_for_prompt;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
@@ -31,15 +30,6 @@ const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because
|
|||||||
const REMOTE_IMAGE_URL_PLACEHOLDER: &str =
|
const REMOTE_IMAGE_URL_PLACEHOLDER: &str =
|
||||||
"image content omitted because remote image URLs are not supported";
|
"image content omitted because remote image URLs are not supported";
|
||||||
|
|
||||||
const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
|
|
||||||
max_dimension: 2048,
|
|
||||||
max_patches: 2_500,
|
|
||||||
};
|
|
||||||
const UNIFIED_IMAGE_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {
|
|
||||||
max_dimension: 6000,
|
|
||||||
max_patches: 10_000,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(crate) enum ImagePreparationMode {
|
pub(crate) enum ImagePreparationMode {
|
||||||
DetailBased,
|
DetailBased,
|
||||||
@@ -288,17 +278,23 @@ fn prepare_image(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let (effective_detail, limits) = match mode {
|
let (effective_detail, image_mode) = match mode {
|
||||||
ImagePreparationMode::UnifiedBudget => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),
|
ImagePreparationMode::UnifiedBudget => (
|
||||||
|
ImageDetailSetting::Original,
|
||||||
|
PromptImageMode::ORIGINAL_DETAIL,
|
||||||
|
),
|
||||||
ImagePreparationMode::DetailBased => match detail {
|
ImagePreparationMode::DetailBased => match detail {
|
||||||
None | Some(ImageDetail::Auto | ImageDetail::High) => {
|
None | Some(ImageDetail::Auto | ImageDetail::High) => {
|
||||||
(ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
|
(ImageDetailSetting::High, PromptImageMode::HIGH_DETAIL)
|
||||||
}
|
}
|
||||||
Some(ImageDetail::Original) => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),
|
Some(ImageDetail::Original) => (
|
||||||
|
ImageDetailSetting::Original,
|
||||||
|
PromptImageMode::ORIGINAL_DETAIL,
|
||||||
|
),
|
||||||
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
|
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
|
let image = load_data_url_for_prompt(image_url, image_mode)?;
|
||||||
metadata.push(ImagePreparationMetadata {
|
metadata.push(ImagePreparationMetadata {
|
||||||
message_role: origin.message_role.map(str::to_string),
|
message_role: origin.message_role.map(str::to_string),
|
||||||
item_id: origin.item_id.map(str::to_string),
|
item_id: origin.item_id.map(str::to_string),
|
||||||
|
|||||||
@@ -276,21 +276,21 @@ fn data_url_processing_rejects_malformed_input() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The shared detail modes apply the policies used by external byte-oriented callers.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn resize_with_limits_respects_dimension_and_patch_budgets() {
|
async fn detail_modes_apply_expected_budgets() {
|
||||||
let image = ImageBuffer::from_pixel(2048, 2048, Rgba([200u8, 10, 10, 255]));
|
for (mode, input_dimensions, expected_dimensions) in [
|
||||||
|
(PromptImageMode::HIGH_DETAIL, (2048, 2048), (1600, 1600)),
|
||||||
|
(PromptImageMode::ORIGINAL_DETAIL, (6401, 100), (6000, 94)),
|
||||||
|
] {
|
||||||
|
let image = ImageBuffer::from_pixel(
|
||||||
|
input_dimensions.0,
|
||||||
|
input_dimensions.1,
|
||||||
|
Rgba([200u8, 10, 10, 255]),
|
||||||
|
);
|
||||||
let original_bytes = image_bytes(&image, ImageFormat::Png);
|
let original_bytes = image_bytes(&image, ImageFormat::Png);
|
||||||
let limits = PromptImageResizeLimits {
|
let processed = load_for_prompt_bytes(Path::new("in-memory-image"), original_bytes, mode)
|
||||||
max_dimension: 2048,
|
.expect("process image with detail mode");
|
||||||
max_patches: 2_500,
|
|
||||||
};
|
|
||||||
|
|
||||||
let processed = load_for_prompt_bytes(
|
|
||||||
Path::new("in-memory-image"),
|
|
||||||
original_bytes,
|
|
||||||
PromptImageMode::ResizeWithLimits(limits),
|
|
||||||
)
|
|
||||||
.expect("process image with explicit limits");
|
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
(
|
(
|
||||||
@@ -299,9 +299,15 @@ async fn resize_with_limits_respects_dimension_and_patch_budgets() {
|
|||||||
processed.width,
|
processed.width,
|
||||||
processed.height,
|
processed.height,
|
||||||
),
|
),
|
||||||
(2048, 2048, 1600, 1600)
|
(
|
||||||
|
input_dimensions.0,
|
||||||
|
input_dimensions.1,
|
||||||
|
expected_dimensions.0,
|
||||||
|
expected_dimensions.1,
|
||||||
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn fails_cleanly_for_invalid_images() {
|
async fn fails_cleanly_for_invalid_images() {
|
||||||
|
|||||||
@@ -70,6 +70,19 @@ pub struct PromptImageResizeLimits {
|
|||||||
pub max_patches: usize,
|
pub max_patches: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PromptImageMode {
|
||||||
|
/// Resize policy for high-detail prompt images.
|
||||||
|
pub const HIGH_DETAIL: Self = Self::ResizeWithLimits(PromptImageResizeLimits {
|
||||||
|
max_dimension: 2048,
|
||||||
|
max_patches: 2_500,
|
||||||
|
});
|
||||||
|
/// Resize policy for original-detail prompt images.
|
||||||
|
pub const ORIGINAL_DETAIL: Self = Self::ResizeWithLimits(PromptImageResizeLimits {
|
||||||
|
max_dimension: 6000,
|
||||||
|
max_patches: 10_000,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
struct ImageMetadata {
|
struct ImageMetadata {
|
||||||
icc_profile: Option<Vec<u8>>,
|
icc_profile: Option<Vec<u8>>,
|
||||||
exif: Option<Vec<u8>>,
|
exif: Option<Vec<u8>>,
|
||||||
|
|||||||
Reference in new issue
Block a user