core: resize all history images behind a feature flag (#27247)

## Summary

Adds complete client-side image preparation behind the default-off
`resize_all_images` feature flag.

When enabled, local image producers defer decoding and resizing. Images
are prepared centrally before insertion into conversation history,
covering user input, `view_image`, and structured tool-output images.

## Behavior

- Processes base64 `data:` images in messages and function/custom tool
outputs.
- Leaves non-data URLs, including HTTP(S) URLs, unchanged.
- Applies image-detail budgets:
  - `high` and omitted: 2048px maximum dimension and 2.5K 32px patches.
  - `original`: 6000px maximum dimension and 10K 32px patches.
  - `auto`: uses the same 2048px / 2.5K-patch budget as high.
  - `low`: unsupported and replaced with an actionable placeholder.
- Preserves original image bytes when no resize or format conversion is
needed.
- Enforces the shared 1 GiB encoded and decoded data-URL sanity limits.
- Replaces only an image that fails preparation, preserving sibling
content and tool-output metadata.
- Uses bounded placeholders distinguishing generic processing failures,
oversized images, and unsupported `low` detail.
- Prepares resumed and forked history before installing it as live
history without modifying persisted rollouts.

## Flag-Off Behavior

When `resize_all_images` is disabled:

- Existing local user-input and `view_image` processing remains
unchanged.
- Existing decoding and error behavior remains unchanged.
- Arbitrary tool-output images are not processed.
- HTTP(S) image URLs continue to be forwarded unchanged.


#### [git stack](https://github.com/magus/git-stack-cli)
- ✅ `1` https://github.com/openai/codex/pull/27245
- 👉 `2` https://github.com/openai/codex/pull/27247
- ⏳ `3` https://github.com/openai/codex/pull/27246
- ⏳ `4` https://github.com/openai/codex/pull/27266
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-06-10 19:21:24 -07:00
committed by GitHub
parent 9d87b771ce
commit a6f435ea94
15 changed files with 907 additions and 58 deletions
+110
View File
@@ -42,7 +42,9 @@ use codex_protocol::models::ActivePermissionProfile;
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
use codex_protocol::models::FileSystemPermissions;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::openai_models::ModelServiceTier;
@@ -1641,6 +1643,114 @@ async fn record_initial_history_reconstructs_resumed_transcript() {
assert_eq!(expected, history.raw_items());
}
#[tokio::test]
async fn resize_all_images_prepares_failures_before_history_insertion() {
let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let item = ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
};
session
.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&item))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
success: Some(true),
},
}]
);
}
#[tokio::test]
async fn resize_all_images_prepares_resumed_history_before_installing_it() {
let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
let resumed_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
detail: Some(ImageDetail::High),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
};
session
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
conversation_id: ThreadId::default(),
history: vec![RolloutItem::ResponseItem(resumed_item)],
rollout_path: Some(PathBuf::from("/tmp/resume.jsonl")),
}))
.await;
assert_eq!(
session.state.lock().await.clone_history().raw_items(),
&[ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputText {
text: "image content omitted because it could not be processed".to_string(),
},
ContentItem::InputText {
text: "keep me".to_string(),
},
],
phase: None,
}]
);
}
#[test]
fn resolve_multi_agent_version_handles_unset_and_legacy_history() {
let thread_id = ThreadId::default();