Report prompt image resizing to the model (#37134)

## What changed

- Add the disabled-by-default `image_resize_notice` feature.
- When enabled, append a developer message after resized images from user
  messages or tool outputs. The notice identifies each image and reports its
  original and prepared dimensions.
- Persist notices with new history while avoiding backfilled notices for images
  processed when a session is resumed.

## Testing

- Cover notice placement, image numbering, failed images, user and tool image
  sources, and resumed-session replay.

GitOrigin-RevId: 11dd808b3ad47eac35e9e2f4a4370f85d0122770
This commit is contained in:
Curtis 'Fjord' Hawthorne authored and copyberry committed 2026-08-05 18:19:22 +00:00
1 parent 2994f545a7
commit fa5d5ae047
10 files changed
+677 -60

No files matched your search

+20 -6
View File
@@ -31,6 +31,7 @@ use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::exec_policy::BANNED_PREFIX_SUGGESTIONS;
use crate::exec_policy::ExecPolicyManager;
use crate::exec_policy::default_policy_path;
use crate::image_preparation::ImageResizeNoticeMode;
use crate::image_preparation::prepare_response_items as prepare_image_response_items;
use crate::parse_turn_item;
use crate::realtime_conversation::RealtimeConversationManager;
@@ -1432,8 +1433,9 @@ impl Session {
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy media is processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// This meets media preparation requirements without modifying persisted rollouts.
prepare_image_response_items(&mut history);
// Never backfill resize notices during replay; only newly recorded items may
// emit them, so the historical model prefix remains unchanged.
let _ = prepare_image_response_items(&mut history, ImageResizeNoticeMode::Disabled);
prepare_audio_response_items(&mut history);
{
let mut state = self.state.lock().await;
@@ -2898,13 +2900,25 @@ impl Session {
turn_context: &TurnContext,
items: &'a [ResponseItem],
) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {
let mut items = Cow::Borrowed(items);
let image_preparations = prepare_image_response_items(items.to_mut());
prepare_audio_response_items(items.to_mut());
let mut items = items.to_vec();
let image_preparations = prepare_image_response_items(
&mut items,
if turn_context
.config
.features
.enabled(Feature::ImageResizeNotice)
{
ImageResizeNoticeMode::Enabled
} else {
ImageResizeNoticeMode::Disabled
},
);
prepare_audio_response_items(&mut items);
// Most response items get their passthrough turn ID at the durable history boundary.
for item in items.to_mut() {
for item in &mut items {
item.set_turn_id_if_missing(&turn_context.sub_id);
}
let items = Cow::Owned(items);
(
Self::assign_missing_response_item_ids(items),
image_preparations,