From aec653daa9873bf44517a623fd033722737817a8 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Sun, 23 Aug 2026 20:04:05 +0000 Subject: [PATCH] Preserve annotations when omitting unsupported media (#40277) ## What changed - Render omitted image and audio inputs as contextual fragments tagged with `images.unsupported` and `audio.unsupported` content kinds. - Normalize message media through annotated content so metadata for surrounding content remains intact when switching to a model without media support. ## Testing - Extend history and model-switching tests to verify content-kind annotations before and after unsupported media is replaced. GitOrigin-RevId: 4ee6577f8b78bd5a850ee01ea9d4590c20272850 --- codex-rs/core/src/context/mod.rs | 2 + .../core/src/context/unsupported_media.rs | 42 +++++++++ .../core/src/context_manager/history_tests.rs | 88 +++++++++---------- .../core/src/context_manager/normalize.rs | 45 +++++----- codex-rs/core/src/session/tests.rs | 2 +- codex-rs/core/tests/suite/model_switching.rs | 6 ++ 6 files changed, 115 insertions(+), 70 deletions(-) create mode 100644 codex-rs/core/src/context/unsupported_media.rs diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs index a2a68544e7..378fe0cb08 100644 --- a/codex-rs/core/src/context/mod.rs +++ b/codex-rs/core/src/context/mod.rs @@ -38,6 +38,7 @@ mod rollout_budget; mod subagent_notification; mod token_budget_context; mod turn_aborted; +mod unsupported_media; mod user_instructions; mod user_shell_command; pub(crate) mod world_state; @@ -98,6 +99,7 @@ pub(crate) use token_budget_context::TokenBudgetContext; pub(crate) use token_budget_context::TokenBudgetRemainingContext; pub(crate) use token_budget_context::TokenBudgetReminder; pub(crate) use turn_aborted::TurnAborted; +pub(crate) use unsupported_media::UnsupportedMedia; pub(crate) use user_instructions::UserInstructions; pub(crate) use user_shell_command::UserShellCommand; pub(crate) use world_state::ManagedDeveloperInstructions; diff --git a/codex-rs/core/src/context/unsupported_media.rs b/codex-rs/core/src/context/unsupported_media.rs new file mode 100644 index 0000000000..8b225649c0 --- /dev/null +++ b/codex-rs/core/src/context/unsupported_media.rs @@ -0,0 +1,42 @@ +use super::ContextualUserFragment; +use codex_protocol::models::ContentItemKind; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct UnsupportedMedia { + text: &'static str, + content_kind: &'static str, +} + +impl UnsupportedMedia { + pub(crate) const IMAGE: Self = Self { + text: "image content omitted because you do not support image input", + content_kind: "images.unsupported", + }; + + pub(crate) const AUDIO: Self = Self { + text: "audio content omitted because you do not support audio input", + content_kind: "audio.unsupported", + }; +} + +impl ContextualUserFragment for UnsupportedMedia { + fn content_kind(&self) -> ContentItemKind { + ContentItemKind(self.content_kind.to_string()) + } + + fn role(&self) -> &'static str { + "user" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ("", "") + } + + fn body(&self) -> String { + self.text.to_string() + } +} diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 5a0c6e2602..06fa8b4b08 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -46,6 +46,13 @@ const EXEC_FORMAT_MAX_BYTES: usize = 10_000; const EXEC_FORMAT_MAX_TOKENS: usize = 2_500; const TEST_WAV_SAMPLE_RATE: u32 = 8_000; +fn unknown_content_metadata() -> InternalChatMessageMetadataPassthrough { + InternalChatMessageMetadataPassthrough { + content_item_kinds: Some(vec![ContentItemKind("unknown".to_string())]), + ..Default::default() + } +} + fn pcm_wav_data_url(sample_count: u32) -> (String, usize) { let padding = sample_count % 2; let mut bytes = Vec::new(); @@ -78,7 +85,7 @@ fn assistant_msg(text: &str) -> ResponseItem { text: text.to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } } @@ -97,7 +104,7 @@ fn inter_agent_assistant_msg(text: &str) -> ResponseItem { text: serde_json::to_string(&communication).unwrap(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } } @@ -257,7 +264,7 @@ fn user_msg(text: &str) -> ResponseItem { text: text.to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } } @@ -269,7 +276,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem { text: text.to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } } @@ -281,7 +288,7 @@ fn developer_msg(text: &str) -> ResponseItem { text: text.to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } } @@ -296,7 +303,10 @@ fn developer_msg_with_fragments(texts: &[&str]) -> ResponseItem { }) .collect(), phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(InternalChatMessageMetadataPassthrough { + content_item_kinds: Some(vec![ContentItemKind("unknown".to_string()); texts.len()]), + ..Default::default() + }), } } @@ -397,41 +407,7 @@ fn filters_non_api_messages() { let a = assistant_msg("hello"); h.record_items([&u, &a], policy); - let items = raw_items(&h); - assert_eq!( - items, - vec![ - ResponseItem::Reasoning { - id: None, - summary: vec![ReasoningItemReasoningSummary::SummaryText { - text: "summary".to_string(), - }], - content: Some(vec![ReasoningItemContent::ReasoningText { - text: "thinking...".to_string(), - }]), - encrypted_content: None, - internal_chat_message_metadata_passthrough: None, - }, - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::OutputText { - text: "hi".to_string() - }], - phase: None, - internal_chat_message_metadata_passthrough: None, - }, - ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: "hello".to_string() - }], - phase: None, - internal_chat_message_metadata_passthrough: None, - } - ] - ); + assert_eq!(raw_items(&h), vec![reasoning, u, a]); } #[test] @@ -680,7 +656,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { }, ], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some( + InternalChatMessageMetadataPassthrough { + content_item_kinds: Some(vec![ + ContentItemKind("user.text".to_string()), + ContentItemKind("user.image".to_string()), + ContentItemKind("user.audio".to_string()), + ContentItemKind("user.text".to_string()), + ]), + ..Default::default() + }, + ), }, ResponseItem::FunctionCall { id: None, @@ -764,7 +750,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { }, ], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some( + InternalChatMessageMetadataPassthrough { + content_item_kinds: Some(vec![ + ContentItemKind("user.text".to_string()), + ContentItemKind("images.unsupported".to_string()), + ContentItemKind("audio.unsupported".to_string()), + ContentItemKind("user.text".to_string()), + ]), + ..Default::default() + }, + ), }, ResponseItem::FunctionCall { id: None, @@ -867,7 +863,7 @@ fn for_prompt_strips_media_when_model_does_not_support_it() { audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), }; let with_audio = create_history_with_items(vec![audio_message.clone()]); assert_eq!( @@ -914,7 +910,7 @@ fn for_prompt_preserves_image_generation_calls_when_images_are_supported() { text: "hi".to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), } ] ); @@ -951,7 +947,7 @@ fn for_prompt_clears_image_generation_result_when_images_are_unsupported() { text: "generate a lobster".to_string(), }], phase: None, - internal_chat_message_metadata_passthrough: None, + internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()), }, ResponseItem::ImageGenerationCall { id: Some(ResponseItemId::with_suffix("ig", "123")), diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs index 0e693ce09f..7fc45ba217 100644 --- a/codex-rs/core/src/context_manager/normalize.rs +++ b/codex-rs/core/src/context_manager/normalize.rs @@ -1,3 +1,5 @@ +use codex_context_fragments::set_annotated_content; +use codex_context_fragments::to_annotated_content; use codex_history::ResponseItemEnvelope; use codex_protocol::ResponseItemId; use codex_protocol::models::ContentItem; @@ -8,13 +10,11 @@ use codex_protocol::openai_models::InputModality; use std::collections::HashSet; use uuid::Uuid; +use crate::context::ContextualUserFragment; +use crate::context::UnsupportedMedia; use crate::util::error_or_panic; use tracing::info; -const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str = - "image content omitted because you do not support image input"; -const AUDIO_CONTENT_OMITTED_PLACEHOLDER: &str = - "audio content omitted because you do not support audio input"; // Changing this value would change model-visible IDs and invalidate prompt caches. const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4); @@ -338,19 +338,16 @@ pub(crate) fn strip_images_when_unsupported( for envelope in items.iter_mut() { match &mut envelope.item { - ResponseItem::Message { content, .. } => { - let mut normalized_content = Vec::with_capacity(content.len()); - for content_item in content.iter() { - match content_item { - ContentItem::InputImage { .. } => { - normalized_content.push(ContentItem::InputText { - text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(), - }); - } - _ => normalized_content.push(content_item.clone()), + ResponseItem::Message { .. } => { + let Some(mut content) = to_annotated_content(&mut envelope.item) else { + continue; + }; + for content_item in &mut content { + if matches!(content_item.content(), ContentItem::InputImage { .. }) { + *content_item = UnsupportedMedia::IMAGE.render_fragment().into_parts().1; } } - *content = normalized_content; + let _ = set_annotated_content(&mut envelope.item, content); } ResponseItem::FunctionCallOutput { output, .. } | ResponseItem::CustomToolCallOutput { output, .. } => { @@ -361,7 +358,7 @@ pub(crate) fn strip_images_when_unsupported( FunctionCallOutputContentItem::InputImage { .. } => { normalized_content_items.push( FunctionCallOutputContentItem::InputText { - text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(), + text: UnsupportedMedia::IMAGE.render(), }, ); } @@ -391,14 +388,16 @@ pub(crate) fn strip_audio_when_unsupported( for envelope in items.iter_mut() { match &mut envelope.item { - ResponseItem::Message { content, .. } => { - for content_item in content.iter_mut() { - if matches!(content_item, ContentItem::InputAudio { .. }) { - *content_item = ContentItem::InputText { - text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), - }; + ResponseItem::Message { .. } => { + let Some(mut content) = to_annotated_content(&mut envelope.item) else { + continue; + }; + for content_item in &mut content { + if matches!(content_item.content(), ContentItem::InputAudio { .. }) { + *content_item = UnsupportedMedia::AUDIO.render_fragment().into_parts().1; } } + let _ = set_annotated_content(&mut envelope.item, content); } ResponseItem::FunctionCallOutput { output, .. } | ResponseItem::CustomToolCallOutput { output, .. } => { @@ -409,7 +408,7 @@ pub(crate) fn strip_audio_when_unsupported( FunctionCallOutputContentItem::InputAudio { .. } ) { *content_item = FunctionCallOutputContentItem::InputText { - text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), + text: UnsupportedMedia::AUDIO.render(), }; } } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index b977ccb9ef..1fa912cc1c 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -11312,7 +11312,7 @@ async fn sample_rollout( ( rollout_items, - live_history.for_prompt(&reconstruction_turn.model_info.input_modalities), + strip_metadata_from_items(&raw_history_items(&live_history)), ) } diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 4bd09d11ae..e5232fe736 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -992,6 +992,7 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re assert_eq!(requests.len(), 2, "expected two model requests"); let first_request = requests.first().expect("expected first request"); + assert!(first_request.has_content_kinds(&["user.image", "user.audio", "user.text"])); assert!( !first_request.message_input_image_urls("user").is_empty(), "first request should include the uploaded image" @@ -1002,6 +1003,11 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re ); let second_request = requests.last().expect("expected second request"); + assert!(second_request.has_content_kinds(&[ + "images.unsupported", + "audio.unsupported", + "user.text", + ])); assert!( second_request.message_input_image_urls("user").is_empty(), "second request should strip unsupported image content"