Preserve annotations when omitting unsupported media (#40277)

## What changed

- Render omitted image and audio inputs as contextual fragments tagged with
  `images.unsupported` and `audio.unsupported` content kinds.
- Normalize message media through annotated content so metadata for surrounding
  content remains intact when switching to a model without media support.

## Testing

- Extend history and model-switching tests to verify content-kind annotations
  before and after unsupported media is replaced.

GitOrigin-RevId: 4ee6577f8b78bd5a850ee01ea9d4590c20272850

codex-rs/core/src/context/mod.rs

@@ -38,6 +38,7 @@ mod rollout_budget;
 mod subagent_notification;
 mod token_budget_context;
 mod turn_aborted;
+mod unsupported_media;
 mod user_instructions;
 mod user_shell_command;
 pub(crate) mod world_state;
@@ -98,6 +99,7 @@ pub(crate) use token_budget_context::TokenBudgetContext;
 pub(crate) use token_budget_context::TokenBudgetRemainingContext;
 pub(crate) use token_budget_context::TokenBudgetReminder;
 pub(crate) use turn_aborted::TurnAborted;
+pub(crate) use unsupported_media::UnsupportedMedia;
 pub(crate) use user_instructions::UserInstructions;
 pub(crate) use user_shell_command::UserShellCommand;
 pub(crate) use world_state::ManagedDeveloperInstructions;

codex-rs/core/src/context/unsupported_media.rs

@@ -0,0 +1,42 @@
+use super::ContextualUserFragment;
+use codex_protocol::models::ContentItemKind;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct UnsupportedMedia {
+    text: &'static str,
+    content_kind: &'static str,
+}
+
+impl UnsupportedMedia {
+    pub(crate) const IMAGE: Self = Self {
+        text: "image content omitted because you do not support image input",
+        content_kind: "images.unsupported",
+    };
+
+    pub(crate) const AUDIO: Self = Self {
+        text: "audio content omitted because you do not support audio input",
+        content_kind: "audio.unsupported",
+    };
+}
+
+impl ContextualUserFragment for UnsupportedMedia {
+    fn content_kind(&self) -> ContentItemKind {
+        ContentItemKind(self.content_kind.to_string())
+    }
+
+    fn role(&self) -> &'static str {
+        "user"
+    }
+
+    fn markers(&self) -> (&'static str, &'static str) {
+        Self::type_markers()
+    }
+
+    fn type_markers() -> (&'static str, &'static str) {
+        ("", "")
+    }
+
+    fn body(&self) -> String {
+        self.text.to_string()
+    }
+}

codex-rs/core/src/context_manager/history_tests.rs

@@ -46,6 +46,13 @@ const EXEC_FORMAT_MAX_BYTES: usize = 10_000;
 const EXEC_FORMAT_MAX_TOKENS: usize = 2_500;
 const TEST_WAV_SAMPLE_RATE: u32 = 8_000;
 
+fn unknown_content_metadata() -> InternalChatMessageMetadataPassthrough {
+    InternalChatMessageMetadataPassthrough {
+        content_item_kinds: Some(vec![ContentItemKind("unknown".to_string())]),
+        ..Default::default()
+    }
+}
+
 fn pcm_wav_data_url(sample_count: u32) -> (String, usize) {
     let padding = sample_count % 2;
     let mut bytes = Vec::new();
@@ -78,7 +85,7 @@ fn assistant_msg(text: &str) -> ResponseItem {
             text: text.to_string(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     }
 }
 
@@ -97,7 +104,7 @@ fn inter_agent_assistant_msg(text: &str) -> ResponseItem {
             text: serde_json::to_string(&communication).unwrap(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     }
 }
 
@@ -257,7 +264,7 @@ fn user_msg(text: &str) -> ResponseItem {
             text: text.to_string(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     }
 }
 
@@ -269,7 +276,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem {
             text: text.to_string(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     }
 }
 
@@ -281,7 +288,7 @@ fn developer_msg(text: &str) -> ResponseItem {
             text: text.to_string(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     }
 }
 
@@ -296,7 +303,10 @@ fn developer_msg_with_fragments(texts: &[&str]) -> ResponseItem {
             })
             .collect(),
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(InternalChatMessageMetadataPassthrough {
+            content_item_kinds: Some(vec![ContentItemKind("unknown".to_string()); texts.len()]),
+            ..Default::default()
+        }),
     }
 }
 
@@ -397,41 +407,7 @@ fn filters_non_api_messages() {
     let a = assistant_msg("hello");
     h.record_items([&u, &a], policy);
 
-    let items = raw_items(&h);
-    assert_eq!(
-        items,
-        vec![
-            ResponseItem::Reasoning {
-                id: None,
-                summary: vec![ReasoningItemReasoningSummary::SummaryText {
-                    text: "summary".to_string(),
-                }],
-                content: Some(vec![ReasoningItemContent::ReasoningText {
-                    text: "thinking...".to_string(),
-                }]),
-                encrypted_content: None,
-                internal_chat_message_metadata_passthrough: None,
-            },
-            ResponseItem::Message {
-                id: None,
-                role: "user".to_string(),
-                content: vec![ContentItem::OutputText {
-                    text: "hi".to_string()
-                }],
-                phase: None,
-                internal_chat_message_metadata_passthrough: None,
-            },
-            ResponseItem::Message {
-                id: None,
-                role: "assistant".to_string(),
-                content: vec![ContentItem::OutputText {
-                    text: "hello".to_string()
-                }],
-                phase: None,
-                internal_chat_message_metadata_passthrough: None,
-            }
-        ]
-    );
+    assert_eq!(raw_items(&h), vec![reasoning, u, a]);
 }
 
 #[test]
@@ -680,7 +656,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
                 },
             ],
             phase: None,
-            internal_chat_message_metadata_passthrough: None,
+            internal_chat_message_metadata_passthrough: Some(
+                InternalChatMessageMetadataPassthrough {
+                    content_item_kinds: Some(vec![
+                        ContentItemKind("user.text".to_string()),
+                        ContentItemKind("user.image".to_string()),
+                        ContentItemKind("user.audio".to_string()),
+                        ContentItemKind("user.text".to_string()),
+                    ]),
+                    ..Default::default()
+                },
+            ),
         },
         ResponseItem::FunctionCall {
             id: None,
@@ -764,7 +750,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
                 },
             ],
             phase: None,
-            internal_chat_message_metadata_passthrough: None,
+            internal_chat_message_metadata_passthrough: Some(
+                InternalChatMessageMetadataPassthrough {
+                    content_item_kinds: Some(vec![
+                        ContentItemKind("user.text".to_string()),
+                        ContentItemKind("images.unsupported".to_string()),
+                        ContentItemKind("audio.unsupported".to_string()),
+                        ContentItemKind("user.text".to_string()),
+                    ]),
+                    ..Default::default()
+                },
+            ),
         },
         ResponseItem::FunctionCall {
             id: None,
@@ -867,7 +863,7 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
             audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
         }],
         phase: None,
-        internal_chat_message_metadata_passthrough: None,
+        internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
     };
     let with_audio = create_history_with_items(vec![audio_message.clone()]);
     assert_eq!(
@@ -914,7 +910,7 @@ fn for_prompt_preserves_image_generation_calls_when_images_are_supported() {
                     text: "hi".to_string(),
                 }],
                 phase: None,
-                internal_chat_message_metadata_passthrough: None,
+                internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
             }
         ]
     );
@@ -951,7 +947,7 @@ fn for_prompt_clears_image_generation_result_when_images_are_unsupported() {
                     text: "generate a lobster".to_string(),
                 }],
                 phase: None,
-                internal_chat_message_metadata_passthrough: None,
+                internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
             },
             ResponseItem::ImageGenerationCall {
                 id: Some(ResponseItemId::with_suffix("ig", "123")),

codex-rs/core/src/context_manager/normalize.rs

@@ -1,3 +1,5 @@
+use codex_context_fragments::set_annotated_content;
+use codex_context_fragments::to_annotated_content;
 use codex_history::ResponseItemEnvelope;
 use codex_protocol::ResponseItemId;
 use codex_protocol::models::ContentItem;
@@ -8,13 +10,11 @@ use codex_protocol::openai_models::InputModality;
 use std::collections::HashSet;
 use uuid::Uuid;
 
+use crate::context::ContextualUserFragment;
+use crate::context::UnsupportedMedia;
 use crate::util::error_or_panic;
 use tracing::info;
 
-const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str =
-    "image content omitted because you do not support image input";
-const AUDIO_CONTENT_OMITTED_PLACEHOLDER: &str =
-    "audio content omitted because you do not support audio input";
 // Changing this value would change model-visible IDs and invalidate prompt caches.
 const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4);
 
@@ -338,19 +338,16 @@ pub(crate) fn strip_images_when_unsupported(
 
     for envelope in items.iter_mut() {
         match &mut envelope.item {
-            ResponseItem::Message { content, .. } => {
-                let mut normalized_content = Vec::with_capacity(content.len());
-                for content_item in content.iter() {
-                    match content_item {
-                        ContentItem::InputImage { .. } => {
-                            normalized_content.push(ContentItem::InputText {
-                                text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),
-                            });
-                        }
-                        _ => normalized_content.push(content_item.clone()),
+            ResponseItem::Message { .. } => {
+                let Some(mut content) = to_annotated_content(&mut envelope.item) else {
+                    continue;
+                };
+                for content_item in &mut content {
+                    if matches!(content_item.content(), ContentItem::InputImage { .. }) {
+                        *content_item = UnsupportedMedia::IMAGE.render_fragment().into_parts().1;
                     }
                 }
-                *content = normalized_content;
+                let _ = set_annotated_content(&mut envelope.item, content);
             }
             ResponseItem::FunctionCallOutput { output, .. }
             | ResponseItem::CustomToolCallOutput { output, .. } => {
@@ -361,7 +358,7 @@ pub(crate) fn strip_images_when_unsupported(
                             FunctionCallOutputContentItem::InputImage { .. } => {
                                 normalized_content_items.push(
                                     FunctionCallOutputContentItem::InputText {
-                                        text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),
+                                        text: UnsupportedMedia::IMAGE.render(),
                                     },
                                 );
                             }
@@ -391,14 +388,16 @@ pub(crate) fn strip_audio_when_unsupported(
 
     for envelope in items.iter_mut() {
         match &mut envelope.item {
-            ResponseItem::Message { content, .. } => {
-                for content_item in content.iter_mut() {
-                    if matches!(content_item, ContentItem::InputAudio { .. }) {
-                        *content_item = ContentItem::InputText {
-                            text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
-                        };
+            ResponseItem::Message { .. } => {
+                let Some(mut content) = to_annotated_content(&mut envelope.item) else {
+                    continue;
+                };
+                for content_item in &mut content {
+                    if matches!(content_item.content(), ContentItem::InputAudio { .. }) {
+                        *content_item = UnsupportedMedia::AUDIO.render_fragment().into_parts().1;
                     }
                 }
+                let _ = set_annotated_content(&mut envelope.item, content);
             }
             ResponseItem::FunctionCallOutput { output, .. }
             | ResponseItem::CustomToolCallOutput { output, .. } => {
@@ -409,7 +408,7 @@ pub(crate) fn strip_audio_when_unsupported(
                             FunctionCallOutputContentItem::InputAudio { .. }
                         ) {
                             *content_item = FunctionCallOutputContentItem::InputText {
-                                text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
+                                text: UnsupportedMedia::AUDIO.render(),
                             };
                         }
                     }

codex-rs/core/src/session/tests.rs

@@ -11312,7 +11312,7 @@ async fn sample_rollout(
 
     (
         rollout_items,
-        live_history.for_prompt(&reconstruction_turn.model_info.input_modalities),
+        strip_metadata_from_items(&raw_history_items(&live_history)),
     )
 }
 

codex-rs/core/tests/suite/model_switching.rs

@@ -992,6 +992,7 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re
     assert_eq!(requests.len(), 2, "expected two model requests");
 
     let first_request = requests.first().expect("expected first request");
+    assert!(first_request.has_content_kinds(&["user.image", "user.audio", "user.text"]));
     assert!(
         !first_request.message_input_image_urls("user").is_empty(),
         "first request should include the uploaded image"
@@ -1002,6 +1003,11 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re
     );
 
     let second_request = requests.last().expect("expected second request");
+    assert!(second_request.has_content_kinds(&[
+        "images.unsupported",
+        "audio.unsupported",
+        "user.text",
+    ]));
     assert!(
         second_request.message_input_image_urls("user").is_empty(),
         "second request should strip unsupported image content"