Gate audio history by model input modalities (#33982)

## What changed

- Add `audio` to the model input modalities exposed by the protocol and generated app-server schemas.
- Preserve audio in prompts for models that advertise audio input, and replace historical audio with an omission marker for models that do not.

## Testing

- Cover prompt normalization with and without audio support.
- Extend the model-switching integration test to verify that switching from a multimodal model to a text-only model strips prior image and audio content.

GitOrigin-RevId: c18e47ad602720216303ff9abb5bca4d508391ee

codex-rs/core/tests/common/responses.rs

@@ -216,6 +216,22 @@ impl ResponsesRequest {
             .collect()
     }
 
+    /// Returns all `input_audio` `audio_url` spans from `message` inputs for the provided role.
+    pub fn message_input_audio_urls(&self, role: &str) -> Vec<String> {
+        self.inputs_of_type("message")
+            .into_iter()
+            .filter(|item| item.get("role").and_then(Value::as_str) == Some(role))
+            .filter_map(|item| item.get("content").and_then(Value::as_array).cloned())
+            .flatten()
+            .filter(|span| span.get("type").and_then(Value::as_str) == Some("input_audio"))
+            .filter_map(|span| {
+                span.get("audio_url")
+                    .and_then(Value::as_str)
+                    .map(str::to_owned)
+            })
+            .collect()
+    }
+
     pub fn input(&self) -> Vec<Value> {
         self.body_json()["input"]
             .as_array()

codex-rs/core/tests/suite/client.rs

@@ -43,6 +43,7 @@ use codex_protocol::models::ReasoningItemContent;
 use codex_protocol::models::ReasoningItemReasoningSummary;
 use codex_protocol::models::ResponseItem;
 use codex_protocol::models::WebSearchAction;
+use codex_protocol::openai_models::InputModality;
 use codex_protocol::openai_models::ReasoningEffort;
 use codex_protocol::protocol::EventMsg;
 use codex_protocol::protocol::Op;
@@ -314,7 +315,14 @@ async fn sends_audio_urls_to_responses() {
         sse(vec![ev_response_created("resp1"), ev_completed("resp1")]),
     )
     .await;
-    let codex = test_codex().build(&server).await.unwrap().codex;
+    let codex = test_codex()
+        .with_model_info_override("gpt-5.5", |model_info| {
+            model_info.input_modalities.push(InputModality::Audio);
+        })
+        .build(&server)
+        .await
+        .unwrap()
+        .codex;
     let audio_url = "data:audio/wav;base64,AAAA";
 
     codex
@@ -357,7 +365,13 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> {
         sse(vec![ev_response_created("resp1"), ev_completed("resp1")]),
     )
     .await;
-    let codex = test_codex().build(&server).await?.codex;
+    let codex = test_codex()
+        .with_model_info_override("gpt-5.5", |model_info| {
+            model_info.input_modalities.push(InputModality::Audio);
+        })
+        .build(&server)
+        .await?
+        .codex;
     let temp_dir = tempfile::tempdir()?;
     let audio_path = temp_dir.path().join("recording.wav");
     std::fs::write(&audio_path, b"audio")?;

codex-rs/core/tests/suite/model_switching.rs

@@ -477,17 +477,21 @@ async fn null_service_tier_override_is_omitted_from_http_turn_with_catalog_defau
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<()> {
+async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Result<()> {
     skip_if_no_network!(Ok(()));
 
     let server = MockServer::start().await;
-    let image_model_slug = "test-image-model";
+    let multimodal_model_slug = "test-multimodal-model";
     let text_model_slug = "test-text-only-model";
-    let image_model = test_model_info(
-        image_model_slug,
-        "Test Image Model",
-        "supports image input",
-        default_input_modalities(),
+    let multimodal_model = test_model_info(
+        multimodal_model_slug,
+        "Test Multimodal Model",
+        "supports image and audio input",
+        vec![
+            InputModality::Text,
+            InputModality::Image,
+            InputModality::Audio,
+        ],
     );
     let text_model = test_model_info(
         text_model_slug,
@@ -498,7 +502,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
     mount_models_once(
         &server,
         ModelsResponse {
-            models: vec![image_model, text_model],
+            models: vec![multimodal_model, text_model],
         },
     )
     .await;
@@ -512,7 +516,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
     let mut builder = test_codex()
         .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
         .with_config(move |config| {
-            config.model = Some(image_model_slug.to_string());
+            config.model = Some(multimodal_model_slug.to_string());
         });
     let test = builder.build(&server).await?;
     let models_manager = test.thread_manager.get_models_manager();
@@ -533,12 +537,15 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
                     image_url: image_url.clone(),
                     detail: None,
                 },
+                UserInput::Audio {
+                    audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
+                },
                 UserInput::Text {
                     text: "first turn".to_string(),
                     text_elements: Vec::new(),
                 },
             ],
-            image_model_slug.to_string(),
+            multimodal_model_slug.to_string(),
         ))
         .await?;
     wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
@@ -563,12 +570,20 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
         !first_request.message_input_image_urls("user").is_empty(),
         "first request should include the uploaded image"
     );
+    assert_eq!(
+        first_request.message_input_audio_urls("user"),
+        vec!["data:audio/wav;base64,YXVkaW8=".to_string()]
+    );
 
     let second_request = requests.last().expect("expected second request");
     assert!(
         second_request.message_input_image_urls("user").is_empty(),
         "second request should strip unsupported image content"
     );
+    assert!(
+        second_request.message_input_audio_urls("user").is_empty(),
+        "second request should strip unsupported audio content"
+    );
     let second_user_texts = second_request.message_input_texts("user");
     assert!(
         second_user_texts
@@ -576,6 +591,12 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
             .any(|text| text == "image content omitted because you do not support image input"),
         "second request should include the image-omitted placeholder text"
     );
+    assert!(
+        second_user_texts
+            .iter()
+            .any(|text| text == "audio content omitted because you do not support audio input"),
+        "second request should include the audio-omitted placeholder text"
+    );
     Ok(())
 }