Omit undersized WAV output from Code Mode (#41934)

## Why Audio models cannot reliably encode tool-generated clips shorter than 25 ms. ## What changed - Measure the available audio frames in base64-encoded PCM and IEEE float WAV outputs. - Replace clips under 25 ms with explanatory text while preserving surrounding Code Mode output. - Leave clips at or above the threshold and unrecognized audio formats unchanged. ## Testing - Cover the duration boundary across multiple sample rates and all supported `audio()` input forms. - Verify bounded and truncated WAV chunks, output after `yield_control()`, and end-to-end mixed text and audio output. GitOrigin-RevId: 342b4bdbc65db4238580f9cdc372763c2576eca5
2026-10-19 16:17:49 +00:00 · 2026-08-31 23:36:04 +00:00
parent a62ff9a5d4
commit 981d6b652b
8 changed files with 326 additions and 0 deletions
@@ -13,6 +13,7 @@ path = "src/lib.rs"
 workspace = true

 [dependencies]
+base64 = { workspace = true }
 codex-code-mode-protocol = { workspace = true }
 codex-protocol = { workspace = true }
 deno_core_icudata = { workspace = true }
@@ -0,0 +1,63 @@
+//! Measures tool-generated PCM WAV clips using the audio bytes actually present.
+//! Unknown formats retain the existing audio output behavior.
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES;
+
+pub(super) fn wav_duration_seconds(audio_url: &str) -> Option<f64> {
+    let (metadata, payload) = audio_url.split_once(',')?;
+    if !metadata
+        .split(';')
+        .skip(1)
+        .any(|part| part.eq_ignore_ascii_case("base64"))
+        || payload.len() > MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4
+    {
+        return None;
+    }
+    let bytes = BASE64_STANDARD.decode(payload).ok()?;
+    if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {
+        return None;
+    }
+
+    let mut chunks = bytes.get(12..)?;
+    let mut format = None;
+    while chunks.len() >= 8 {
+        let chunk_id = &chunks[..4];
+        let size = u32::from_le_bytes(chunks[4..8].try_into().ok()?) as usize;
+        let remaining = &chunks[8..];
+        // Streaming WAV headers can declare more data than the file contains.
+        let chunk = &remaining[..size.min(remaining.len())];
+        match chunk_id {
+            b"fmt " => {
+                let mut encoding = u16::from_le_bytes(chunk.get(..2)?.try_into().ok()?);
+                if encoding == 0xfffe {
+                    // WAVE_FORMAT_EXTENSIBLE stores the encoding in a subtype GUID.
+                    if chunk.get(26..40)?
+                        != [0, 0, 0, 0, 0x10, 0, 0x80, 0, 0, 0xaa, 0, 0x38, 0x9b, 0x71]
+                    {
+                        return None;
+                    }
+                    encoding = u16::from_le_bytes(chunk.get(24..26)?.try_into().ok()?);
+                }
+                if !matches!(encoding, 1 | 3) {
+                    return None;
+                }
+                let sample_rate = u32::from_le_bytes(chunk.get(4..8)?.try_into().ok()?);
+                let block_align = u16::from_le_bytes(chunk.get(12..14)?.try_into().ok()?);
+                if sample_rate == 0 || block_align == 0 {
+                    return None;
+                }
+                format = Some((sample_rate, block_align));
+            }
+            b"data" => {
+                let (sample_rate, block_align) = format?;
+                let frames = chunk.len() / usize::from(block_align);
+                return Some(frames as f64 / f64::from(sample_rate));
+            }
+            _ => {}
+        }
+        chunks = remaining.get(size.checked_add(size % 2)?..)?;
+    }
+    None
+}
@@ -1,3 +1,4 @@
+mod audio;
 mod callbacks;
 mod globals;
 mod module_loader;
@@ -4,6 +4,8 @@ use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL;
 use codex_code_mode_protocol::FunctionCallOutputContentItem;
 use codex_code_mode_protocol::ImageDetail;

+use super::audio::wav_duration_seconds;
+
 const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
 const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
 const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
@@ -214,6 +216,14 @@ pub(super) fn normalize_output_audio(
            return Err(INVALID_AUDIO_URL_ERROR.to_string());
        }

+        // Tiny tool-generated clips cannot be encoded reliably by audio models.
+        if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) {
+            return Ok(FunctionCallOutputContentItem::InputText {
+                text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."
+                    .to_string(),
+            });
+        }
+
        Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
    })();

@@ -0,0 +1,163 @@
+//! Covers per-item short-WAV omissions without disrupting other Code Mode outputs.
+
+use super::cell_id;
+use super::execute;
+use super::execute_request;
+use crate::ExecuteRequest;
+use crate::FunctionCallOutputContentItem;
+use crate::InProcessCodeModeSession;
+use crate::RuntimeResponse;
+use crate::WaitOutcome;
+use crate::WaitRequest;
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use pretty_assertions::assert_eq;
+
+const SHORT_AUDIO_OMISSION_TEXT: &str =
+    "Audio output omitted because the clip is shorter than 25 ms; use a longer clip.";
+
+fn pcm16_wav(sample_rate: u32, frames: usize) -> Vec<u8> {
+    let data_size = u32::try_from(frames * 2).unwrap();
+    let mut wav = b"RIFF".to_vec();
+    wav.extend_from_slice(&(36 + data_size).to_le_bytes());
+    wav.extend_from_slice(b"WAVEfmt ");
+    wav.extend_from_slice(&16_u32.to_le_bytes());
+    wav.extend_from_slice(&1_u16.to_le_bytes());
+    wav.extend_from_slice(&1_u16.to_le_bytes());
+    wav.extend_from_slice(&sample_rate.to_le_bytes());
+    wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());
+    wav.extend_from_slice(&2_u16.to_le_bytes());
+    wav.extend_from_slice(&16_u16.to_le_bytes());
+    wav.extend_from_slice(b"data");
+    wav.extend_from_slice(&data_size.to_le_bytes());
+    wav.resize(44 + frames * 2, /*value*/ 0);
+    wav
+}
+
+#[tokio::test]
+async fn audio_helper_omits_short_wav_in_all_input_forms() {
+    for (sample_rate, frames, omitted) in [
+        (24_000, 0, true),
+        (24_000, 1, true),
+        (24_000, 599, true),
+        (24_000, 600, false),
+        (24_000, 601, false),
+        (44_100, 223, true),
+        (44_100, 1_102, true),
+        (44_100, 1_103, false),
+        (30_870, 223, true),
+        (30_870, 771, true),
+        (30_870, 772, false),
+        (34_398, 221, true),
+        (34_398, 859, true),
+        (34_398, 860, false),
+    ] {
+        let payload = BASE64_STANDARD.encode(pcm16_wav(sample_rate, frames));
+        let audio_url = format!("data:audio/wav;base64,{payload}");
+        let service = InProcessCodeModeSession::new();
+        let response = execute(
+            &service,
+            ExecuteRequest {
+                source: format!(
+                    r#"audio({audio_url:?}); audio({{audio_url:{audio_url:?}}});
+audio({{type:"audio",mimeType:"audio/wav",data:{payload:?}}});"#
+                ),
+                yield_time_ms: None,
+                ..execute_request("")
+            },
+        )
+        .await;
+        let expected = if omitted {
+            FunctionCallOutputContentItem::InputText {
+                text: SHORT_AUDIO_OMISSION_TEXT.to_string(),
+            }
+        } else {
+            FunctionCallOutputContentItem::InputAudio { audio_url }
+        };
+        assert_eq!(
+            response,
+            RuntimeResponse::Result {
+                code_mode_host_duration: None,
+                cell_id: cell_id("1"),
+                content_items: vec![expected; 3],
+                error_text: None,
+            },
+            "sample rate {sample_rate}, frames {frames}"
+        );
+    }
+}
+
+#[tokio::test]
+async fn audio_helper_bounds_wav_data_and_preserves_outputs_after_yield() {
+    let mut metadata_short = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 1);
+    let mut valid = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);
+    let mut list = b"LIST".to_vec();
+    list.extend_from_slice(&(4_u32 + 8 + 2048).to_le_bytes());
+    list.extend_from_slice(b"INFOINAM");
+    list.extend_from_slice(&2048_u32.to_le_bytes());
+    list.resize(8 + 4 + 8 + 2048, /*value*/ 0);
+    list.extend_from_slice(b"JUNK\x01\0\0\0x\0");
+    for wav in [&mut metadata_short, &mut valid] {
+        drop(wav.splice(36..36, list.iter().copied()));
+        let riff_size = u32::try_from(wav.len() - 8).unwrap();
+        wav[4..8].copy_from_slice(&riff_size.to_le_bytes());
+    }
+    let mut truncated = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 0);
+    truncated[4..8].copy_from_slice(&48_036_u32.to_le_bytes());
+    truncated[40..44].copy_from_slice(&48_000_u32.to_le_bytes());
+    let mut bounded = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);
+    bounded[40..44].copy_from_slice(&2_u32.to_le_bytes());
+    let urls = [metadata_short, truncated, bounded, valid]
+        .map(|wav| format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav)));
+    let audio_urls = serde_json::to_string(&urls).unwrap();
+    let service = InProcessCodeModeSession::new();
+    let response = execute(
+        &service,
+        ExecuteRequest {
+            source: format!(
+                "text('before'); await yield_control(); \
+                 for (const audio_url of {audio_urls}) audio({{audio_url}}); text('after');"
+            ),
+            yield_time_ms: None,
+            ..execute_request("")
+        },
+    )
+    .await;
+    assert_eq!(
+        response,
+        RuntimeResponse::Yielded {
+            code_mode_host_duration: None,
+            cell_id: cell_id("1"),
+            content_items: vec![FunctionCallOutputContentItem::InputText {
+                text: "before".to_string(),
+            }],
+        }
+    );
+    let mut expected = vec![
+        FunctionCallOutputContentItem::InputText {
+            text: SHORT_AUDIO_OMISSION_TEXT.to_string(),
+        };
+        3
+    ];
+    expected.push(FunctionCallOutputContentItem::InputAudio {
+        audio_url: urls[3].clone(),
+    });
+    expected.push(FunctionCallOutputContentItem::InputText {
+        text: "after".to_string(),
+    });
+    assert_eq!(
+        service
+            .wait(WaitRequest {
+                cell_id: cell_id("1"),
+                yield_time_ms: 60_000,
+            })
+            .await
+            .unwrap(),
+        WaitOutcome::LiveCell(RuntimeResponse::Result {
+            code_mode_host_duration: None,
+            cell_id: cell_id("1"),
+            content_items: expected,
+            error_text: None,
+        })
+    );
+}
@@ -1189,6 +1189,9 @@ audio({
    );
 }

+#[path = "service_audio_tests.rs"]
+mod audio_tests;
+
 #[tokio::test]
 async fn audio_helper_rejects_non_data_urls() {
    for source in [