Forward audio inputs to the Responses API (#33932)

## Why

Audio variants were available in the user-input protocol but were replaced with unsupported-input placeholders instead of being sent to the model.

## What changed

- Serialize audio data URLs as `input_audio` content and convert local `wav`, `mp3`, `m4a`, `webm`, and `ogg` files to labeled data URLs.
- Validate and canonicalize base64 audio before requests, enforce the 50 MiB input limit, and replace invalid, unsupported, or oversized audio with explanatory text.
- Preserve audio attachments when mapping response items back to user-message events and document the app-server input variants.

## Testing

- Add unit coverage for local-file conversion, data URL validation, size and format errors, event mapping, and attachment extraction.
- Add client tests that verify data URL and local audio payloads sent to the Responses API.

GitOrigin-RevId: f72cd6b6e43ab12757eeb47621b1d670594ac7d9
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-18 00:26:24 +00:00
1 parent bcdc695877
commit 23899f7cb6
13 files changed
+802 -40

No files matched your search

+15 -4
View File
@@ -10,8 +10,12 @@ use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::WebSearchAction;
use codex_protocol::models::is_audio_close_tag_text;
use codex_protocol::models::is_audio_open_tag_text;
use codex_protocol::models::is_image_close_tag_text;
use codex_protocol::models::is_image_open_tag_text;
use codex_protocol::models::is_local_audio_close_tag_text;
use codex_protocol::models::is_local_audio_open_tag_text;
use codex_protocol::models::is_local_image_close_tag_text;
use codex_protocol::models::is_local_image_open_tag_text;
use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG;
@@ -93,12 +97,19 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
for (idx, content_item) in message.iter().enumerate() {
match content_item {
ContentItem::InputText { text } => {
if (is_local_image_open_tag_text(text) || is_image_open_tag_text(text))
&& (matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. })))
let is_image_label = ((is_local_image_open_tag_text(text)
|| is_image_open_tag_text(text))
&& matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. })))
|| (idx > 0
&& (is_local_image_close_tag_text(text) || is_image_close_tag_text(text))
&& matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. })))
{
&& matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. })));
let is_audio_label = ((is_local_audio_open_tag_text(text)
|| is_audio_open_tag_text(text))
&& matches!(message.get(idx + 1), Some(ContentItem::InputAudio { .. })))
|| (idx > 0
&& (is_local_audio_close_tag_text(text) || is_audio_close_tag_text(text))
&& matches!(message.get(idx - 1), Some(ContentItem::InputAudio { .. })));
if is_image_label || is_audio_label {
continue;
}
content.push(UserInput::Text {