mirror of
https://github.com/openai/codex.git
synced 2026-11-18 23:02:23 +00:00
Forward audio inputs to the Responses API (#33932)
## Why Audio variants were available in the user-input protocol but were replaced with unsupported-input placeholders instead of being sent to the model. ## What changed - Serialize audio data URLs as `input_audio` content and convert local `wav`, `mp3`, `m4a`, `webm`, and `ogg` files to labeled data URLs. - Validate and canonicalize base64 audio before requests, enforce the 50 MiB input limit, and replace invalid, unsupported, or oversized audio with explanatory text. - Preserve audio attachments when mapping response items back to user-message events and document the app-server input variants. ## Testing - Add unit coverage for local-file conversion, data URL validation, size and format errors, event mapping, and attachment extraction. - Add client tests that verify data URL and local audio payloads sent to the Responses API. GitOrigin-RevId: f72cd6b6e43ab12757eeb47621b1d670594ac7d9
This commit is contained in:
1 parent
bcdc695877
commit
23899f7cb6
13 files changed
+802
-40
No files matched your search
@@ -10,8 +10,12 @@ use codex_protocol::models::ReasoningItemContent;
|
||||
use codex_protocol::models::ReasoningItemReasoningSummary;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::models::WebSearchAction;
|
||||
use codex_protocol::models::is_audio_close_tag_text;
|
||||
use codex_protocol::models::is_audio_open_tag_text;
|
||||
use codex_protocol::models::is_image_close_tag_text;
|
||||
use codex_protocol::models::is_image_open_tag_text;
|
||||
use codex_protocol::models::is_local_audio_close_tag_text;
|
||||
use codex_protocol::models::is_local_audio_open_tag_text;
|
||||
use codex_protocol::models::is_local_image_close_tag_text;
|
||||
use codex_protocol::models::is_local_image_open_tag_text;
|
||||
use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG;
|
||||
@@ -93,12 +97,19 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
|
||||
for (idx, content_item) in message.iter().enumerate() {
|
||||
match content_item {
|
||||
ContentItem::InputText { text } => {
|
||||
if (is_local_image_open_tag_text(text) || is_image_open_tag_text(text))
|
||||
&& (matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. })))
|
||||
let is_image_label = ((is_local_image_open_tag_text(text)
|
||||
|| is_image_open_tag_text(text))
|
||||
&& matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. })))
|
||||
|| (idx > 0
|
||||
&& (is_local_image_close_tag_text(text) || is_image_close_tag_text(text))
|
||||
&& matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. })))
|
||||
{
|
||||
&& matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. })));
|
||||
let is_audio_label = ((is_local_audio_open_tag_text(text)
|
||||
|| is_audio_open_tag_text(text))
|
||||
&& matches!(message.get(idx + 1), Some(ContentItem::InputAudio { .. })))
|
||||
|| (idx > 0
|
||||
&& (is_local_audio_close_tag_text(text) || is_audio_close_tag_text(text))
|
||||
&& matches!(message.get(idx - 1), Some(ContentItem::InputAudio { .. })));
|
||||
if is_image_label || is_audio_label {
|
||||
continue;
|
||||
}
|
||||
content.push(UserInput::Text {
|
||||
|
||||
Reference in new issue
Block a user