mirror of
https://github.com/openai/codex.git
synced 2026-11-03 16:23:51 +00:00
## Why Audio variants were available in the user-input protocol but were replaced with unsupported-input placeholders instead of being sent to the model. ## What changed - Serialize audio data URLs as `input_audio` content and convert local `wav`, `mp3`, `m4a`, `webm`, and `ogg` files to labeled data URLs. - Validate and canonicalize base64 audio before requests, enforce the 50 MiB input limit, and replace invalid, unsupported, or oversized audio with explanatory text. - Preserve audio attachments when mapping response items back to user-message events and document the app-server input variants. ## Testing - Add unit coverage for local-file conversion, data URL validation, size and format errors, event mapping, and attachment extraction. - Add client tests that verify data URL and local audio payloads sent to the Responses API. GitOrigin-RevId: f72cd6b6e43ab12757eeb47621b1d670594ac7d9
160 lines
5.3 KiB
Rust
160 lines
5.3 KiB
Rust
use codex_protocol::models::FunctionCallOutputBody;
|
|
use codex_protocol::models::FunctionCallOutputPayload;
|
|
use pretty_assertions::assert_eq;
|
|
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() {
|
|
let mut items = vec![ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(),
|
|
},
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
|
},
|
|
ContentItem::InputAudio {
|
|
audio_url: "https://example.com/audio.mp3".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}];
|
|
|
|
prepare_response_items(&mut items);
|
|
|
|
assert_eq!(
|
|
items,
|
|
vec![ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
|
},
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
|
},
|
|
ContentItem::InputText {
|
|
text: "audio content omitted because it could not be processed".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn preparation_replaces_invalid_message_audio_with_placeholders() {
|
|
let mut items = vec![ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/wav;base64,%%%".to_string(),
|
|
},
|
|
ContentItem::InputAudio {
|
|
audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}];
|
|
|
|
prepare_response_items(&mut items);
|
|
|
|
assert_eq!(
|
|
items,
|
|
vec![ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputText {
|
|
text: "audio content omitted because it could not be processed".to_string(),
|
|
},
|
|
ContentItem::InputText {
|
|
text: "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() {
|
|
let mut items = vec![ResponseItem::FunctionCallOutput {
|
|
id: None,
|
|
call_id: "call-1".to_string(),
|
|
output: FunctionCallOutputPayload {
|
|
body: FunctionCallOutputBody::ContentItems(vec![
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "before".to_string(),
|
|
},
|
|
FunctionCallOutputContentItem::InputAudio {
|
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
|
},
|
|
FunctionCallOutputContentItem::InputAudio {
|
|
audio_url: "data:audio/wav,not-base64".to_string(),
|
|
},
|
|
]),
|
|
success: Some(true),
|
|
},
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}];
|
|
|
|
prepare_response_items(&mut items);
|
|
|
|
assert_eq!(
|
|
items,
|
|
vec![ResponseItem::FunctionCallOutput {
|
|
id: None,
|
|
call_id: "call-1".to_string(),
|
|
output: FunctionCallOutputPayload {
|
|
body: FunctionCallOutputBody::ContentItems(vec![
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "before".to_string(),
|
|
},
|
|
FunctionCallOutputContentItem::InputAudio {
|
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
|
},
|
|
FunctionCallOutputContentItem::InputText {
|
|
text: "audio content omitted because it could not be processed".to_string(),
|
|
},
|
|
]),
|
|
success: Some(true),
|
|
},
|
|
internal_chat_message_metadata_passthrough: None,
|
|
}]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn preparation_errors_map_to_expected_placeholders() {
|
|
let cases = [
|
|
(
|
|
AudioPreparationError::InvalidDataUrl {
|
|
reason: "details remain in logs",
|
|
},
|
|
"audio content omitted because it could not be processed",
|
|
),
|
|
(
|
|
AudioPreparationError::UnsupportedFormat,
|
|
"audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg",
|
|
),
|
|
(
|
|
AudioPreparationError::AudioTooLarge { size: usize::MAX },
|
|
"audio content omitted because it exceeded the supported size limit; use a smaller audio file",
|
|
),
|
|
];
|
|
|
|
for (error, expected) in cases {
|
|
assert_eq!(error.placeholder(), expected);
|
|
}
|
|
}
|