Add audio output support to dynamic tools and code mode (#34080)

## What changed

- Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas.
- Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks.
- Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item.
- Reject non-data audio URLs and track audio item counts in dynamic tool analytics.

## Testing

- Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling.

GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-18 23:22:13 +00:00
1 parent 312caf176a
commit 643de86a19
46 files changed
+987 -72

No files matched your search

@@ -1552,6 +1552,8 @@ pub enum DynamicToolCallOutputContentItem {
InputText { text: String },
#[serde(rename_all = "camelCase")]
InputImage { image_url: String },
#[serde(rename_all = "camelCase")]
InputAudio { audio_url: String },
}
impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
@@ -1565,6 +1567,9 @@ impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
image_url,
} => Self::InputImage { image_url },
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
audio_url,
} => Self::InputAudio { audio_url },
}
}
}
@@ -1578,6 +1583,9 @@ impl From<DynamicToolCallOutputContentItem>
DynamicToolCallOutputContentItem::InputImage { image_url } => {
Self::InputImage { image_url }
}
DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
}
}
}
}
@@ -2719,6 +2719,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
},
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]),
success: Some(true),
error: None,
@@ -2733,9 +2739,17 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
tool: "lookup".to_string(),
arguments: json!({"id": "123"}),
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
}]),
content_items: Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: "ok".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]),
success: Some(true),
duration_ms: Some(5),
}
@@ -3954,7 +3968,7 @@ fn dynamic_tool_response_serializes_content_items() {
}
#[test]
fn dynamic_tool_response_serializes_text_and_image_content_items() {
fn dynamic_tool_response_serializes_text_image_and_audio_content_items() {
let value = serde_json::to_value(DynamicToolCallResponse {
content_items: vec![
DynamicToolCallOutputContentItem::InputText {
@@ -3963,6 +3977,9 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
},
DynamicToolCallOutputContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
],
success: true,
})
@@ -3979,6 +3996,10 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
{
"type": "inputImage",
"imageUrl": "data:image/png;base64,AAA"
},
{
"type": "inputAudio",
"audioUrl": "data:audio/wav;base64,YXVkaW8="
}
],
"success": true,