mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Add audio output support to dynamic tools and code mode (#34080)
## What changed - Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas. - Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks. - Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item. - Reject non-data audio URLs and track audio item counts in dynamic tool analytics. ## Testing - Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling. GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
1 parent
312caf176a
commit
643de86a19
46 files changed
+987
-72
No files matched your search
@@ -1552,6 +1552,8 @@ pub enum DynamicToolCallOutputContentItem {
|
||||
InputText { text: String },
|
||||
#[serde(rename_all = "camelCase")]
|
||||
InputImage { image_url: String },
|
||||
#[serde(rename_all = "camelCase")]
|
||||
InputAudio { audio_url: String },
|
||||
}
|
||||
|
||||
impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
|
||||
@@ -1565,6 +1567,9 @@ impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
|
||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
} => Self::InputImage { image_url },
|
||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url,
|
||||
} => Self::InputAudio { audio_url },
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1578,6 +1583,9 @@ impl From<DynamicToolCallOutputContentItem>
|
||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||
Self::InputImage { image_url }
|
||||
}
|
||||
DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
|
||||
Self::InputAudio { audio_url }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2719,6 +2719,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText {
|
||||
text: "ok".to_string(),
|
||||
},
|
||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
error: None,
|
||||
@@ -2733,9 +2739,17 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
tool: "lookup".to_string(),
|
||||
arguments: json!({"id": "123"}),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: "ok".to_string(),
|
||||
}]),
|
||||
content_items: Some(vec![
|
||||
DynamicToolCallOutputContentItem::InputText {
|
||||
text: "ok".to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(5),
|
||||
}
|
||||
@@ -3954,7 +3968,7 @@ fn dynamic_tool_response_serializes_content_items() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
||||
fn dynamic_tool_response_serializes_text_image_and_audio_content_items() {
|
||||
let value = serde_json::to_value(DynamicToolCallResponse {
|
||||
content_items: vec![
|
||||
DynamicToolCallOutputContentItem::InputText {
|
||||
@@ -3963,6 +3977,9 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
],
|
||||
success: true,
|
||||
})
|
||||
@@ -3979,6 +3996,10 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
||||
{
|
||||
"type": "inputImage",
|
||||
"imageUrl": "data:image/png;base64,AAA"
|
||||
},
|
||||
{
|
||||
"type": "inputAudio",
|
||||
"audioUrl": "data:audio/wav;base64,YXVkaW8="
|
||||
}
|
||||
],
|
||||
"success": true,
|
||||
|
||||
Reference in new issue
Block a user