mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Add audio output support to dynamic tools and code mode (#34080)
## What changed - Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas. - Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks. - Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item. - Reject non-data audio URLs and track audio item counts in dynamic tool analytics. ## Testing - Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling. GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
1 parent
312caf176a
commit
643de86a19
46 files changed
+987
-72
No files matched your search
@@ -707,6 +707,7 @@ pub(crate) struct CodexDynamicToolCallEventParams {
|
|||||||
pub(crate) output_content_item_count: Option<u64>,
|
pub(crate) output_content_item_count: Option<u64>,
|
||||||
pub(crate) output_text_item_count: Option<u64>,
|
pub(crate) output_text_item_count: Option<u64>,
|
||||||
pub(crate) output_image_item_count: Option<u64>,
|
pub(crate) output_image_item_count: Option<u64>,
|
||||||
|
pub(crate) output_audio_item_count: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|||||||
@@ -1968,6 +1968,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option<TrackEventRequest> {
|
|||||||
output_content_item_count: counts.map(|counts| counts.total),
|
output_content_item_count: counts.map(|counts| counts.total),
|
||||||
output_text_item_count: counts.map(|counts| counts.text),
|
output_text_item_count: counts.map(|counts| counts.text),
|
||||||
output_image_item_count: counts.map(|counts| counts.image),
|
output_image_item_count: counts.map(|counts| counts.image),
|
||||||
|
output_audio_item_count: counts.map(|counts| counts.audio),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
))
|
))
|
||||||
@@ -2505,26 +2506,30 @@ fn file_change_counts(changes: &[codex_app_server_protocol::FileUpdateChange]) -
|
|||||||
counts
|
counts
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
struct DynamicContentCounts {
|
struct DynamicContentCounts {
|
||||||
total: u64,
|
total: u64,
|
||||||
text: u64,
|
text: u64,
|
||||||
image: u64,
|
image: u64,
|
||||||
|
audio: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn dynamic_content_counts(items: &[DynamicToolCallOutputContentItem]) -> DynamicContentCounts {
|
fn dynamic_content_counts(items: &[DynamicToolCallOutputContentItem]) -> DynamicContentCounts {
|
||||||
let mut text = 0;
|
let mut text = 0;
|
||||||
let mut image = 0;
|
let mut image = 0;
|
||||||
|
let mut audio = 0;
|
||||||
for item in items {
|
for item in items {
|
||||||
match item {
|
match item {
|
||||||
DynamicToolCallOutputContentItem::InputText { .. } => text += 1,
|
DynamicToolCallOutputContentItem::InputText { .. } => text += 1,
|
||||||
DynamicToolCallOutputContentItem::InputImage { .. } => image += 1,
|
DynamicToolCallOutputContentItem::InputImage { .. } => image += 1,
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio { .. } => audio += 1,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
DynamicContentCounts {
|
DynamicContentCounts {
|
||||||
total: usize_to_u64(items.len()),
|
total: usize_to_u64(items.len()),
|
||||||
text,
|
text,
|
||||||
image,
|
image,
|
||||||
|
audio,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2830,6 +2835,7 @@ mod tests {
|
|||||||
use codex_protocol::models::SandboxEnforcement;
|
use codex_protocol::models::SandboxEnforcement;
|
||||||
use codex_protocol::permissions::FileSystemSandboxPolicy;
|
use codex_protocol::permissions::FileSystemSandboxPolicy;
|
||||||
use codex_protocol::permissions::NetworkSandboxPolicy;
|
use codex_protocol::permissions::NetworkSandboxPolicy;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn managed_full_disk_with_restricted_network_reports_external_sandbox() {
|
fn managed_full_disk_with_restricted_network_reports_external_sandbox() {
|
||||||
@@ -2853,4 +2859,29 @@ mod tests {
|
|||||||
Some((ReviewStatus::TimedOut, ReviewResolution::None))
|
Some((ReviewStatus::TimedOut, ReviewResolution::None))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dynamic_content_counts_include_audio() {
|
||||||
|
let items = vec![
|
||||||
|
DynamicToolCallOutputContentItem::InputText {
|
||||||
|
text: "ok".to_string(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
dynamic_content_counts(&items),
|
||||||
|
DynamicContentCounts {
|
||||||
|
total: 3,
|
||||||
|
text: 1,
|
||||||
|
image: 1,
|
||||||
|
audio: 1,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -42,6 +42,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1129,6 +1129,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
+20
@@ -9163,6 +9163,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
+20
@@ -5356,6 +5356,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -240,6 +240,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -240,6 +240,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -378,6 +378,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -459,6 +459,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
+20
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -459,6 +459,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -459,6 +459,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -381,6 +381,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -378,6 +378,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -378,6 +378,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -378,6 +378,26 @@
|
|||||||
],
|
],
|
||||||
"title": "InputImageDynamicToolCallOutputContentItem",
|
"title": "InputImageDynamicToolCallOutputContentItem",
|
||||||
"type": "object"
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"audioUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"enum": [
|
||||||
|
"inputAudio"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItemType",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"audioUrl",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"title": "InputAudioDynamicToolCallOutputContentItem",
|
||||||
|
"type": "object"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
+1
-1
@@ -2,4 +2,4 @@
|
|||||||
|
|
||||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||||
|
|
||||||
export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, };
|
export type DynamicToolCallOutputContentItem = { "type": "inputText", text: string, } | { "type": "inputImage", imageUrl: string, } | { "type": "inputAudio", audioUrl: string, };
|
||||||
@@ -59,6 +59,9 @@ pub fn item_event_to_server_notification(
|
|||||||
CoreDynamicToolCallOutputContentItem::InputImage { image_url } => {
|
CoreDynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||||
DynamicToolCallOutputContentItem::InputImage { image_url }
|
DynamicToolCallOutputContentItem::InputImage { image_url }
|
||||||
}
|
}
|
||||||
|
CoreDynamicToolCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio { audio_url }
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -1498,6 +1498,9 @@ fn convert_dynamic_tool_content_items(
|
|||||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
||||||
image_url,
|
image_url,
|
||||||
} => DynamicToolCallOutputContentItem::InputImage { image_url },
|
} => DynamicToolCallOutputContentItem::InputImage { image_url },
|
||||||
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url,
|
||||||
|
} => DynamicToolCallOutputContentItem::InputAudio { audio_url },
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
@@ -2879,9 +2882,17 @@ mod tests {
|
|||||||
namespace: Some("codex_app".into()),
|
namespace: Some("codex_app".into()),
|
||||||
tool: "lookup_ticket".into(),
|
tool: "lookup_ticket".into(),
|
||||||
arguments: serde_json::json!({"id":"ABC-123"}),
|
arguments: serde_json::json!({"id":"ABC-123"}),
|
||||||
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
|
content_items: vec![
|
||||||
text: "Ticket is open".into(),
|
CoreDynamicToolCallOutputContentItem::InputText {
|
||||||
}],
|
text: "Ticket is open".into(),
|
||||||
|
},
|
||||||
|
CoreDynamicToolCallOutputContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,AAA".into(),
|
||||||
|
},
|
||||||
|
CoreDynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
|
||||||
|
},
|
||||||
|
],
|
||||||
success: true,
|
success: true,
|
||||||
error: None,
|
error: None,
|
||||||
duration: Duration::from_millis(42),
|
duration: Duration::from_millis(42),
|
||||||
@@ -2903,9 +2914,17 @@ mod tests {
|
|||||||
tool: "lookup_ticket".into(),
|
tool: "lookup_ticket".into(),
|
||||||
arguments: serde_json::json!({"id":"ABC-123"}),
|
arguments: serde_json::json!({"id":"ABC-123"}),
|
||||||
status: DynamicToolCallStatus::Completed,
|
status: DynamicToolCallStatus::Completed,
|
||||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
content_items: Some(vec![
|
||||||
text: "Ticket is open".into(),
|
DynamicToolCallOutputContentItem::InputText {
|
||||||
}]),
|
text: "Ticket is open".into(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,AAA".into(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
|
||||||
|
},
|
||||||
|
]),
|
||||||
success: Some(true),
|
success: Some(true),
|
||||||
duration_ms: Some(42),
|
duration_ms: Some(42),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1552,6 +1552,8 @@ pub enum DynamicToolCallOutputContentItem {
|
|||||||
InputText { text: String },
|
InputText { text: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
InputImage { image_url: String },
|
InputImage { image_url: String },
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
InputAudio { audio_url: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
|
impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
|
||||||
@@ -1565,6 +1567,9 @@ impl From<codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem>
|
|||||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
||||||
image_url,
|
image_url,
|
||||||
} => Self::InputImage { image_url },
|
} => Self::InputImage { image_url },
|
||||||
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url,
|
||||||
|
} => Self::InputAudio { audio_url },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1578,6 +1583,9 @@ impl From<DynamicToolCallOutputContentItem>
|
|||||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||||
Self::InputImage { image_url }
|
Self::InputImage { image_url }
|
||||||
}
|
}
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
Self::InputAudio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2719,6 +2719,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
|||||||
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText {
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputText {
|
||||||
text: "ok".to_string(),
|
text: "ok".to_string(),
|
||||||
},
|
},
|
||||||
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
|
},
|
||||||
|
codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
]),
|
]),
|
||||||
success: Some(true),
|
success: Some(true),
|
||||||
error: None,
|
error: None,
|
||||||
@@ -2733,9 +2739,17 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
|||||||
tool: "lookup".to_string(),
|
tool: "lookup".to_string(),
|
||||||
arguments: json!({"id": "123"}),
|
arguments: json!({"id": "123"}),
|
||||||
status: DynamicToolCallStatus::Completed,
|
status: DynamicToolCallStatus::Completed,
|
||||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
content_items: Some(vec![
|
||||||
text: "ok".to_string(),
|
DynamicToolCallOutputContentItem::InputText {
|
||||||
}]),
|
text: "ok".to_string(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
|
]),
|
||||||
success: Some(true),
|
success: Some(true),
|
||||||
duration_ms: Some(5),
|
duration_ms: Some(5),
|
||||||
}
|
}
|
||||||
@@ -3954,7 +3968,7 @@ fn dynamic_tool_response_serializes_content_items() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
fn dynamic_tool_response_serializes_text_image_and_audio_content_items() {
|
||||||
let value = serde_json::to_value(DynamicToolCallResponse {
|
let value = serde_json::to_value(DynamicToolCallResponse {
|
||||||
content_items: vec![
|
content_items: vec![
|
||||||
DynamicToolCallOutputContentItem::InputText {
|
DynamicToolCallOutputContentItem::InputText {
|
||||||
@@ -3963,6 +3977,9 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
|||||||
DynamicToolCallOutputContentItem::InputImage {
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
image_url: "data:image/png;base64,AAA".to_string(),
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
},
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
],
|
],
|
||||||
success: true,
|
success: true,
|
||||||
})
|
})
|
||||||
@@ -3979,6 +3996,10 @@ fn dynamic_tool_response_serializes_text_and_image_content_items() {
|
|||||||
{
|
{
|
||||||
"type": "inputImage",
|
"type": "inputImage",
|
||||||
"imageUrl": "data:image/png;base64,AAA"
|
"imageUrl": "data:image/png;base64,AAA"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "inputAudio",
|
||||||
|
"audioUrl": "data:audio/wav;base64,YXVkaW8="
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"success": true,
|
"success": true,
|
||||||
|
|||||||
@@ -1671,7 +1671,7 @@ The server also emits item lifecycle notifications around the request:
|
|||||||
3. Client response.
|
3. Client response.
|
||||||
4. `item/completed` with `item.type = "dynamicToolCall"`, final `status`, and the returned `contentItems`/`success`.
|
4. `item/completed` with `item.type = "dynamicToolCall"`, final `status`, and the returned `contentItems`/`success`.
|
||||||
|
|
||||||
The client must respond with content items. Use `inputText` for text and `inputImage` for inline data URLs. Remote HTTP(S) image URLs make the dynamic tool response invalid.
|
The client must respond with content items. Use `inputText` for text, `inputImage` for inline image data URLs, and `inputAudio` for inline audio data URLs. Audio data URLs accept wav, mp3, m4a, webm, and ogg media types. Remote HTTP(S) image URLs and non-data audio URLs make the dynamic tool response invalid.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -1679,7 +1679,8 @@ The client must respond with content items. Use `inputText` for text and `inputI
|
|||||||
"result": {
|
"result": {
|
||||||
"contentItems": [
|
"contentItems": [
|
||||||
{ "type": "inputText", "text": "Ticket ABC-123 is open." },
|
{ "type": "inputText", "text": "Ticket ABC-123 is open." },
|
||||||
{ "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" }
|
{ "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" },
|
||||||
|
{ "type": "inputAudio", "audioUrl": "data:audio/wav;base64,AAA" }
|
||||||
],
|
],
|
||||||
"success": true
|
"success": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ use crate::image_url::is_remote_image_url;
|
|||||||
use crate::outgoing_message::ClientRequestResult;
|
use crate::outgoing_message::ClientRequestResult;
|
||||||
use crate::server_request_error::is_turn_transition_server_request_error;
|
use crate::server_request_error::is_turn_transition_server_request_error;
|
||||||
|
|
||||||
|
const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL";
|
||||||
|
|
||||||
pub(crate) async fn on_call_response(
|
pub(crate) async fn on_call_response(
|
||||||
call_id: String,
|
call_id: String,
|
||||||
receiver: oneshot::Receiver<ClientRequestResult>,
|
receiver: oneshot::Receiver<ClientRequestResult>,
|
||||||
@@ -71,6 +73,23 @@ fn decode_response(value: serde_json::Value) -> (DynamicToolCallResponse, Option
|
|||||||
);
|
);
|
||||||
fallback_response(REMOTE_IMAGE_URL_ERROR)
|
fallback_response(REMOTE_IMAGE_URL_ERROR)
|
||||||
}
|
}
|
||||||
|
Ok(response)
|
||||||
|
if response.content_items.iter().any(|item| {
|
||||||
|
matches!(
|
||||||
|
item,
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio { audio_url }
|
||||||
|
if !audio_url
|
||||||
|
.get(.."data:".len())
|
||||||
|
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||||
|
)
|
||||||
|
}) =>
|
||||||
|
{
|
||||||
|
error!(
|
||||||
|
message = INVALID_AUDIO_URL_ERROR,
|
||||||
|
"dynamic tool response was invalid"
|
||||||
|
);
|
||||||
|
fallback_response(INVALID_AUDIO_URL_ERROR)
|
||||||
|
}
|
||||||
Ok(response) => (response, None),
|
Ok(response) => (response, None),
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
error!("failed to deserialize DynamicToolCallResponse: {err}");
|
error!("failed to deserialize DynamicToolCallResponse: {err}");
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use app_test_support::TestAppServer;
|
|||||||
use app_test_support::create_final_assistant_message_sse_response;
|
use app_test_support::create_final_assistant_message_sse_response;
|
||||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||||
use app_test_support::to_response;
|
use app_test_support::to_response;
|
||||||
|
use app_test_support::write_models_cache_with_models;
|
||||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||||
use codex_app_server_protocol::DynamicToolCallParams;
|
use codex_app_server_protocol::DynamicToolCallParams;
|
||||||
use codex_app_server_protocol::DynamicToolCallResponse;
|
use codex_app_server_protocol::DynamicToolCallResponse;
|
||||||
@@ -28,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
|||||||
use codex_protocol::models::FunctionCallOutputBody;
|
use codex_protocol::models::FunctionCallOutputBody;
|
||||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||||
use codex_protocol::models::FunctionCallOutputPayload;
|
use codex_protocol::models::FunctionCallOutputPayload;
|
||||||
|
use core_test_support::load_default_config_for_test;
|
||||||
use core_test_support::responses;
|
use core_test_support::responses;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -39,6 +41,8 @@ use tokio::time::timeout;
|
|||||||
use wiremock::MockServer;
|
use wiremock::MockServer;
|
||||||
|
|
||||||
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
|
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
|
||||||
|
const INLINE_AUDIO_DATA_URL: &str = "data:audio/wav;base64,YXVkaW8=";
|
||||||
|
const INVALID_AUDIO_URL_ERROR: &str = "audio URLs must use an inline data URL";
|
||||||
const REMOTE_IMAGE_URL_ERROR: &str =
|
const REMOTE_IMAGE_URL_ERROR: &str =
|
||||||
"remote image URLs are not supported; use an inline data URL instead";
|
"remote image URLs are not supported; use an inline data URL instead";
|
||||||
|
|
||||||
@@ -592,6 +596,9 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result<PendingDynami
|
|||||||
|
|
||||||
let codex_home = TempDir::new()?;
|
let codex_home = TempDir::new()?;
|
||||||
create_config_toml(codex_home.path(), &server.uri())?;
|
create_config_toml(codex_home.path(), &server.uri())?;
|
||||||
|
let config = load_default_config_for_test(&codex_home).await;
|
||||||
|
let model_info = codex_core::test_support::construct_model_info_offline("mock-model", &config);
|
||||||
|
write_models_cache_with_models(codex_home.path(), vec![model_info])?;
|
||||||
|
|
||||||
let mut mcp = TestAppServer::builder()
|
let mut mcp = TestAppServer::builder()
|
||||||
.with_codex_home(codex_home.path())
|
.with_codex_home(codex_home.path())
|
||||||
@@ -680,7 +687,7 @@ async fn start_function_dynamic_tool_call(call_id: &str) -> Result<PendingDynami
|
|||||||
|
|
||||||
/// Ensures dynamic tool call responses can include structured content items.
|
/// Ensures dynamic tool call responses can include structured content items.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<()> {
|
async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
|
||||||
let call_id = "dyn-call-items-1";
|
let call_id = "dyn-call-items-1";
|
||||||
let PendingDynamicToolCall {
|
let PendingDynamicToolCall {
|
||||||
mut mcp,
|
mut mcp,
|
||||||
@@ -696,22 +703,21 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
|
|||||||
DynamicToolCallOutputContentItem::InputImage {
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
image_url: TINY_PNG_DATA_URL.to_string(),
|
image_url: TINY_PNG_DATA_URL.to_string(),
|
||||||
},
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
// Audio remains available on the app-server item, but the base truncation policy intentionally
|
||||||
|
// omits it from the model-visible payload until the history/accounting layer.
|
||||||
|
let model_content_items = vec![
|
||||||
|
FunctionCallOutputContentItem::InputText {
|
||||||
|
text: "dynamic-ok".to_string(),
|
||||||
|
},
|
||||||
|
FunctionCallOutputContentItem::InputImage {
|
||||||
|
image_url: TINY_PNG_DATA_URL.to_string(),
|
||||||
|
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||||
|
},
|
||||||
];
|
];
|
||||||
let content_items = response_content_items
|
|
||||||
.clone()
|
|
||||||
.into_iter()
|
|
||||||
.map(|item| match item {
|
|
||||||
DynamicToolCallOutputContentItem::InputText { text } => {
|
|
||||||
FunctionCallOutputContentItem::InputText { text }
|
|
||||||
}
|
|
||||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
|
||||||
FunctionCallOutputContentItem::InputImage {
|
|
||||||
image_url,
|
|
||||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<FunctionCallOutputContentItem>>();
|
|
||||||
let response = DynamicToolCallResponse {
|
let response = DynamicToolCallResponse {
|
||||||
content_items: response_content_items,
|
content_items: response_content_items,
|
||||||
success: true,
|
success: true,
|
||||||
@@ -741,6 +747,9 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
|
|||||||
DynamicToolCallOutputContentItem::InputImage {
|
DynamicToolCallOutputContentItem::InputImage {
|
||||||
image_url: TINY_PNG_DATA_URL.to_string(),
|
image_url: TINY_PNG_DATA_URL.to_string(),
|
||||||
},
|
},
|
||||||
|
DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: INLINE_AUDIO_DATA_URL.to_string(),
|
||||||
|
},
|
||||||
])
|
])
|
||||||
);
|
);
|
||||||
assert_eq!(success, Some(true));
|
assert_eq!(success, Some(true));
|
||||||
@@ -777,12 +786,12 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
|
|||||||
.context("expected function_call_output in follow-up request")?;
|
.context("expected function_call_output in follow-up request")?;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
payload.body,
|
payload.body,
|
||||||
FunctionCallOutputBody::ContentItems(content_items.clone())
|
FunctionCallOutputBody::ContentItems(model_content_items.clone())
|
||||||
);
|
);
|
||||||
assert_eq!(payload.success, None);
|
assert_eq!(payload.success, None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::to_string(&payload)?,
|
serde_json::to_string(&payload)?,
|
||||||
serde_json::to_string(&content_items)?
|
serde_json::to_string(&model_content_items)?
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -844,6 +853,62 @@ async fn dynamic_tool_remote_image_response_becomes_model_visible_error() -> Res
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn dynamic_tool_remote_audio_response_becomes_model_visible_error() -> Result<()> {
|
||||||
|
let call_id = "dyn-call-remote-audio";
|
||||||
|
let PendingDynamicToolCall {
|
||||||
|
mut mcp,
|
||||||
|
server,
|
||||||
|
request_id,
|
||||||
|
params,
|
||||||
|
} = start_function_dynamic_tool_call(call_id).await?;
|
||||||
|
|
||||||
|
let response = DynamicToolCallResponse {
|
||||||
|
content_items: vec![DynamicToolCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "https://example.com/tool.wav".to_string(),
|
||||||
|
}],
|
||||||
|
success: true,
|
||||||
|
};
|
||||||
|
mcp.send_response(request_id, serde_json::to_value(response)?)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||||
|
assert_eq!(completed.thread_id, params.thread_id);
|
||||||
|
assert_eq!(completed.turn_id, params.turn_id);
|
||||||
|
let ThreadItem::DynamicToolCall {
|
||||||
|
status,
|
||||||
|
content_items,
|
||||||
|
success,
|
||||||
|
..
|
||||||
|
} = completed.item
|
||||||
|
else {
|
||||||
|
panic!("expected dynamic tool call item");
|
||||||
|
};
|
||||||
|
assert_eq!(status, DynamicToolCallStatus::Failed);
|
||||||
|
assert_eq!(
|
||||||
|
content_items,
|
||||||
|
Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||||
|
text: INVALID_AUDIO_URL_ERROR.to_string(),
|
||||||
|
}])
|
||||||
|
);
|
||||||
|
assert_eq!(success, Some(false));
|
||||||
|
|
||||||
|
timeout(
|
||||||
|
DEFAULT_READ_TIMEOUT,
|
||||||
|
mcp.read_stream_until_notification_message("turn/completed"),
|
||||||
|
)
|
||||||
|
.await??;
|
||||||
|
|
||||||
|
let output = responses_bodies(&server)
|
||||||
|
.await?
|
||||||
|
.iter()
|
||||||
|
.find_map(|body| function_call_output_raw_output(body, call_id))
|
||||||
|
.context("expected function_call_output output in follow-up request")?;
|
||||||
|
assert_eq!(output, json!(INVALID_AUDIO_URL_ERROR));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn responses_bodies(server: &MockServer) -> Result<Vec<Value>> {
|
async fn responses_bodies(server: &MockServer) -> Result<Vec<Value>> {
|
||||||
let requests = server
|
let requests = server
|
||||||
.received_requests()
|
.received_requests()
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co
|
|||||||
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
|
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
|
||||||
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
|
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
|
||||||
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
|
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
|
||||||
|
- `audio(audioUrlOrItem: string | { audio_url: string } | AudioContent)`: Appends an audio item. `audio_url` should be a base64-encoded `data:` URL. To forward an MCP tool audio block, pass an individual `AudioContent` block from `result.content`, for example `audio(result.content[0])`.
|
||||||
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. HTTP(S) URLs are not supported.
|
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint. HTTP(S) URLs are not supported.
|
||||||
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
|
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
|
||||||
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
|
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
|
||||||
@@ -880,6 +881,7 @@ bar"
|
|||||||
fn exec_description_mentions_timeout_helpers() {
|
fn exec_description_mentions_timeout_helpers() {
|
||||||
let description =
|
let description =
|
||||||
build_exec_tool_description(&[], &[], &BTreeMap::new(), /*code_mode_only*/ false);
|
build_exec_tool_description(&[], &[], &BTreeMap::new(), /*code_mode_only*/ false);
|
||||||
|
assert!(description.contains("`audio(audioUrlOrItem:"));
|
||||||
assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`"));
|
assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`"));
|
||||||
assert!(description.contains("`clearTimeout(timeoutId?: number)`"));
|
assert!(description.contains("`clearTimeout(timeoutId?: number)`"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -130,6 +130,9 @@ fn content_items() -> Vec<WireContentItem> {
|
|||||||
image_url: "data:image/png;base64,original".to_string(),
|
image_url: "data:image/png;base64,original".to_string(),
|
||||||
detail: Some(WireImageDetail::Original),
|
detail: Some(WireImageDetail::Original),
|
||||||
},
|
},
|
||||||
|
WireContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,6 +160,10 @@ fn content_items_json() -> Value {
|
|||||||
"image_url": "data:image/png;base64,original",
|
"image_url": "data:image/png;base64,original",
|
||||||
"detail": "original",
|
"detail": "original",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"type": "input_audio",
|
||||||
|
"audio_url": "data:audio/wav;base64,YXVkaW8=",
|
||||||
|
},
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -245,6 +245,9 @@ pub enum WireContentItem {
|
|||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
detail: Option<WireImageDetail>,
|
detail: Option<WireImageDetail>,
|
||||||
},
|
},
|
||||||
|
InputAudio {
|
||||||
|
audio_url: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<FunctionCallOutputContentItem> for WireContentItem {
|
impl From<FunctionCallOutputContentItem> for WireContentItem {
|
||||||
@@ -255,6 +258,9 @@ impl From<FunctionCallOutputContentItem> for WireContentItem {
|
|||||||
image_url,
|
image_url,
|
||||||
detail: detail.map(Into::into),
|
detail: detail.map(Into::into),
|
||||||
},
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
Self::InputAudio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -267,6 +273,7 @@ impl From<WireContentItem> for FunctionCallOutputContentItem {
|
|||||||
image_url,
|
image_url,
|
||||||
detail: detail.map(Into::into),
|
detail: detail.map(Into::into),
|
||||||
},
|
},
|
||||||
|
WireContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,4 +23,7 @@ pub enum FunctionCallOutputContentItem {
|
|||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
detail: Option<ImageDetail>,
|
detail: Option<ImageDetail>,
|
||||||
},
|
},
|
||||||
|
InputAudio {
|
||||||
|
audio_url: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
@@ -56,5 +56,8 @@ pub(super) fn output_item(item: FunctionCallOutputContentItem) -> CellOutputItem
|
|||||||
ImageDetail::Original => CellImageDetail::Original,
|
ImageDetail::Original => CellImageDetail::Original,
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
CellOutputItem::Audio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,6 +5,7 @@ use super::RuntimeEvent;
|
|||||||
use super::RuntimeState;
|
use super::RuntimeState;
|
||||||
use super::timers;
|
use super::timers;
|
||||||
use super::value::json_to_v8;
|
use super::value::json_to_v8;
|
||||||
|
use super::value::normalize_output_audio;
|
||||||
use super::value::normalize_output_image;
|
use super::value::normalize_output_image;
|
||||||
use super::value::serialize_output_text;
|
use super::value::serialize_output_text;
|
||||||
use super::value::throw_type_error;
|
use super::value::throw_type_error;
|
||||||
@@ -96,6 +97,26 @@ pub(super) fn text_callback(
|
|||||||
retval.set(v8::undefined(scope).into());
|
retval.set(v8::undefined(scope).into());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn audio_callback(
|
||||||
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
|
args: v8::FunctionCallbackArguments,
|
||||||
|
mut retval: v8::ReturnValue<v8::Value>,
|
||||||
|
) {
|
||||||
|
let value = if args.length() == 0 {
|
||||||
|
v8::undefined(scope).into()
|
||||||
|
} else {
|
||||||
|
args.get(0)
|
||||||
|
};
|
||||||
|
let audio_item = match normalize_output_audio(scope, value) {
|
||||||
|
Ok(audio_item) => audio_item,
|
||||||
|
Err(()) => return,
|
||||||
|
};
|
||||||
|
if let Some(state) = scope.get_slot::<RuntimeState>() {
|
||||||
|
let _ = state.event_tx.send(RuntimeEvent::ContentItem(audio_item));
|
||||||
|
}
|
||||||
|
retval.set(v8::undefined(scope).into());
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn image_callback(
|
pub(super) fn image_callback(
|
||||||
scope: &mut v8::PinScope<'_, '_>,
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
args: v8::FunctionCallbackArguments,
|
args: v8::FunctionCallbackArguments,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use super::RuntimeState;
|
use super::RuntimeState;
|
||||||
|
use super::callbacks::audio_callback;
|
||||||
use super::callbacks::clear_timeout_callback;
|
use super::callbacks::clear_timeout_callback;
|
||||||
use super::callbacks::exit_callback;
|
use super::callbacks::exit_callback;
|
||||||
use super::callbacks::generated_image_callback;
|
use super::callbacks::generated_image_callback;
|
||||||
@@ -24,6 +25,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
|
|||||||
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
|
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
|
||||||
let text = helper_function(scope, "text", text_callback)?;
|
let text = helper_function(scope, "text", text_callback)?;
|
||||||
let image = helper_function(scope, "image", image_callback)?;
|
let image = helper_function(scope, "image", image_callback)?;
|
||||||
|
let audio = helper_function(scope, "audio", audio_callback)?;
|
||||||
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
|
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
|
||||||
let store = helper_function(scope, "store", store_callback)?;
|
let store = helper_function(scope, "store", store_callback)?;
|
||||||
let load = helper_function(scope, "load", load_callback)?;
|
let load = helper_function(scope, "load", load_callback)?;
|
||||||
@@ -37,6 +39,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
|
|||||||
set_global(scope, global, "setTimeout", set_timeout.into())?;
|
set_global(scope, global, "setTimeout", set_timeout.into())?;
|
||||||
set_global(scope, global, "text", text.into())?;
|
set_global(scope, global, "text", text.into())?;
|
||||||
set_global(scope, global, "image", image.into())?;
|
set_global(scope, global, "image", image.into())?;
|
||||||
|
set_global(scope, global, "audio", audio.into())?;
|
||||||
set_global(scope, global, "generatedImage", generated_image.into())?;
|
set_global(scope, global, "generatedImage", generated_image.into())?;
|
||||||
set_global(scope, global, "store", store.into())?;
|
set_global(scope, global, "store", store.into())?;
|
||||||
set_global(scope, global, "load", load.into())?;
|
set_global(scope, global, "load", load.into())?;
|
||||||
|
|||||||
@@ -5,9 +5,12 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem;
|
|||||||
use codex_code_mode_protocol::ImageDetail;
|
use codex_code_mode_protocol::ImageDetail;
|
||||||
|
|
||||||
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
|
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
|
||||||
|
const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
|
||||||
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
|
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
|
||||||
const INVALID_IMAGE_URL_ERROR: &str =
|
const INVALID_IMAGE_URL_ERROR: &str =
|
||||||
"Tool call failed: invalid image output. Pass a base64 data URI instead";
|
"Tool call failed: invalid image output. Pass a base64 data URI instead";
|
||||||
|
const INVALID_AUDIO_URL_ERROR: &str =
|
||||||
|
"Tool call failed: invalid audio output. Pass a base64 data URI instead";
|
||||||
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
|
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
|
||||||
|
|
||||||
pub(super) fn serialize_output_text(
|
pub(super) fn serialize_output_text(
|
||||||
@@ -182,6 +185,104 @@ fn parse_image_detail_value<'s>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn normalize_output_audio(
|
||||||
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
|
value: v8::Local<'_, v8::Value>,
|
||||||
|
) -> Result<FunctionCallOutputContentItem, ()> {
|
||||||
|
let result = (|| -> Result<FunctionCallOutputContentItem, String> {
|
||||||
|
let audio_url = if value.is_string() {
|
||||||
|
value.to_rust_string_lossy(scope)
|
||||||
|
} else if value.is_object() && !value.is_array() {
|
||||||
|
let object = v8::Local::<v8::Object>::try_from(value)
|
||||||
|
.map_err(|_| AUDIO_HELPER_EXPECTS_MESSAGE.to_string())?;
|
||||||
|
if let Some(audio_url) = parse_non_mcp_output_audio(scope, object)? {
|
||||||
|
audio_url
|
||||||
|
} else {
|
||||||
|
parse_mcp_output_audio(scope, value)?
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
};
|
||||||
|
|
||||||
|
if audio_url.is_empty() {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
}
|
||||||
|
let Some((scheme, _)) = audio_url.split_once(':') else {
|
||||||
|
return Err(INVALID_AUDIO_URL_ERROR.to_string());
|
||||||
|
};
|
||||||
|
if !scheme.eq_ignore_ascii_case("data") {
|
||||||
|
return Err(INVALID_AUDIO_URL_ERROR.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
|
||||||
|
})();
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(item) => Ok(item),
|
||||||
|
Err(error_text) => {
|
||||||
|
throw_type_error(scope, &error_text);
|
||||||
|
Err(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_non_mcp_output_audio(
|
||||||
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
|
object: v8::Local<'_, v8::Object>,
|
||||||
|
) -> Result<Option<String>, String> {
|
||||||
|
let audio_url_key = v8::String::new(scope, "audio_url")
|
||||||
|
.ok_or_else(|| "failed to allocate audio helper keys".to_string())?;
|
||||||
|
let Some(audio_url) = object.get(scope, audio_url_key.into()) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if audio_url.is_undefined() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if !audio_url.is_string() {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
}
|
||||||
|
Ok(Some(audio_url.to_rust_string_lossy(scope)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_mcp_output_audio(
|
||||||
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
|
value: v8::Local<'_, v8::Value>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
let Some(result) = v8_value_to_json(scope, value)? else {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
};
|
||||||
|
let JsonValue::Object(result) = result else {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
};
|
||||||
|
let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else {
|
||||||
|
return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string());
|
||||||
|
};
|
||||||
|
if item_type != "audio" {
|
||||||
|
return Err(format!(
|
||||||
|
"audio only accepts MCP audio blocks, got \"{item_type}\""
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let data = result
|
||||||
|
.get("data")
|
||||||
|
.and_then(JsonValue::as_str)
|
||||||
|
.ok_or_else(|| "audio expected MCP audio data".to_string())?;
|
||||||
|
if data.is_empty() {
|
||||||
|
return Err("audio expected MCP audio data".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.to_ascii_lowercase().starts_with("data:") {
|
||||||
|
Ok(data.to_string())
|
||||||
|
} else {
|
||||||
|
let mime_type = result
|
||||||
|
.get("mimeType")
|
||||||
|
.or_else(|| result.get("mime_type"))
|
||||||
|
.and_then(JsonValue::as_str)
|
||||||
|
.filter(|mime_type| !mime_type.is_empty())
|
||||||
|
.unwrap_or("application/octet-stream");
|
||||||
|
Ok(format!("data:{mime_type};base64,{data}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn v8_value_to_json(
|
pub(super) fn v8_value_to_json(
|
||||||
scope: &mut v8::PinScope<'_, '_>,
|
scope: &mut v8::PinScope<'_, '_>,
|
||||||
value: v8::Local<'_, v8::Value>,
|
value: v8::Local<'_, v8::Value>,
|
||||||
|
|||||||
@@ -403,6 +403,9 @@ fn output_item(item: runtime::OutputItem) -> FunctionCallOutputContentItem {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
runtime::OutputItem::Audio { audio_url } => {
|
||||||
|
FunctionCallOutputContentItem::InputAudio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -737,6 +737,7 @@ async fn output_helpers_return_undefined() {
|
|||||||
const returnsUndefined = [
|
const returnsUndefined = [
|
||||||
text("first"),
|
text("first"),
|
||||||
image("data:image/png;base64,AAA"),
|
image("data:image/png;base64,AAA"),
|
||||||
|
audio("data:audio/wav;base64,YXVkaW8="),
|
||||||
notify("ping"),
|
notify("ping"),
|
||||||
].map((value) => value === undefined);
|
].map((value) => value === undefined);
|
||||||
text(JSON.stringify(returnsUndefined));
|
text(JSON.stringify(returnsUndefined));
|
||||||
@@ -760,8 +761,11 @@ text(JSON.stringify(returnsUndefined));
|
|||||||
image_url: "data:image/png;base64,AAA".to_string(),
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
||||||
},
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
FunctionCallOutputContentItem::InputText {
|
FunctionCallOutputContentItem::InputText {
|
||||||
text: "[true,true,true]".to_string(),
|
text: "[true,true,true,true]".to_string(),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
error_text: None,
|
error_text: None,
|
||||||
@@ -769,6 +773,79 @@ text(JSON.stringify(returnsUndefined));
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn audio_helper_accepts_audio_url_object_and_raw_mcp_audio_block() {
|
||||||
|
let service = InProcessCodeModeSession::new();
|
||||||
|
|
||||||
|
let response = execute(
|
||||||
|
&service,
|
||||||
|
ExecuteRequest {
|
||||||
|
source: r#"
|
||||||
|
audio({
|
||||||
|
audio_url: "data:audio/mpeg;base64,YXVkaW8=",
|
||||||
|
});
|
||||||
|
audio({
|
||||||
|
type: "audio",
|
||||||
|
data: "YXVkaW8=",
|
||||||
|
mimeType: "audio/wav",
|
||||||
|
});
|
||||||
|
"#
|
||||||
|
.to_string(),
|
||||||
|
yield_time_ms: None,
|
||||||
|
..execute_request("")
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response,
|
||||||
|
RuntimeResponse::Result {
|
||||||
|
cell_id: cell_id("1"),
|
||||||
|
content_items: vec![
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
error_text: None,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn audio_helper_rejects_non_data_urls() {
|
||||||
|
for source in [
|
||||||
|
r#"audio("https://example.com/audio.wav");"#,
|
||||||
|
r#"audio({ audio_url: "file:///tmp/audio.wav" });"#,
|
||||||
|
] {
|
||||||
|
let service = InProcessCodeModeSession::new();
|
||||||
|
|
||||||
|
let response = execute(
|
||||||
|
&service,
|
||||||
|
ExecuteRequest {
|
||||||
|
source: source.to_string(),
|
||||||
|
yield_time_ms: None,
|
||||||
|
..execute_request("")
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response,
|
||||||
|
RuntimeResponse::Result {
|
||||||
|
cell_id: cell_id("1"),
|
||||||
|
content_items: Vec::new(),
|
||||||
|
error_text: Some(
|
||||||
|
"Tool call failed: invalid audio output. Pass a base64 data URI instead"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() {
|
async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() {
|
||||||
let service = InProcessCodeModeSession::new();
|
let service = InProcessCodeModeSession::new();
|
||||||
|
|||||||
@@ -61,6 +61,9 @@ pub(crate) enum OutputItem {
|
|||||||
image_url: String,
|
image_url: String,
|
||||||
detail: Option<ImageDetail>,
|
detail: Option<ImageDetail>,
|
||||||
},
|
},
|
||||||
|
Audio {
|
||||||
|
audio_url: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Requested image fidelity for an output image.
|
/// Requested image fidelity for an output image.
|
||||||
|
|||||||
@@ -612,13 +612,8 @@ async fn execute_mcp_tool_call(
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("tool call error: {e:?}"))?;
|
.map_err(|e| format!("tool call error: {e:?}"))?;
|
||||||
let result = sanitize_mcp_tool_result_for_model(
|
let result =
|
||||||
turn_context
|
sanitize_mcp_tool_result_for_model(&turn_context.model_info.input_modalities, Ok(result))?;
|
||||||
.model_info
|
|
||||||
.input_modalities
|
|
||||||
.contains(&InputModality::Image),
|
|
||||||
Ok(result),
|
|
||||||
)?;
|
|
||||||
Ok(maybe_request_codex_apps_auth_elicitation(
|
Ok(maybe_request_codex_apps_auth_elicitation(
|
||||||
sess,
|
sess,
|
||||||
turn_context,
|
turn_context,
|
||||||
@@ -813,10 +808,12 @@ async fn maybe_mark_thread_memory_mode_polluted(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn sanitize_mcp_tool_result_for_model(
|
fn sanitize_mcp_tool_result_for_model(
|
||||||
supports_image_input: bool,
|
input_modalities: &[InputModality],
|
||||||
result: Result<CallToolResult, String>,
|
result: Result<CallToolResult, String>,
|
||||||
) -> Result<CallToolResult, String> {
|
) -> Result<CallToolResult, String> {
|
||||||
if supports_image_input {
|
let supports_image_input = input_modalities.contains(&InputModality::Image);
|
||||||
|
let supports_audio_input = input_modalities.contains(&InputModality::Audio);
|
||||||
|
if supports_image_input && supports_audio_input {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -825,13 +822,19 @@ fn sanitize_mcp_tool_result_for_model(
|
|||||||
.content
|
.content
|
||||||
.iter()
|
.iter()
|
||||||
.map(|block| {
|
.map(|block| {
|
||||||
if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str)
|
if let Some(content_type) = block.get("type").and_then(serde_json::Value::as_str) {
|
||||||
&& content_type == "image"
|
if content_type == "image" && !supports_image_input {
|
||||||
{
|
return serde_json::json!({
|
||||||
return serde_json::json!({
|
"type": "text",
|
||||||
"type": "text",
|
"text": "<image content omitted because you do not support image input>",
|
||||||
"text": "<image content omitted because you do not support image input>",
|
});
|
||||||
});
|
}
|
||||||
|
if content_type == "audio" && !supports_audio_input {
|
||||||
|
return serde_json::json!({
|
||||||
|
"type": "text",
|
||||||
|
"text": "<audio content omitted because you do not support audio input>",
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
block.clone()
|
block.clone()
|
||||||
|
|||||||
@@ -962,7 +962,7 @@ fn codex_apps_connectors_support_persistent_approval() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
|
fn sanitize_mcp_tool_result_for_model_rewrites_unsupported_media_content() {
|
||||||
let result = Ok(CallToolResult {
|
let result = Ok(CallToolResult {
|
||||||
content: vec![
|
content: vec![
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
@@ -970,6 +970,11 @@ fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
|
|||||||
"data": "Zm9v",
|
"data": "Zm9v",
|
||||||
"mimeType": "image/png",
|
"mimeType": "image/png",
|
||||||
}),
|
}),
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "audio",
|
||||||
|
"data": "YmFy",
|
||||||
|
"mimeType": "audio/wav",
|
||||||
|
}),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"text": "hello",
|
"text": "hello",
|
||||||
@@ -980,39 +985,59 @@ fn sanitize_mcp_tool_result_for_model_rewrites_image_content() {
|
|||||||
meta: None,
|
meta: None,
|
||||||
});
|
});
|
||||||
|
|
||||||
let got = sanitize_mcp_tool_result_for_model(/*supports_image_input*/ false, result)
|
let got = sanitize_mcp_tool_result_for_model(&[InputModality::Text], result)
|
||||||
.expect("sanitized result");
|
.expect("sanitized result");
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
got.content,
|
got,
|
||||||
vec![
|
CallToolResult {
|
||||||
serde_json::json!({
|
content: vec![
|
||||||
"type": "text",
|
serde_json::json!({
|
||||||
"text": "<image content omitted because you do not support image input>",
|
"type": "text",
|
||||||
}),
|
"text": "<image content omitted because you do not support image input>",
|
||||||
serde_json::json!({
|
}),
|
||||||
"type": "text",
|
serde_json::json!({
|
||||||
"text": "hello",
|
"type": "text",
|
||||||
}),
|
"text": "<audio content omitted because you do not support audio input>",
|
||||||
]
|
}),
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "text",
|
||||||
|
"text": "hello",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
structured_content: None,
|
||||||
|
is_error: Some(false),
|
||||||
|
meta: None,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sanitize_mcp_tool_result_for_model_preserves_image_when_supported() {
|
fn sanitize_mcp_tool_result_for_model_preserves_supported_media() {
|
||||||
let original = CallToolResult {
|
let original = CallToolResult {
|
||||||
content: vec![serde_json::json!({
|
content: vec![
|
||||||
"type": "image",
|
serde_json::json!({
|
||||||
"data": "Zm9v",
|
"type": "image",
|
||||||
"mimeType": "image/png",
|
"data": "Zm9v",
|
||||||
})],
|
"mimeType": "image/png",
|
||||||
|
}),
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "audio",
|
||||||
|
"data": "YmFy",
|
||||||
|
"mimeType": "audio/wav",
|
||||||
|
}),
|
||||||
|
],
|
||||||
structured_content: Some(serde_json::json!({"x": 1})),
|
structured_content: Some(serde_json::json!({"x": 1})),
|
||||||
is_error: Some(false),
|
is_error: Some(false),
|
||||||
meta: Some(serde_json::json!({"k": "v"})),
|
meta: Some(serde_json::json!({"k": "v"})),
|
||||||
};
|
};
|
||||||
|
|
||||||
let got = sanitize_mcp_tool_result_for_model(
|
let got = sanitize_mcp_tool_result_for_model(
|
||||||
/*supports_image_input*/ true,
|
&[
|
||||||
|
InputModality::Text,
|
||||||
|
InputModality::Image,
|
||||||
|
InputModality::Audio,
|
||||||
|
],
|
||||||
Ok(original.clone()),
|
Ok(original.clone()),
|
||||||
)
|
)
|
||||||
.expect("unsanitized result");
|
.expect("unsanitized result");
|
||||||
|
|||||||
@@ -42,6 +42,9 @@ impl IntoProtocol<FunctionCallOutputContentItem>
|
|||||||
.or(Some(DEFAULT_IMAGE_DETAIL)),
|
.or(Some(DEFAULT_IMAGE_DETAIL)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
codex_code_mode::FunctionCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
FunctionCallOutputContentItem::InputAudio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3500,6 +3500,7 @@ text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort()));
|
|||||||
"WeakSet",
|
"WeakSet",
|
||||||
"__codexContentItems",
|
"__codexContentItems",
|
||||||
"add_content",
|
"add_content",
|
||||||
|
"audio",
|
||||||
"decodeURI",
|
"decodeURI",
|
||||||
"decodeURIComponent",
|
"decodeURIComponent",
|
||||||
"encodeURI",
|
"encodeURI",
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ pub enum DynamicToolCallOutputContentItem {
|
|||||||
InputText { text: String },
|
InputText { text: String },
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
InputImage { image_url: String },
|
InputImage { image_url: String },
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
InputAudio { audio_url: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Former flat `SessionMeta` shape, including the old `exposeToContext` flag.
|
/// Former flat `SessionMeta` shape, including the old `exposeToContext` flag.
|
||||||
|
|||||||
@@ -1856,7 +1856,7 @@ pub enum FunctionCallOutputContentItem {
|
|||||||
///
|
///
|
||||||
/// This conversion is intentionally lossy:
|
/// This conversion is intentionally lossy:
|
||||||
/// - only `input_text` items are included
|
/// - only `input_text` items are included
|
||||||
/// - image items are ignored
|
/// - image and audio items are ignored
|
||||||
///
|
///
|
||||||
/// We use this helper where callers still need a string representation (for
|
/// We use this helper where callers still need a string representation (for
|
||||||
/// example telemetry previews or legacy string-only output paths) while keeping
|
/// example telemetry previews or legacy string-only output paths) while keeping
|
||||||
@@ -1899,6 +1899,9 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
|
|||||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputAudio { audio_url } => {
|
||||||
|
Self::InputAudio { audio_url }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2118,6 +2121,14 @@ fn convert_mcp_content_to_items(
|
|||||||
#[serde(rename = "_meta", default)]
|
#[serde(rename = "_meta", default)]
|
||||||
meta: Option<serde_json::Value>,
|
meta: Option<serde_json::Value>,
|
||||||
},
|
},
|
||||||
|
#[serde(rename = "audio")]
|
||||||
|
Audio {
|
||||||
|
data: String,
|
||||||
|
#[serde(rename = "mimeType", alias = "mime_type")]
|
||||||
|
mime_type: Option<String>,
|
||||||
|
#[serde(rename = "_meta", default)]
|
||||||
|
_meta: Option<serde_json::Value>,
|
||||||
|
},
|
||||||
#[serde(other)]
|
#[serde(other)]
|
||||||
Unknown,
|
Unknown,
|
||||||
}
|
}
|
||||||
@@ -2171,6 +2182,18 @@ fn convert_mcp_content_to_items(
|
|||||||
.or(Some(DEFAULT_IMAGE_DETAIL)),
|
.or(Some(DEFAULT_IMAGE_DETAIL)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(McpContent::Audio {
|
||||||
|
data, mime_type, ..
|
||||||
|
}) => {
|
||||||
|
saw_content_item = true;
|
||||||
|
let audio_url = if data.starts_with("data:") {
|
||||||
|
data
|
||||||
|
} else {
|
||||||
|
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
|
||||||
|
format!("data:{mime_type};base64,{data}")
|
||||||
|
};
|
||||||
|
FunctionCallOutputContentItem::InputAudio { audio_url }
|
||||||
|
}
|
||||||
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
|
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
|
||||||
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
|
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
|
||||||
},
|
},
|
||||||
@@ -2726,7 +2749,36 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn convert_mcp_content_to_items_returns_none_without_images() {
|
fn convert_mcp_audio_content_builds_data_urls_and_preserves_existing_data_urls() {
|
||||||
|
let contents = vec![
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "audio",
|
||||||
|
"data": "Zm9v",
|
||||||
|
"mimeType": "audio/wav",
|
||||||
|
"_meta": {"source": "microphone"},
|
||||||
|
}),
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "audio",
|
||||||
|
"data": "data:audio/ogg;base64,YmFy",
|
||||||
|
"mimeType": "audio/ogg",
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
convert_mcp_content_to_items(&contents),
|
||||||
|
Some(vec![
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,Zm9v".to_string(),
|
||||||
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/ogg;base64,YmFy".to_string(),
|
||||||
|
},
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_mcp_content_to_items_returns_none_without_media() {
|
||||||
let contents = vec![serde_json::json!({
|
let contents = vec![serde_json::json!({
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"text": "hello",
|
"text": "hello",
|
||||||
@@ -2755,7 +2807,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn function_call_output_content_items_to_text_ignores_blank_text_and_images() {
|
fn function_call_output_content_items_to_text_ignores_blank_text_and_media() {
|
||||||
let content_items = vec![
|
let content_items = vec![
|
||||||
FunctionCallOutputContentItem::InputText {
|
FunctionCallOutputContentItem::InputText {
|
||||||
text: " ".to_string(),
|
text: " ".to_string(),
|
||||||
@@ -2764,6 +2816,9 @@ mod tests {
|
|||||||
image_url: "data:image/png;base64,AAA".to_string(),
|
image_url: "data:image/png;base64,AAA".to_string(),
|
||||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||||
},
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||||
|
},
|
||||||
FunctionCallOutputContentItem::EncryptedContent {
|
FunctionCallOutputContentItem::EncryptedContent {
|
||||||
encrypted_content: "enc_opaque".to_string(),
|
encrypted_content: "enc_opaque".to_string(),
|
||||||
},
|
},
|
||||||
@@ -2953,6 +3008,54 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serializes_audio_outputs_as_array() -> Result<()> {
|
||||||
|
let call_tool_result = CallToolResult {
|
||||||
|
content: vec![
|
||||||
|
serde_json::json!({"type":"text","text":"caption"}),
|
||||||
|
serde_json::json!({"type":"audio","data":"BASE64","mimeType":"audio/wav"}),
|
||||||
|
],
|
||||||
|
structured_content: None,
|
||||||
|
is_error: Some(false),
|
||||||
|
meta: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let payload = call_tool_result.into_function_call_output_payload();
|
||||||
|
assert_eq!(
|
||||||
|
payload,
|
||||||
|
FunctionCallOutputPayload {
|
||||||
|
body: FunctionCallOutputBody::ContentItems(vec![
|
||||||
|
FunctionCallOutputContentItem::InputText {
|
||||||
|
text: "caption".into(),
|
||||||
|
},
|
||||||
|
FunctionCallOutputContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,BASE64".into(),
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
success: Some(true),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let item = ResponseInputItem::FunctionCallOutput {
|
||||||
|
call_id: "call1".into(),
|
||||||
|
output: payload,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(item)?,
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "function_call_output",
|
||||||
|
"call_id": "call1",
|
||||||
|
"output": [
|
||||||
|
{"type": "input_text", "text": "caption"},
|
||||||
|
{"type": "input_audio", "audio_url": "data:audio/wav;base64,BASE64"},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serializes_custom_tool_image_outputs_as_array() -> Result<()> {
|
fn serializes_custom_tool_image_outputs_as_array() -> Result<()> {
|
||||||
let item = ResponseInputItem::CustomToolCallOutput {
|
let item = ResponseInputItem::CustomToolCallOutput {
|
||||||
|
|||||||
Reference in new issue
Block a user