Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-17 23:14:37 +00:00
1 parent e98765c212
commit 97dbd01a3d
49 files changed
+1330 -33

No files matched your search

@@ -0,0 +1,6 @@
---
source: tui/src/chatwidget/tests/app_server.rs
expression: "lines_to_single_string(&inserted[0])"
---
› Please inspect the attachments.
@@ -538,6 +538,43 @@ async fn live_app_server_user_message_item_completed_does_not_duplicate_rendered
assert!(drain_insert_history(&mut rx).is_empty());
}
#[tokio::test]
async fn live_app_server_user_message_omits_unsupported_media() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
chat.handle_server_notification(
ServerNotification::ItemCompleted(ItemCompletedNotification {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
completed_at_ms: 0,
item: AppServerThreadItem::UserMessage {
id: "user-1".to_string(),
client_id: None,
content: vec![
AppServerUserInput::Text {
text: "Please inspect the attachments.".to_string(),
text_elements: Vec::new(),
},
AppServerUserInput::Audio {
url: "https://example.com/one.wav".to_string(),
},
AppServerUserInput::LocalAudio {
path: test_path_buf("/tmp/two.wav"),
},
],
},
}),
/*replay_kind*/ None,
);
let inserted = drain_insert_history(&mut rx);
assert_eq!(inserted.len(), 1);
assert_chatwidget_snapshot!(
"live_app_server_user_message_omits_unsupported_media",
lines_to_single_string(&inserted[0]),
);
}
#[tokio::test]
async fn live_app_server_turn_completed_clears_working_status_after_answer_item() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
+19 -3
View File
@@ -589,7 +589,11 @@ pub(crate) fn mention_bindings_from_user_inputs(
path: path.clone(),
})
}
UserInput::Text { .. } | UserInput::Image { .. } | UserInput::LocalImage { .. } => None,
UserInput::Text { .. }
| UserInput::Image { .. }
| UserInput::LocalImage { .. }
| UserInput::Audio { .. }
| UserInput::LocalAudio { .. } => None,
})
.collect();
mention_bindings.sort_by_key(|binding| {
@@ -655,7 +659,10 @@ impl ChatWidget {
match item {
UserInput::Text { text, .. } => message.push_str(text),
UserInput::Image { .. } | UserInput::LocalImage { .. } => image_count += 1,
UserInput::Skill { .. } | UserInput::Mention { .. } => {}
UserInput::Audio { .. }
| UserInput::LocalAudio { .. }
| UserInput::Skill { .. }
| UserInput::Mention { .. } => {}
}
}
@@ -666,6 +673,12 @@ impl ChatWidget {
}
pub(crate) fn user_message_display_from_inputs(items: &[UserInput]) -> UserMessageDisplay {
if items
.iter()
.any(|item| matches!(item, UserInput::Audio { .. } | UserInput::LocalAudio { .. }))
{
tracing::warn!("audio user inputs are not supported by the TUI and will be omitted");
}
let mut message = String::new();
let mut remote_image_urls = Vec::new();
let mut local_images = Vec::new();
@@ -694,7 +707,10 @@ impl ChatWidget {
),
UserInput::Image { url, .. } => remote_image_urls.push(url.clone()),
UserInput::LocalImage { path, .. } => local_images.push(path.clone()),
UserInput::Skill { .. } | UserInput::Mention { .. } => {}
UserInput::Audio { .. }
| UserInput::LocalAudio { .. }
| UserInput::Skill { .. }
| UserInput::Mention { .. } => {}
}
}