mirror of
https://github.com/openai/codex.git
synced 2026-11-04 00:24:43 +00:00
Add audio variants to user input protocols (#33923)
## What changed - Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas. - Preserve audio fields when converting app-server inputs and response content into thread items. - Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input. ## Testing - Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation. GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
1 parent
e98765c212
commit
97dbd01a3d
49 files changed
+1330
-33
No files matched your search
+6
@@ -0,0 +1,6 @@
|
||||
---
|
||||
source: tui/src/chatwidget/tests/app_server.rs
|
||||
expression: "lines_to_single_string(&inserted[0])"
|
||||
---
|
||||
|
||||
› Please inspect the attachments.
|
||||
@@ -538,6 +538,43 @@ async fn live_app_server_user_message_item_completed_does_not_duplicate_rendered
|
||||
assert!(drain_insert_history(&mut rx).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_app_server_user_message_omits_unsupported_media() {
|
||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
|
||||
chat.handle_server_notification(
|
||||
ServerNotification::ItemCompleted(ItemCompletedNotification {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
completed_at_ms: 0,
|
||||
item: AppServerThreadItem::UserMessage {
|
||||
id: "user-1".to_string(),
|
||||
client_id: None,
|
||||
content: vec![
|
||||
AppServerUserInput::Text {
|
||||
text: "Please inspect the attachments.".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
AppServerUserInput::Audio {
|
||||
url: "https://example.com/one.wav".to_string(),
|
||||
},
|
||||
AppServerUserInput::LocalAudio {
|
||||
path: test_path_buf("/tmp/two.wav"),
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
/*replay_kind*/ None,
|
||||
);
|
||||
|
||||
let inserted = drain_insert_history(&mut rx);
|
||||
assert_eq!(inserted.len(), 1);
|
||||
assert_chatwidget_snapshot!(
|
||||
"live_app_server_user_message_omits_unsupported_media",
|
||||
lines_to_single_string(&inserted[0]),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_app_server_turn_completed_clears_working_status_after_answer_item() {
|
||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
|
||||
@@ -589,7 +589,11 @@ pub(crate) fn mention_bindings_from_user_inputs(
|
||||
path: path.clone(),
|
||||
})
|
||||
}
|
||||
UserInput::Text { .. } | UserInput::Image { .. } | UserInput::LocalImage { .. } => None,
|
||||
UserInput::Text { .. }
|
||||
| UserInput::Image { .. }
|
||||
| UserInput::LocalImage { .. }
|
||||
| UserInput::Audio { .. }
|
||||
| UserInput::LocalAudio { .. } => None,
|
||||
})
|
||||
.collect();
|
||||
mention_bindings.sort_by_key(|binding| {
|
||||
@@ -655,7 +659,10 @@ impl ChatWidget {
|
||||
match item {
|
||||
UserInput::Text { text, .. } => message.push_str(text),
|
||||
UserInput::Image { .. } | UserInput::LocalImage { .. } => image_count += 1,
|
||||
UserInput::Skill { .. } | UserInput::Mention { .. } => {}
|
||||
UserInput::Audio { .. }
|
||||
| UserInput::LocalAudio { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -666,6 +673,12 @@ impl ChatWidget {
|
||||
}
|
||||
|
||||
pub(crate) fn user_message_display_from_inputs(items: &[UserInput]) -> UserMessageDisplay {
|
||||
if items
|
||||
.iter()
|
||||
.any(|item| matches!(item, UserInput::Audio { .. } | UserInput::LocalAudio { .. }))
|
||||
{
|
||||
tracing::warn!("audio user inputs are not supported by the TUI and will be omitted");
|
||||
}
|
||||
let mut message = String::new();
|
||||
let mut remote_image_urls = Vec::new();
|
||||
let mut local_images = Vec::new();
|
||||
@@ -694,7 +707,10 @@ impl ChatWidget {
|
||||
),
|
||||
UserInput::Image { url, .. } => remote_image_urls.push(url.clone()),
|
||||
UserInput::LocalImage { path, .. } => local_images.push(path.clone()),
|
||||
UserInput::Skill { .. } | UserInput::Mention { .. } => {}
|
||||
UserInput::Audio { .. }
|
||||
| UserInput::LocalAudio { .. }
|
||||
| UserInput::Skill { .. }
|
||||
| UserInput::Mention { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user