mirror of
https://github.com/openai/codex.git
synced 2026-11-19 23:05:23 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
1 parent
4110342321
commit
b01c3986fd
70 files changed
+4421
-69
No files matched your search
@@ -21,7 +21,7 @@ use codex_app_server_protocol::ThreadGoalStatus;
|
||||
#[cfg(target_os = "windows")]
|
||||
use codex_config::types::WindowsSandboxModeToml;
|
||||
|
||||
const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
|
||||
pub(super) const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
|
||||
|
||||
impl App {
|
||||
pub(super) async fn handle_event(
|
||||
@@ -785,6 +785,23 @@ impl App {
|
||||
.apply_reserve_fallback_to_pending_turn(&mut op);
|
||||
}
|
||||
let is_user_turn = matches!(&op, AppCommand::UserTurn { .. });
|
||||
let is_realtime_stop = matches!(&op, AppCommand::RealtimeConversationStop { .. });
|
||||
let realtime_stop_thread_id = match &op {
|
||||
AppCommand::RealtimeConversationStop { thread_id } => Some(*thread_id),
|
||||
_ => None,
|
||||
};
|
||||
let realtime_speech_delivery_id = match &op {
|
||||
AppCommand::RealtimeConversationSpeech { delivery_id, .. } => {
|
||||
Some(*delivery_id)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let is_realtime_conversation = matches!(
|
||||
&op,
|
||||
AppCommand::RealtimeConversationStart { .. }
|
||||
| AppCommand::RealtimeConversationStop { .. }
|
||||
| AppCommand::RealtimeConversationSpeech { .. }
|
||||
);
|
||||
if is_user_turn {
|
||||
let screen_size = tui.terminal.last_known_screen_size;
|
||||
self.handle_draw_pre_render(tui, screen_size)?;
|
||||
@@ -797,6 +814,10 @@ impl App {
|
||||
}
|
||||
self.chat_widget.prepare_local_op_submission(&op);
|
||||
if let Err(err) = self.submit_active_thread_op(app_server, op).await {
|
||||
if let Some(delivery_id) = realtime_speech_delivery_id {
|
||||
self.chat_widget
|
||||
.restore_undelivered_realtime_speech(delivery_id);
|
||||
}
|
||||
if self.recover_transport_error(&err)
|
||||
{
|
||||
return Ok(AppRunControl::Continue);
|
||||
@@ -817,10 +838,22 @@ impl App {
|
||||
&& self
|
||||
.chat_widget
|
||||
.handle_turn_start_rejection(format!("Failed to start turn: {err:#}"));
|
||||
if !handled {
|
||||
if is_realtime_conversation {
|
||||
let message = format!("Voice conversation failed: {err:#}");
|
||||
if is_realtime_stop {
|
||||
if self.chat_widget.thread_id() == realtime_stop_thread_id {
|
||||
self.chat_widget.reset_realtime_conversation();
|
||||
self.chat_widget.add_error_message(message);
|
||||
}
|
||||
} else {
|
||||
self.chat_widget.on_realtime_error(message);
|
||||
}
|
||||
tracing::error!(error = ?err, "realtime conversation request failed");
|
||||
} else if handled {
|
||||
tracing::error!(error = ?err, "failed to start turn through app server");
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
tracing::error!(error = ?err, "failed to start turn through app server");
|
||||
}
|
||||
}
|
||||
AppEvent::ConfirmSafetyBufferedRetry {
|
||||
@@ -1656,6 +1689,47 @@ impl App {
|
||||
self.sync_active_thread_personality_setting(app_server, personality)
|
||||
.await;
|
||||
}
|
||||
AppEvent::RealtimeWebrtcOfferCreated {
|
||||
thread_id,
|
||||
attempt_id,
|
||||
result,
|
||||
} => {
|
||||
if self.chat_widget.thread_id() == Some(thread_id) {
|
||||
self.chat_widget
|
||||
.on_realtime_webrtc_offer_created(thread_id, attempt_id, result);
|
||||
} else if let Ok(offer) = result {
|
||||
offer.handle.close();
|
||||
}
|
||||
}
|
||||
AppEvent::RealtimeWebrtcConnected {
|
||||
thread_id,
|
||||
attempt_id,
|
||||
result,
|
||||
} => {
|
||||
if self.chat_widget.thread_id() == Some(thread_id) {
|
||||
self.chat_widget
|
||||
.on_realtime_webrtc_connected(attempt_id, result);
|
||||
}
|
||||
}
|
||||
AppEvent::StopRealtimeConversation { thread_id } => {
|
||||
match tokio::time::timeout(
|
||||
SHUTDOWN_FIRST_EXIT_TIMEOUT,
|
||||
app_server.thread_realtime_stop(thread_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => tracing::warn!(
|
||||
%thread_id,
|
||||
%error,
|
||||
"failed to stop voice conversation after switching threads"
|
||||
),
|
||||
Err(_) => tracing::warn!(
|
||||
%thread_id,
|
||||
"timed out stopping voice conversation after switching threads"
|
||||
),
|
||||
}
|
||||
}
|
||||
AppEvent::SettingsSelectionClosed => {
|
||||
self.app_event_tx.send(AppEvent::SettingsSelectionSettled);
|
||||
}
|
||||
@@ -3369,6 +3443,7 @@ impl App {
|
||||
})
|
||||
}
|
||||
ExitMode::Immediate => {
|
||||
self.stop_realtime_conversation(app_server).await;
|
||||
self.pending_shutdown_exit_thread_id = None;
|
||||
AppRunControl::Exit(ExitReason::UserRequested)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user