Add live WebRTC voice conversations to the TUI (#43581)

## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson authored and copyberry committed 2026-09-07 20:25:30 +00:00
1 parent 4110342321
commit b01c3986fd
70 files changed
+4421 -69

No files matched your search

+78 -3
View File
@@ -21,7 +21,7 @@ use codex_app_server_protocol::ThreadGoalStatus;
#[cfg(target_os = "windows")]
use codex_config::types::WindowsSandboxModeToml;
const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
pub(super) const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
impl App {
pub(super) async fn handle_event(
@@ -785,6 +785,23 @@ impl App {
.apply_reserve_fallback_to_pending_turn(&mut op);
}
let is_user_turn = matches!(&op, AppCommand::UserTurn { .. });
let is_realtime_stop = matches!(&op, AppCommand::RealtimeConversationStop { .. });
let realtime_stop_thread_id = match &op {
AppCommand::RealtimeConversationStop { thread_id } => Some(*thread_id),
_ => None,
};
let realtime_speech_delivery_id = match &op {
AppCommand::RealtimeConversationSpeech { delivery_id, .. } => {
Some(*delivery_id)
}
_ => None,
};
let is_realtime_conversation = matches!(
&op,
AppCommand::RealtimeConversationStart { .. }
| AppCommand::RealtimeConversationStop { .. }
| AppCommand::RealtimeConversationSpeech { .. }
);
if is_user_turn {
let screen_size = tui.terminal.last_known_screen_size;
self.handle_draw_pre_render(tui, screen_size)?;
@@ -797,6 +814,10 @@ impl App {
}
self.chat_widget.prepare_local_op_submission(&op);
if let Err(err) = self.submit_active_thread_op(app_server, op).await {
if let Some(delivery_id) = realtime_speech_delivery_id {
self.chat_widget
.restore_undelivered_realtime_speech(delivery_id);
}
if self.recover_transport_error(&err)
{
return Ok(AppRunControl::Continue);
@@ -817,10 +838,22 @@ impl App {
&& self
.chat_widget
.handle_turn_start_rejection(format!("Failed to start turn: {err:#}"));
if !handled {
if is_realtime_conversation {
let message = format!("Voice conversation failed: {err:#}");
if is_realtime_stop {
if self.chat_widget.thread_id() == realtime_stop_thread_id {
self.chat_widget.reset_realtime_conversation();
self.chat_widget.add_error_message(message);
}
} else {
self.chat_widget.on_realtime_error(message);
}
tracing::error!(error = ?err, "realtime conversation request failed");
} else if handled {
tracing::error!(error = ?err, "failed to start turn through app server");
} else {
return Err(err);
}
tracing::error!(error = ?err, "failed to start turn through app server");
}
}
AppEvent::ConfirmSafetyBufferedRetry {
@@ -1656,6 +1689,47 @@ impl App {
self.sync_active_thread_personality_setting(app_server, personality)
.await;
}
AppEvent::RealtimeWebrtcOfferCreated {
thread_id,
attempt_id,
result,
} => {
if self.chat_widget.thread_id() == Some(thread_id) {
self.chat_widget
.on_realtime_webrtc_offer_created(thread_id, attempt_id, result);
} else if let Ok(offer) = result {
offer.handle.close();
}
}
AppEvent::RealtimeWebrtcConnected {
thread_id,
attempt_id,
result,
} => {
if self.chat_widget.thread_id() == Some(thread_id) {
self.chat_widget
.on_realtime_webrtc_connected(attempt_id, result);
}
}
AppEvent::StopRealtimeConversation { thread_id } => {
match tokio::time::timeout(
SHUTDOWN_FIRST_EXIT_TIMEOUT,
app_server.thread_realtime_stop(thread_id),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(error)) => tracing::warn!(
%thread_id,
%error,
"failed to stop voice conversation after switching threads"
),
Err(_) => tracing::warn!(
%thread_id,
"timed out stopping voice conversation after switching threads"
),
}
}
AppEvent::SettingsSelectionClosed => {
self.app_event_tx.send(AppEvent::SettingsSelectionSettled);
}
@@ -3369,6 +3443,7 @@ impl App {
})
}
ExitMode::Immediate => {
self.stop_realtime_conversation(app_server).await;
self.pending_shutdown_exit_thread_id = None;
AppRunControl::Exit(ExitReason::UserRequested)
}