github/codex
4
0 Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-13 22:47:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add live WebRTC voice conversations to the TUI (#43581)

Browse Source 
## What changed

- Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling.
- Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches.
- Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken.
- Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects.
- Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs.

## Testing

Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering.

GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
Benjamin Carlsson authored and copyberry committed 2026-09-07 20:25:30 +00:00
1 parent 4110342321
commit b01c3986fd
70 files changed
+4421 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

codex-rs/tui/src/chatwidget/protocol.rs
+116 -15
View File
@@ -61,7 +61,10 @@ impl ChatWidget {
}
ServerNotification::TurnStarted(notification) => {
if replay_kind.is_none() {
self.clear_misalignment_for_new_turn(&notification.turn.id);
self.clear_misalignment_for_new_turn(
&notification.turn.id,
MisalignmentTurnSource::ServerNotification,
);
}
self.turn_lifecycle.last_turn_id = Some(notification.turn.id);
self.last_non_retry_error = None;
@@ -80,18 +83,43 @@ impl ChatWidget {
self.handle_item_completed_notification(notification, replay_kind);
}
ServerNotification::AgentMessageDelta(notification) => {
self.on_agent_message_delta(notification.delta);
if !self.is_realtime_delegated_reasoning_turn(&notification.turn_id)
&& (from_replay
|| !self.is_realtime_delegated_agent_item(
&notification.turn_id,
&notification.item_id,
))
{
self.on_agent_message_delta(notification.delta);
}
}
ServerNotification::PlanDelta(notification) => self.on_plan_delta(notification.delta),
ServerNotification::ReasoningSummaryTextDelta(notification) => {
self.on_agent_reasoning_delta(notification.delta);
}
ServerNotification::ReasoningTextDelta(notification) => {
if self.config.show_raw_agent_reasoning {
if !self.is_realtime_delegated_reasoning_item(
&notification.turn_id,
&notification.item_id,
) {
self.on_agent_reasoning_delta(notification.delta);
}
}
ServerNotification::ReasoningSummaryPartAdded(_) => self.on_reasoning_section_break(),
ServerNotification::ReasoningTextDelta(notification) => {
if self.config.show_raw_agent_reasoning
&& !self.is_realtime_delegated_reasoning_item(
&notification.turn_id,
&notification.item_id,
)
{
self.on_agent_reasoning_delta(notification.delta);
}
}
ServerNotification::ReasoningSummaryPartAdded(notification) => {
if !self.is_realtime_delegated_reasoning_item(
&notification.turn_id,
&notification.item_id,
) {
self.on_reasoning_section_break();
}
}
ServerNotification::TerminalInteraction(notification) => {
self.on_terminal_interaction(notification.process_id, notification.stdin)
}
@@ -235,6 +263,36 @@ impl ChatWidget {
self.on_shutdown_complete();
}
}
ServerNotification::ThreadRealtimeSdp(notification) => {
if !from_replay {
self.on_realtime_conversation_sdp(notification.sdp);
}
}
ServerNotification::ThreadRealtimeStarted(_) => {
if !from_replay {
self.on_realtime_conversation_started();
}
}
ServerNotification::ThreadRealtimeTranscriptDelta(notification) => {
if !from_replay {
self.on_realtime_transcript_delta(notification.role, notification.delta);
}
}
ServerNotification::ThreadRealtimeTranscriptDone(notification) => {
if !from_replay {
self.on_realtime_transcript_done(notification.role, notification.text);
}
}
ServerNotification::ThreadRealtimeError(notification) => {
if !from_replay {
self.on_realtime_error(notification.message);
}
}
ServerNotification::ThreadRealtimeClosed(notification) => {
if !from_replay {
self.on_realtime_conversation_closed(notification.reason);
}
}
ServerNotification::ServerRequestResolved(_)
| ServerNotification::AccountUpdated(_)
| ServerNotification::AccountRateLimitsUpdated(_)
@@ -264,17 +322,11 @@ impl ChatWidget {
| ServerNotification::TurnModerationMetadata(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
| ServerNotification::ThreadRealtimeStarted(_)
| ServerNotification::ThreadRealtimeItemAdded(_)
| ServerNotification::ThreadRealtimeItemStarted(_)
| ServerNotification::ThreadRealtimeItemTranscriptDelta(_)
| ServerNotification::ThreadRealtimeItemCompleted(_)
| ServerNotification::ThreadRealtimeOutputAudioDelta(_)
| ServerNotification::ThreadRealtimeError(_)
| ServerNotification::ThreadRealtimeClosed(_)
| ServerNotification::ThreadRealtimeSdp(_)
| ServerNotification::ThreadRealtimeTranscriptDelta(_)
| ServerNotification::ThreadRealtimeTranscriptDone(_)
| ServerNotification::WindowsWorldWritableWarning(_)
| ServerNotification::WindowsSandboxSetupCompleted(_)
| ServerNotification::AccountLoginCompleted(_)
@@ -310,7 +362,12 @@ impl ChatWidget {
text,
phase: Some(MessagePhase::FinalAnswer) | None,
..
} => Some((item.clone(), id.clone(), text.clone())),
} if !self
.is_realtime_delegated_reasoning_turn(&notification.turn.id)
|| !realtime::is_private_realtime_agent_item(item) =>
{
Some((item.clone(), id.clone(), text.clone()))
}
_ => None,
});
if let Some((item, id, _)) = &last_agent_message
@@ -329,6 +386,11 @@ impl ChatWidget {
.map_or(ThreadItemRenderSource::Live, ThreadItemRenderSource::Replay),
);
}
if replay_kind.is_none()
&& let Some((item, _, _)) = &last_agent_message
{
self.speak_completed_realtime_delegation(&notification.turn.id, item);
}
self.last_non_retry_error = None;
let completion = self.completion_cell(&notification.turn, replay_kind);
self.on_task_complete(
@@ -355,7 +417,10 @@ impl ChatWidget {
&& error.codex_error_info
== Some(AppServerCodexErrorInfo::MisalignmentPolicyViolation)
{
self.on_misalignment_error(Some(notification.turn.id), error.misalignment);
self.on_misalignment_error(
Some(notification.turn.id.clone()),
error.misalignment,
);
self.last_non_retry_error = None;
} else if self.last_non_retry_error.as_ref()
== Some(&(notification.turn.id.clone(), error.message.clone()))
@@ -373,6 +438,9 @@ impl ChatWidget {
}
TurnStatus::InProgress => {}
}
if replay_kind.is_none() {
self.finish_realtime_turn(&notification.turn.id);
}
self.thread_usage.replaying_turn_completion = was_replaying_turn_completion;
}
@@ -382,6 +450,28 @@ impl ChatWidget {
replay_kind: Option<ReplayKind>,
) {
match notification.item {
ThreadItem::UserMessage { content, .. } if replay_kind.is_none() => {
self.note_realtime_user_item_started(&notification.turn_id, &content);
}
ThreadItem::UserMessage { content, .. }
if realtime::realtime_delegation_input(&content).is_some() =>
{
self.remember_realtime_delegated_reasoning_turn(&notification.turn_id);
}
ThreadItem::AgentMessage { id, .. } if replay_kind.is_none() => {
self.is_realtime_delegated_agent_item(&notification.turn_id, &id);
}
ThreadItem::Reasoning { id, .. }
if replay_kind.is_none()
&& !self.is_realtime_delegated_reasoning_turn(&notification.turn_id) =>
{
// A later voice handoff can steer this turn without making an
// already-started typed reasoning item private.
self.realtime_conversation
.agent_items
.entry((notification.turn_id.clone(), id))
.or_insert(realtime::RealtimeAgentItemOrigin::Typed);
}
ThreadItem::ContextCompaction { id }
if !matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)) =>
{
@@ -441,6 +531,17 @@ impl ChatWidget {
notification: ItemCompletedNotification,
replay_kind: Option<ReplayKind>,
) {
if replay_kind.is_none()
&& self.is_realtime_delegated_reasoning_turn(&notification.turn_id)
&& realtime::is_private_realtime_agent_item(&notification.item)
&& !matches!(&notification.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. }
if matches!(
self.realtime_conversation.agent_items.get(&(notification.turn_id.clone(), id.clone())),
Some(realtime::RealtimeAgentItemOrigin::Typed)
))
{
return;
}
// Buffered live notifications can introduce questions; historical turn replay cannot.
if replay_kind == Some(ReplayKind::ThreadSnapshot)
&& let ThreadItem::AgentMessage {