mirror of
https://github.com/openai/codex.git
synced 2026-10-20 00:18:49 +00:00
Accept voice response audio before captions on quiet turns (#44925)
## Why Starting a voice turn while the speaker is idle suppresses response audio until assistant captions arrive, preventing audio that arrives first from being accepted. ## What changed Make speaker suppression conditional when user transcripts start a new turn. Keep idle output ready for audio while preserving interruption of active assistant output and older, uncaptioned speech that is queued or accepted. ## Testing Add regression coverage for quiet turns with incremental or final-only user transcripts and for interruption of older pending speech across queue and caption states. GitOrigin-RevId: 288d2071766a1d0c1b8c7a10c0ab434dc4d7cee1
This commit is contained in:
committed by
copyberry
parent
7ef70f95d5
commit
1b5e27c7f0
@@ -1201,7 +1201,7 @@ impl ChatWidget {
|
||||
self.realtime_conversation.interruption_acknowledged_until =
|
||||
Some(Instant::now() + INTERRUPTION_ACKNOWLEDGMENT);
|
||||
}
|
||||
self.suppress_realtime_speaker();
|
||||
self.suppress_active_realtime_speaker();
|
||||
}
|
||||
if active
|
||||
&& role == "assistant"
|
||||
@@ -1454,7 +1454,7 @@ impl ChatWidget {
|
||||
.input_generation
|
||||
.wrapping_add(/*rhs*/ 1);
|
||||
self.realtime_conversation.latest_input_was_voice = true;
|
||||
self.suppress_realtime_speaker();
|
||||
self.suppress_active_realtime_speaker();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +76,39 @@ impl ChatWidget {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn suppress_active_realtime_speaker(&mut self) {
|
||||
// Quiet turns must not wait for captions before accepting their first
|
||||
// audio packets. Only interrupt output that may belong to an older turn.
|
||||
if self
|
||||
.realtime_conversation
|
||||
.assistant_transcript_generation
|
||||
.is_some()
|
||||
|| self
|
||||
.realtime_conversation
|
||||
.pending_speech
|
||||
.iter()
|
||||
.any(|pending| {
|
||||
pending.input_generation != self.realtime_conversation.input_generation
|
||||
&& !pending.captioned
|
||||
&& matches!(
|
||||
pending.state,
|
||||
PendingSpeechState::Queued(_) | PendingSpeechState::Accepted
|
||||
)
|
||||
})
|
||||
|| self.realtime_conversation.speaker_level > 0
|
||||
|| self
|
||||
.realtime_conversation
|
||||
.speaker_active_until
|
||||
.is_some_and(|deadline| deadline > Instant::now())
|
||||
|| self
|
||||
.realtime_conversation
|
||||
.speaker_suppression_generation
|
||||
.is_some()
|
||||
{
|
||||
self.suppress_realtime_speaker();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn suppress_realtime_speaker(&mut self) {
|
||||
self.realtime_conversation.speaker_suppression_generation =
|
||||
Some(self.realtime_conversation.input_generation);
|
||||
|
||||
@@ -416,7 +416,7 @@ async fn newer_voice_delegation_on_the_same_turn_speaks_only_its_own_answer() {
|
||||
if text.as_str() == "answer to the newer question"
|
||||
));
|
||||
let owner = chat.realtime_conversation.speaker_suppression_generation;
|
||||
assert_eq!(owner, Some(chat.realtime_conversation.input_generation));
|
||||
assert_eq!(owner, None);
|
||||
chat.on_realtime_transcript_delta("assistant".to_string(), "answer".to_string());
|
||||
let owner = chat.realtime_conversation.speaker_suppression_generation;
|
||||
assert!(owner.is_none());
|
||||
|
||||
@@ -122,6 +122,7 @@ async fn completed_user_transcript_keeps_the_old_assistant_turn_suppressed() {
|
||||
async fn final_only_assistant_transcript_cannot_release_an_interruption() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
activate_voice(&mut chat);
|
||||
chat.realtime_conversation.speaker_level = 1;
|
||||
chat.on_realtime_transcript_delta("user".into(), "Stop".into());
|
||||
let generation = chat.realtime_conversation.input_generation;
|
||||
chat.on_realtime_transcript_done("user".into(), "Stop".into());
|
||||
@@ -153,7 +154,7 @@ async fn transcript_deltas_track_the_current_speaker() {
|
||||
chat.on_realtime_transcript_done("assistant".to_string(), "Old".to_string());
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.speaker_suppression_generation,
|
||||
Some(chat.realtime_conversation.input_generation)
|
||||
None
|
||||
);
|
||||
|
||||
chat.on_realtime_transcript_done("user".to_string(), "Hello there".to_string());
|
||||
@@ -163,9 +164,9 @@ async fn transcript_deltas_track_the_current_speaker() {
|
||||
|
||||
chat.on_realtime_transcript_delta("user".to_string(), "Again".to_string());
|
||||
chat.on_realtime_transcript_done("user".to_string(), "Again".to_string());
|
||||
assert!(owner(&chat).is_some());
|
||||
assert!(owner(&chat).is_none());
|
||||
chat.on_realtime_transcript_delta("assistant".to_string(), "".to_string());
|
||||
assert!(owner(&chat).is_some());
|
||||
assert!(owner(&chat).is_none());
|
||||
chat.on_realtime_transcript_delta("assistant".to_string(), "Hi".to_string());
|
||||
assert!(owner(&chat).is_none());
|
||||
|
||||
@@ -772,3 +773,57 @@ async fn spoken_user_transcript_preserves_red_chevron_and_canonical_history() {
|
||||
.any(|line| line.to_string() == "› hello world")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quiet_voice_turns_accept_audio_before_assistant_captions() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
activate_voice(&mut chat);
|
||||
for user_delta in [Some("First question"), None, Some("Third question")] {
|
||||
if let Some(delta) = user_delta {
|
||||
chat.on_realtime_transcript_delta("user".into(), delta.into());
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.speaker_suppression_generation,
|
||||
None
|
||||
);
|
||||
}
|
||||
chat.on_realtime_transcript_done("user".into(), "Question".into());
|
||||
// Audio can arrive before captions: keep the helper accepting it now.
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.speaker_suppression_generation,
|
||||
None
|
||||
);
|
||||
chat.on_realtime_transcript_delta("assistant".into(), "Answer".into());
|
||||
chat.on_realtime_transcript_done("assistant".into(), "Answer".into());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_voice_turn_interrupts_uncaptioned_queued_speech() {
|
||||
use super::super::PendingRealtimeSpeech;
|
||||
use super::super::PendingSpeechState;
|
||||
|
||||
for (state, captioned, expected) in [
|
||||
(PendingSpeechState::Queued(1), false, Some(1)),
|
||||
(PendingSpeechState::Accepted, false, Some(1)),
|
||||
(PendingSpeechState::AwaitingTurn, false, None),
|
||||
(PendingSpeechState::Accepted, true, None),
|
||||
] {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
let thread_id = activate_voice(&mut chat);
|
||||
chat.realtime_conversation
|
||||
.pending_speech
|
||||
.push_back(PendingRealtimeSpeech {
|
||||
state,
|
||||
captioned,
|
||||
input_generation: 0,
|
||||
thread_id,
|
||||
turn_id: "old-turn".into(),
|
||||
item: agent_item("answer", "Old answer", Some(MessagePhase::FinalAnswer)),
|
||||
});
|
||||
chat.on_realtime_transcript_delta("user".into(), "New question".into());
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.speaker_suppression_generation,
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ async fn voice_live_transcript_renders_beside_the_streamed_cell() {
|
||||
let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
chat.local_settings.tui.animations = false;
|
||||
activate_voice_for_thread(&mut chat, ThreadId::new());
|
||||
chat.update_realtime_footer();
|
||||
chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
|
||||
vec![Line::from("Agent answer arriving").into()],
|
||||
/*is_first_line*/ true,
|
||||
|
||||
Reference in New Issue
Block a user