Accept voice response audio before captions on quiet turns (#44925)

## Why

Starting a voice turn while the speaker is idle suppresses response audio until assistant captions arrive, preventing audio that arrives first from being accepted.

## What changed

Make speaker suppression conditional when user transcripts start a new turn. Keep idle output ready for audio while preserving interruption of active assistant output and older, uncaptioned speech that is queued or accepted.

## Testing

Add regression coverage for quiet turns with incremental or final-only user transcripts and for interruption of older pending speech across queue and caption states.

GitOrigin-RevId: 288d2071766a1d0c1b8c7a10c0ab434dc4d7cee1
This commit is contained in:
Benjamin Carlsson
2026-09-11 22:09:26 +00:00
committed by copyberry
parent 7ef70f95d5
commit 1b5e27c7f0
5 changed files with 95 additions and 6 deletions
+2 -2
View File
@@ -1201,7 +1201,7 @@ impl ChatWidget {
self.realtime_conversation.interruption_acknowledged_until =
Some(Instant::now() + INTERRUPTION_ACKNOWLEDGMENT);
}
self.suppress_realtime_speaker();
self.suppress_active_realtime_speaker();
}
if active
&& role == "assistant"
@@ -1454,7 +1454,7 @@ impl ChatWidget {
.input_generation
.wrapping_add(/*rhs*/ 1);
self.realtime_conversation.latest_input_was_voice = true;
self.suppress_realtime_speaker();
self.suppress_active_realtime_speaker();
}
}
}
@@ -76,6 +76,39 @@ impl ChatWidget {
}
}
pub(super) fn suppress_active_realtime_speaker(&mut self) {
// Quiet turns must not wait for captions before accepting their first
// audio packets. Only interrupt output that may belong to an older turn.
if self
.realtime_conversation
.assistant_transcript_generation
.is_some()
|| self
.realtime_conversation
.pending_speech
.iter()
.any(|pending| {
pending.input_generation != self.realtime_conversation.input_generation
&& !pending.captioned
&& matches!(
pending.state,
PendingSpeechState::Queued(_) | PendingSpeechState::Accepted
)
})
|| self.realtime_conversation.speaker_level > 0
|| self
.realtime_conversation
.speaker_active_until
.is_some_and(|deadline| deadline > Instant::now())
|| self
.realtime_conversation
.speaker_suppression_generation
.is_some()
{
self.suppress_realtime_speaker();
}
}
pub(super) fn suppress_realtime_speaker(&mut self) {
self.realtime_conversation.speaker_suppression_generation =
Some(self.realtime_conversation.input_generation);
@@ -416,7 +416,7 @@ async fn newer_voice_delegation_on_the_same_turn_speaks_only_its_own_answer() {
if text.as_str() == "answer to the newer question"
));
let owner = chat.realtime_conversation.speaker_suppression_generation;
assert_eq!(owner, Some(chat.realtime_conversation.input_generation));
assert_eq!(owner, None);
chat.on_realtime_transcript_delta("assistant".to_string(), "answer".to_string());
let owner = chat.realtime_conversation.speaker_suppression_generation;
assert!(owner.is_none());
@@ -122,6 +122,7 @@ async fn completed_user_transcript_keeps_the_old_assistant_turn_suppressed() {
async fn final_only_assistant_transcript_cannot_release_an_interruption() {
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
activate_voice(&mut chat);
chat.realtime_conversation.speaker_level = 1;
chat.on_realtime_transcript_delta("user".into(), "Stop".into());
let generation = chat.realtime_conversation.input_generation;
chat.on_realtime_transcript_done("user".into(), "Stop".into());
@@ -153,7 +154,7 @@ async fn transcript_deltas_track_the_current_speaker() {
chat.on_realtime_transcript_done("assistant".to_string(), "Old".to_string());
assert_eq!(
chat.realtime_conversation.speaker_suppression_generation,
Some(chat.realtime_conversation.input_generation)
None
);
chat.on_realtime_transcript_done("user".to_string(), "Hello there".to_string());
@@ -163,9 +164,9 @@ async fn transcript_deltas_track_the_current_speaker() {
chat.on_realtime_transcript_delta("user".to_string(), "Again".to_string());
chat.on_realtime_transcript_done("user".to_string(), "Again".to_string());
assert!(owner(&chat).is_some());
assert!(owner(&chat).is_none());
chat.on_realtime_transcript_delta("assistant".to_string(), "".to_string());
assert!(owner(&chat).is_some());
assert!(owner(&chat).is_none());
chat.on_realtime_transcript_delta("assistant".to_string(), "Hi".to_string());
assert!(owner(&chat).is_none());
@@ -772,3 +773,57 @@ async fn spoken_user_transcript_preserves_red_chevron_and_canonical_history() {
.any(|line| line.to_string() == "› hello world")
);
}
#[tokio::test]
async fn quiet_voice_turns_accept_audio_before_assistant_captions() {
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
activate_voice(&mut chat);
for user_delta in [Some("First question"), None, Some("Third question")] {
if let Some(delta) = user_delta {
chat.on_realtime_transcript_delta("user".into(), delta.into());
assert_eq!(
chat.realtime_conversation.speaker_suppression_generation,
None
);
}
chat.on_realtime_transcript_done("user".into(), "Question".into());
// Audio can arrive before captions: keep the helper accepting it now.
assert_eq!(
chat.realtime_conversation.speaker_suppression_generation,
None
);
chat.on_realtime_transcript_delta("assistant".into(), "Answer".into());
chat.on_realtime_transcript_done("assistant".into(), "Answer".into());
}
}
#[tokio::test]
async fn new_voice_turn_interrupts_uncaptioned_queued_speech() {
use super::super::PendingRealtimeSpeech;
use super::super::PendingSpeechState;
for (state, captioned, expected) in [
(PendingSpeechState::Queued(1), false, Some(1)),
(PendingSpeechState::Accepted, false, Some(1)),
(PendingSpeechState::AwaitingTurn, false, None),
(PendingSpeechState::Accepted, true, None),
] {
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
let thread_id = activate_voice(&mut chat);
chat.realtime_conversation
.pending_speech
.push_back(PendingRealtimeSpeech {
state,
captioned,
input_generation: 0,
thread_id,
turn_id: "old-turn".into(),
item: agent_item("answer", "Old answer", Some(MessagePhase::FinalAnswer)),
});
chat.on_realtime_transcript_delta("user".into(), "New question".into());
assert_eq!(
chat.realtime_conversation.speaker_suppression_generation,
expected
);
}
}
@@ -41,6 +41,7 @@ async fn voice_live_transcript_renders_beside_the_streamed_cell() {
let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
chat.local_settings.tui.animations = false;
activate_voice_for_thread(&mut chat, ThreadId::new());
chat.update_realtime_footer();
chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
vec![Line::from("Agent answer arriving").into()],
/*is_first_line*/ true,