From 4e93cf9b4e4e86f49473478c8288426cb6d6b119 Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Tue, 8 Sep 2026 02:37:03 +0000 Subject: [PATCH] Animate live voice transcripts with split-flap tiles (#43656) ## What changed Render live voice transcripts with a split-flap effect, a black background, and brief speaker-colored highlights as characters settle. Preserve settled characters when text is appended and include animation ticks in the active transcript cache key so frames refresh. Respect `animations = false`, preserve Unicode graphemes and raw transcript text, and keep completed transcripts as ordinary history cells. Stop requesting animation frames once the effect settles. ## Testing Add frame snapshots and regression coverage for row styling, speaker colors, appended text, reduced motion, Unicode width, frame scheduling, and unchanged final transcript history. GitOrigin-RevId: 934f577c7279158f9161d6232634087ab32b0ea3 --- codex-rs/tui/src/chatwidget.rs | 4 +- codex-rs/tui/src/chatwidget/realtime.rs | 19 +- .../tui/src/chatwidget/realtime_split_flap.rs | 298 +++++++++++++++++- .../chatwidget/realtime_split_flap_tests.rs | 211 ++++++++++++- .../realtime_tests/caption_replay.rs | 1 + .../chatwidget/realtime_tests/transcripts.rs | 32 ++ 6 files changed, 560 insertions(+), 5 deletions(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index cb8ff353e0..0ccb35e6e6 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1972,7 +1972,9 @@ impl ChatWidget { is_stream_continuation: cell .map(|cell| cell.is_stream_continuation()) .unwrap_or(false), - animation_tick: cell.and_then(|cell| cell.transcript_animation_tick()), + animation_tick: cell + .and_then(|cell| cell.transcript_animation_tick()) + .or_else(|| realtime_cell.and_then(|cell| cell.transcript_animation_tick())), }) } diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs index 48beb9ec90..0c0950ed32 100644 --- a/codex-rs/tui/src/chatwidget/realtime.rs +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -6,11 +6,13 @@ mod recording_controls; use super::ChatWidget; use super::HistoryCell; use super::PARENT_OWNED_INPUT_MESSAGE; +use super::realtime_split_flap::SplitFlapTranscriptCell; use super::realtime_split_flap::VoiceAmplitudeHistory; use crate::app_command::AppCommand; use crate::app_event::AppEvent; use crate::history_cell; use crate::key_hint; +use crate::motion::MotionMode; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::UserInput; use codex_features::Feature; @@ -1244,9 +1246,22 @@ impl ChatWidget { .transcript_role .as_deref() .unwrap_or(""); - self.realtime_conversation.live_transcript_cell = Some( - self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript), + let previous = self + .realtime_conversation + .live_transcript_cell + .as_deref() + .and_then(|cell| cell.as_any().downcast_ref::()); + let cell = + self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript); + let live_cell = SplitFlapTranscriptCell::new( + cell, + role, + &self.realtime_conversation.transcript, + previous, + MotionMode::from_animations_enabled(self.config.animations), + self.frame_requester.clone(), ); + self.realtime_conversation.live_transcript_cell = Some(Box::new(live_cell)); self.bump_active_cell_revision(); self.request_redraw(); } diff --git a/codex-rs/tui/src/chatwidget/realtime_split_flap.rs b/codex-rs/tui/src/chatwidget/realtime_split_flap.rs index cff8ef47a9..2adf54120c 100644 --- a/codex-rs/tui/src/chatwidget/realtime_split_flap.rs +++ b/codex-rs/tui/src/chatwidget/realtime_split_flap.rs @@ -1,9 +1,28 @@ -//! Bounded real amplitude histories for voice recording controls. +//! A bounded, presentation-only split-flap board for live voice transcripts. + +use super::HistoryCell; +use crate::motion::MotionMode; +use crate::terminal_hyperlinks::HyperlinkLine; +use crate::terminal_hyperlinks::visible_lines; +use crate::tui::FrameRequester; +use ratatui::style::Color; +use ratatui::style::Style; +use ratatui::text::Line; +use ratatui::text::Span; +use std::time::Duration; +use std::time::Instant; +use unicode_segmentation::UnicodeSegmentation; #[cfg(test)] #[path = "realtime_split_flap_tests.rs"] mod tests; +const FRAME_INTERVAL: Duration = Duration::from_millis(45); +const ANIMATION_DURATION: Duration = Duration::from_millis(675); +const TILE_SETTLE_DURATION: Duration = Duration::from_millis(180); +const TILE_AFTERGLOW_DURATION: Duration = Duration::from_millis(135); +const FLAP_SAMPLE_WINDOW: usize = 24; + /// Four bounded, calibrated amplitude samples; this is not frequency analysis. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(super) struct VoiceAmplitudeHistory([usize; 4]); @@ -31,3 +50,280 @@ impl VoiceAmplitudeHistory { self.0.iter().copied().max().unwrap_or_default() } } + +#[derive(Debug)] +pub(super) struct SplitFlapTranscriptCell { + inner: Box, + role: String, + target: String, + visible_target: String, + flap_sample: Vec, + tile_arrivals: Vec, + started_at: Instant, + phase_started_at: Instant, + motion_mode: MotionMode, + frame_requester: FrameRequester, +} + +impl SplitFlapTranscriptCell { + pub(super) fn new( + inner: Box, + role: &str, + target: &str, + previous: Option<&Self>, + motion_mode: MotionMode, + frame_requester: FrameRequester, + ) -> Self { + let now = Instant::now(); + let visible_target = visible_glyphs(inner.as_ref()); + let mut flap_sample = visible_target + .bytes() + .rev() + .filter(u8::is_ascii_alphabetic) + .take(FLAP_SAMPLE_WINDOW) + .collect::>(); + if flap_sample.is_empty() { + flap_sample.extend(visible_target.bytes().rev().take(FLAP_SAMPLE_WINDOW)); + } + flap_sample.reverse(); + let previous = previous.filter(|cell| { + cell.role == role + && target.starts_with(&cell.target) + && visible_target.starts_with(&cell.visible_target) + }); + let mut tile_arrivals = previous + .map(|cell| cell.tile_arrivals.clone()) + .unwrap_or_default(); + let existing_tiles = tile_arrivals.len(); + for index in existing_tiles..visible_target.len() { + let offset = u64::try_from(index.saturating_sub(existing_tiles)) + .unwrap_or(u64::MAX) + .saturating_mul(/*rhs*/ 8) + .min(/*other*/ 56); + let start = if previous.is_some() { + now.checked_sub(FRAME_INTERVAL).unwrap_or(now) + } else { + now + }; + tile_arrivals.push(start + Duration::from_millis(offset)); + } + let cell = Self { + inner, + role: role.to_string(), + target: target.to_string(), + visible_target, + flap_sample, + tile_arrivals, + started_at: now, + phase_started_at: previous.map(|cell| cell.phase_started_at).unwrap_or(now), + motion_mode, + frame_requester, + }; + if cell.is_animating(Duration::ZERO) { + cell.frame_requester.schedule_frame_in(FRAME_INTERVAL); + } + cell + } + + fn is_animating(&self, elapsed: Duration) -> bool { + self.motion_mode == MotionMode::Animated + && elapsed < ANIMATION_DURATION + && !self.tile_arrivals.is_empty() + } + + fn animate_lines( + &self, + mut lines: Vec, + width: u16, + elapsed: Duration, + ) -> Vec { + if self.motion_mode == MotionMode::Reduced { + return lines; + } + + let board_style = Style::new().bg(Color::Black).fg(Color::Gray); + let mut glyph_index = 0; + let mut word_is_unsettled = false; + let now = self.started_at + elapsed; + let phase_elapsed = now.saturating_duration_since(self.phase_started_at); + let final_line = lines.iter().rposition(|line| { + line.line + .spans + .iter() + .any(|span| !span.content.trim().is_empty()) + }); + for (line_index, line) in lines.iter_mut().enumerate() { + if line + .line + .spans + .iter() + .all(|span| span.content.trim().is_empty()) + { + continue; + } + + let original_spans = std::mem::take(&mut line.line.spans); + line.line.style = line.line.style.patch(board_style); + for span in original_spans { + for grapheme in span.content.graphemes(/*is_extended*/ true) { + let mut style = span.style.patch(board_style); + let text = if is_flippable(grapheme) { + let position = glyph_index; + glyph_index += 1; + let arrival = self + .tile_arrivals + .get(position) + .copied() + .unwrap_or(self.started_at); + let tile_elapsed = now.saturating_duration_since(arrival); + let (text, flipping) = flap_glyph( + grapheme, + position, + tile_elapsed, + phase_elapsed, + &self.flap_sample, + ); + if flipping { + word_is_unsettled = true; + style = style.fg(Color::DarkGray); + } else if tile_elapsed < TILE_SETTLE_DURATION + TILE_AFTERGLOW_DURATION { + style = style.fg(if self.role == "user" { + Color::Cyan + } else { + Color::Magenta + }); + } + text + } else { + if grapheme == "›" { + style = style.fg(Color::Cyan); + } else if grapheme == "•" { + style = style.fg(Color::Magenta); + } + if grapheme.chars().all(char::is_whitespace) { + word_is_unsettled = false; + } + if word_is_unsettled && is_punctuation(grapheme) { + " ".to_string() + } else { + grapheme.to_string() + } + }; + line.line.spans.push(Span::styled(text, style)); + } + } + + let remaining = usize::from(width).saturating_sub(line.width()); + if Some(line_index) == final_line + && remaining > 1 + && self.is_animating(elapsed) + && self.tile_arrivals.first().is_some_and(|arrival| { + now.saturating_duration_since(*arrival) >= TILE_SETTLE_DURATION + }) + { + let phase = usize::try_from(phase_elapsed.as_millis() / FRAME_INTERVAL.as_millis()) + .unwrap_or(usize::MAX); + let runway = (0..remaining.saturating_sub(/*rhs*/ 1).min(/*other*/ 3)) + .map(|index| { + char::from(self.flap_sample[(phase + index) % self.flap_sample.len()]) + }) + .collect::(); + line.line.spans.push(Span::styled( + format!(" {runway}"), + board_style.fg(Color::DarkGray), + )); + } + let padding = usize::from(width).saturating_sub(line.width()); + if padding > 0 { + line.line + .spans + .push(Span::styled(" ".repeat(padding), board_style)); + } + } + + if self.is_animating(elapsed) { + self.frame_requester.schedule_frame_in(FRAME_INTERVAL); + } + lines + } +} + +impl HistoryCell for SplitFlapTranscriptCell { + fn display_lines(&self, width: u16) -> Vec> { + visible_lines(self.display_hyperlink_lines(width)) + } + + fn raw_lines(&self) -> Vec> { + self.inner.raw_lines() + } + + fn display_hyperlink_lines(&self, width: u16) -> Vec { + self.animate_lines( + self.inner.display_hyperlink_lines(width), + width, + self.started_at.elapsed(), + ) + } + + fn transcript_hyperlink_lines(&self, width: u16) -> Vec { + self.animate_lines( + self.inner.transcript_hyperlink_lines(width), + width, + self.started_at.elapsed(), + ) + } + + fn transcript_animation_tick(&self) -> Option { + let elapsed = self.started_at.elapsed(); + self.is_animating(elapsed).then(|| { + u64::try_from(elapsed.as_millis() / FRAME_INTERVAL.as_millis()).unwrap_or(u64::MAX) + }) + } +} + +fn visible_glyphs(cell: &dyn HistoryCell) -> String { + let mut glyphs = String::new(); + for line in cell.display_hyperlink_lines(u16::MAX) { + for span in line.line.spans { + for grapheme in span.content.graphemes(/*is_extended*/ true) { + if is_flippable(grapheme) { + glyphs.push_str(grapheme); + } + } + } + } + glyphs +} + +fn is_flippable(grapheme: &str) -> bool { + grapheme.len() == 1 && grapheme.as_bytes()[0].is_ascii_alphanumeric() +} + +fn is_punctuation(grapheme: &str) -> bool { + grapheme.len() == 1 && grapheme.as_bytes()[0].is_ascii_punctuation() +} + +fn flap_glyph( + grapheme: &str, + index: usize, + elapsed: Duration, + phase_elapsed: Duration, + sample: &[u8], +) -> (String, bool) { + let milliseconds = elapsed.as_millis(); + if milliseconds < 40 { + return (" ".to_string(), true); + } + if elapsed >= TILE_SETTLE_DURATION { + return (grapheme.to_string(), false); + } + + let frame = usize::try_from(phase_elapsed.as_millis() / FRAME_INTERVAL.as_millis()) + .unwrap_or(usize::MAX); + let target = usize::from(grapheme.as_bytes()[0]); + let offset = target + .wrapping_add(index.wrapping_mul(/*rhs*/ 7)) + .wrapping_add(frame.wrapping_mul(/*rhs*/ 11)); + let tile = char::from(sample[offset % sample.len()]); + (tile.to_string(), true) +} diff --git a/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs b/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs index 1214e12b54..b3c1597ced 100644 --- a/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs +++ b/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs @@ -1,8 +1,217 @@ //! Regressions for bounded state used by realtime voice presentation. //! Retained history must preserve ordering without growing beyond its limits. -use super::VoiceAmplitudeHistory; +use super::*; use pretty_assertions::assert_eq; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; + +#[derive(Debug)] +struct PlainTranscriptCell(String); + +impl HistoryCell for PlainTranscriptCell { + fn display_lines(&self, _width: u16) -> Vec> { + vec![Line::from(vec![Span::raw("› "), Span::raw(self.0.clone())])] + } + + fn raw_lines(&self) -> Vec> { + vec![Line::from(self.0.clone())] + } +} + +fn cell(text: &str, motion_mode: MotionMode) -> SplitFlapTranscriptCell { + SplitFlapTranscriptCell::new( + Box::new(PlainTranscriptCell(text.to_string())), + "user", + text, + /*previous*/ None, + motion_mode, + FrameRequester::test_dummy(), + ) +} + +fn frame(cell: &SplitFlapTranscriptCell, milliseconds: u64) -> String { + cell.animate_lines( + cell.inner.display_hyperlink_lines(/*width*/ 16), + /*width*/ 16, + Duration::from_millis(milliseconds), + ) + .into_iter() + .map(|line| line.line.to_string().trim_end().to_string()) + .collect::>() + .join("\n") +} + +#[test] +fn split_flap_frames_assemble_a_transcript_from_dark_tiles() { + let board = cell("GATE 73", MotionMode::Animated); + let frames = [0, 45, 90, 180, 285, 675] + .into_iter() + .map(|milliseconds| format!("{milliseconds:>3}ms {}", frame(&board, milliseconds))) + .collect::>() + .join("\n"); + + insta::assert_snapshot!(frames, @r" + 0ms › + 45ms › T + 90ms › ATGG AG + 180ms › GGTT ET GAT + 285ms › GATE 73 TEG + 675ms › GATE 73 + "); + + let punctuated = cell("HELLO, WORLD?!", MotionMode::Animated); + assert!(!frame(&punctuated, /*milliseconds*/ 90).contains(',')); + assert!(!frame(&punctuated, /*milliseconds*/ 180).contains(',')); + assert_eq!(frame(&punctuated, /*milliseconds*/ 285), "› HELLO, WORLD?!"); + assert_ne!( + frame(&board, /*milliseconds*/ 285), + frame(&board, /*milliseconds*/ 330) + ); + + let lowercase = cell("please keep going", MotionMode::Animated); + assert!(lowercase.flap_sample.iter().all(u8::is_ascii_lowercase)); + for elapsed in [90, 285] { + assert!( + frame(&lowercase, elapsed) + .chars() + .all(|ch| !ch.is_ascii_uppercase()) + ); + } +} + +#[test] +fn split_flap_paints_the_entire_transcript_row_black() { + let cell = cell("GATE", MotionMode::Animated); + let lines = cell.animate_lines( + cell.inner.display_hyperlink_lines(/*width*/ 12), + /*width*/ 12, + Duration::ZERO, + ); + let area = Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 12, /*height*/ 1, + ); + let mut buffer = Buffer::empty(area); + Paragraph::new(visible_lines(lines)).render(area, &mut buffer); + + assert!(buffer.content.iter().all(|tile| tile.bg == Color::Black)); + assert_eq!(buffer[(0, 0)].fg, Color::Cyan); +} + +#[test] +fn settled_tiles_briefly_glow_in_the_speakers_color() { + for (role, motion_mode, expected) in [ + ("user", MotionMode::Animated, Some(Color::Cyan)), + ("assistant", MotionMode::Animated, Some(Color::Magenta)), + ("user", MotionMode::Reduced, None), + ] { + let cell = SplitFlapTranscriptCell::new( + Box::new(PlainTranscriptCell("A".to_string())), + role, + "A", + /*previous*/ None, + motion_mode, + FrameRequester::test_dummy(), + ); + let tile_color = |elapsed| { + cell.animate_lines( + cell.inner.display_hyperlink_lines(/*width*/ 8), + /*width*/ 8, + elapsed, + )[0] + .line + .spans + .iter() + .find(|span| span.content == "A") + .and_then(|span| span.style.fg) + }; + + assert_eq!(tile_color(TILE_SETTLE_DURATION), expected); + assert_eq!( + tile_color(TILE_SETTLE_DURATION + TILE_AFTERGLOW_DURATION), + (motion_mode == MotionMode::Animated).then_some(Color::Gray) + ); + } +} + +#[test] +fn reduced_motion_keeps_the_original_transcript_and_requests_no_frames() { + let (frame_requester, mut requests) = FrameRequester::test_channel(); + let cell = SplitFlapTranscriptCell::new( + Box::new(PlainTranscriptCell("GATE 73".to_string())), + "user", + "GATE 73", + /*previous*/ None, + MotionMode::Reduced, + frame_requester, + ); + + assert_eq!(cell.display_lines(/*width*/ 16)[0].to_string(), "› GATE 73"); + assert_eq!(cell.transcript_animation_tick(), None); + assert!(requests.try_recv().is_err()); +} + +#[test] +fn appended_transcript_keeps_the_previous_words_settled() { + let mut first = cell("GATE", MotionMode::Animated); + for arrival in &mut first.tile_arrivals { + *arrival -= ANIMATION_DURATION; + } + let appended = SplitFlapTranscriptCell::new( + Box::new(PlainTranscriptCell("GATE 73".to_string())), + "user", + "GATE 73", + Some(&first), + MotionMode::Animated, + FrameRequester::test_dummy(), + ); + + assert!(frame(&appended, /*milliseconds*/ 0).starts_with("› GATE ")); + assert_eq!(appended.phase_started_at, first.phase_started_at); + assert_eq!(appended.tile_arrivals[..4], first.tile_arrivals); +} + +#[test] +fn unicode_graphemes_and_terminal_width_are_preserved() { + let text = "a\u{301} 東京 👩‍💻 B"; + let cell = cell(text, MotionMode::Animated); + let lines = cell.animate_lines( + cell.inner.display_hyperlink_lines(/*width*/ 24), + /*width*/ 24, + Duration::ZERO, + ); + + assert!(lines[0].line.to_string().contains("a\u{301} 東京 👩‍💻")); + assert_eq!(lines[0].width(), 24); + assert_eq!(cell.raw_lines()[0].to_string(), text); +} + +#[test] +fn settled_split_flap_stops_scheduling_animation_frames() { + let (frame_requester, mut requests) = FrameRequester::test_channel(); + let mut cell = SplitFlapTranscriptCell::new( + Box::new(PlainTranscriptCell("GATE".to_string())), + "user", + "GATE", + /*previous*/ None, + MotionMode::Animated, + frame_requester, + ); + assert!(requests.try_recv().is_ok()); + cell.started_at = Instant::now() - ANIMATION_DURATION; + for arrival in &mut cell.tile_arrivals { + *arrival -= ANIMATION_DURATION; + } + + assert_eq!(cell.transcript_animation_tick(), None); + assert_eq!( + cell.display_lines(/*width*/ 12)[0].to_string().trim_end(), + "› GATE" + ); + assert!(requests.try_recv().is_err()); +} #[test] fn voice_meter_shows_four_real_amplitude_samples_in_two_braille_cells() { diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs b/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs index 6aaa9d1130..c1956f016a 100644 --- a/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs +++ b/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs @@ -327,6 +327,7 @@ async fn captioned_voice_answer_does_not_duplicate_on_close() { #[tokio::test] async fn restored_partial_caption_accepts_late_completion_without_duplicate_history() { let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await; + chat.config.animations = false; chat.restore_realtime_transcript_cells(VecDeque::from([ super::super::RealtimeTranscriptRecord { role: "user".into(), diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs index b49d31e4aa..39096aef9f 100644 --- a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs +++ b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs @@ -526,6 +526,7 @@ async fn transcript_completion_waits_for_normal_agent_stream_consolidation() { #[tokio::test] async fn transcript_handoff_moves_deferred_repeats_and_partial_once() { let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await; + chat.config.animations = false; activate_voice(&mut chat); chat.on_agent_message_delta("ordinary stream".to_string()); for _ in 0..super::super::MAX_PENDING_TRANSCRIPT_CELLS { @@ -694,3 +695,34 @@ async fn completed_transcript_preserves_the_other_speakers_caption() { (Some("assistant"), "Still speaking") ); } + +#[tokio::test] +async fn live_voice_split_flap_animates_without_changing_final_history() { + let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await; + chat.config.animations = true; + activate_voice(&mut chat); + + chat.on_realtime_transcript_delta("assistant".to_string(), "gate 73".to_string()); + + let Some(cell) = chat.realtime_conversation.live_transcript_cell.as_ref() else { + panic!("live voice transcripts should have an animation cell"); + }; + assert_eq!(cell.raw_lines()[0].to_string(), "gate 73"); + assert!( + chat.active_cell_transcript_key() + .is_some_and(|key| { key.animation_tick.is_some() }) + ); + + chat.on_realtime_transcript_done("assistant".to_string(), "gate 73".to_string()); + + assert!(chat.active_cell_transcript_key().is_none()); + let Ok(AppEvent::InsertHistoryCell(cell)) = events.try_recv() else { + panic!("a completed voice transcript should commit ordinary history"); + }; + assert!(cell.transcript_animation_tick().is_none()); + assert!( + cell.display_lines(/*width*/ 80) + .iter() + .any(|line| { line.to_string().contains("gate 73") }) + ); +}