Fix TUI layout for halfwidth Japanese sound marks (#35962)

## What changed

- Use Ratatui-compatible terminal widths throughout TUI layout, preserving
  full-width counters for long lines.
- Wrap and truncate on grapheme boundaries so halfwidth kana sound marks and
  emoji sequences stay intact.
- Apply the corrected sizing to text input, selection popups, status and
  history cells, Markdown, hyperlinks, and scrollable cloud diffs.

## Testing

- Add focused unit and snapshot coverage for wrapping, truncation, cursor and
  mask alignment, popup columns, status output, session headers, Markdown
  tables, process output, and scrollable diffs.

GitOrigin-RevId: 294ec0bcff743c6bb16d55ec26308524ce741879
This commit is contained in:
Charlie Marsh
2026-07-29 14:01:31 +00:00
committed by copyberry
parent cef3910ea4
commit 3725f02cf3
38 changed files with 1335 additions and 245 deletions
+2
View File
@@ -2482,6 +2482,7 @@ dependencies = [
"codex-utils-cli",
"crossterm",
"http 1.4.0",
"insta",
"owo-colors",
"pretty_assertions",
"ratatui",
@@ -2492,6 +2493,7 @@ dependencies = [
"tokio-stream",
"tracing",
"tracing-subscriber",
"unicode-segmentation",
"unicode-width 0.2.1",
]
+2
View File
@@ -37,7 +37,9 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
tokio-stream = { workspace = true }
tracing = { workspace = true, features = ["log"] }
tracing-subscriber = { workspace = true, features = ["env-filter"] }
unicode-segmentation = { workspace = true }
unicode-width = { workspace = true }
[dev-dependencies]
insta = { workspace = true }
pretty_assertions = { workspace = true }
+47 -15
View File
@@ -1,4 +1,4 @@
use unicode_width::UnicodeWidthChar;
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
/// Scroll position and geometry for a vertical scroll view.
@@ -131,38 +131,57 @@ impl ScrollableDiff {
let mut line = String::new();
let mut line_cols = 0usize;
let mut last_soft_idx: Option<usize> = None; // last whitespace or punctuation break
for (_i, ch) in raw.char_indices() {
if ch == '\n' {
for grapheme in raw
.graphemes(/*is_extended*/ true)
.flat_map(|grapheme| grapheme.split_inclusive(char::is_whitespace))
{
if grapheme == "\n" {
out.push(std::mem::take(&mut line));
out_idx.push(raw_idx);
line_cols = 0;
last_soft_idx = None;
continue;
}
let w = UnicodeWidthChar::width(ch).unwrap_or(0);
if line_cols.saturating_add(w) > max_cols {
let grapheme_width = display_width(grapheme);
if line_cols.saturating_add(grapheme_width) > max_cols {
if let Some(split) = last_soft_idx {
let (prefix, rest) = line.split_at(split);
out.push(prefix.trim_end().to_string());
out_idx.push(raw_idx);
let prefix = prefix.trim_end();
if !prefix.is_empty() {
out.push(prefix.to_string());
out_idx.push(raw_idx);
}
line = rest.trim_start().to_string();
last_soft_idx = None;
// retry add current ch now that line may be shorter
// Retry adding the current grapheme now that the line may be shorter.
} else if !line.is_empty() {
out.push(std::mem::take(&mut line));
out_idx.push(raw_idx);
}
}
if ch.is_whitespace()
|| matches!(
ch,
',' | ';' | '.' | ':' | ')' | ']' | '}' | '|' | '/' | '?' | '!' | '-' | '_'
)
if grapheme.chars().all(char::is_whitespace)
|| grapheme.chars().any(|ch| {
matches!(
ch,
',' | ';'
| '.'
| ':'
| ')'
| ']'
| '}'
| '|'
| '/'
| '?'
| '!'
| '-'
| '_'
)
})
{
last_soft_idx = Some(line.len());
}
line.push(ch);
line_cols = UnicodeWidthStr::width(line.as_str());
line.push_str(grapheme);
line_cols = display_width(&line);
}
if !line.is_empty() {
out.push(line);
@@ -174,3 +193,16 @@ impl ScrollableDiff {
self.state.content_h = self.wrapped.len() as u16;
}
}
/// Counts terminal cells, including the halfwidth sound marks omitted by `unicode-width`.
fn display_width(text: &str) -> usize {
UnicodeWidthStr::width(text)
+ text
.chars()
.filter(|ch| matches!(ch, '\u{FF9E}' | '\u{FF9F}'))
.count()
}
#[cfg(test)]
#[path = "scrollable_diff_tests.rs"]
mod tests;
@@ -0,0 +1,48 @@
use super::ScrollableDiff;
use pretty_assertions::assert_eq;
#[test]
fn wraps_halfwidth_sound_marks_using_ratatui_width() {
let mut diff = ScrollableDiff::new();
diff.set_content(vec![
"aガbパc".to_string(),
"a ゙b".to_string(),
"a ゚b".to_string(),
]);
diff.set_width(/*width*/ 2);
insta::assert_snapshot!(
diff.wrapped_lines().join("\n"),
@r"
a
ガ
b
パ
c
a
゙b
a
゚b
"
);
assert_eq!(diff.wrapped_src_indices(), [0, 0, 0, 0, 0, 1, 1, 2, 2]);
}
#[test]
fn does_not_emit_empty_rows_after_full_width_sound_mark_graphemes() {
for grapheme in ["ガ", "パ"] {
let mut diff = ScrollableDiff::new();
diff.set_content(vec![format!("a {grapheme} tail")]);
diff.set_width(/*width*/ 2);
let expected = ["a", grapheme, "ta", "il"];
let actual = diff
.wrapped_lines()
.iter()
.map(String::as_str)
.collect::<Vec<_>>();
assert_eq!(actual, expected);
assert_eq!(diff.wrapped_src_indices(), [0, 0, 0, 0]);
assert_eq!(diff.state.content_h, 4);
}
}
@@ -2569,6 +2569,41 @@ mod tests {
);
}
#[test]
fn snapshot_narrow_width_counts_halfwidth_sound_marks() {
let (tx_raw, _rx) = unbounded_channel::<AppEvent>();
let tx = AppEventSender::new(tx_raw);
let items = vec![
SelectionItem {
name: "abガc".to_string(),
description: Some("dakuten description".to_string()),
dismiss_on_select: true,
..Default::default()
},
SelectionItem {
name: "aパc".to_string(),
description: Some("handakuten description".to_string()),
dismiss_on_select: true,
..Default::default()
},
];
let view = new_view(
SelectionViewParams {
title: Some("Halfwidth sound marks".to_string()),
items,
..Default::default()
},
tx,
);
let rendered = format!(
"width 20:\n{}\n\nwidth 24:\n{}",
render_lines_with_width(&view, /*width*/ 20),
render_lines_with_width(&view, /*width*/ 24)
);
assert_snapshot!("list_selection_halfwidth_sound_marks_narrow", rendered);
}
#[test]
fn snapshot_auto_visible_col_width_mode_scroll_behavior() {
assert_snapshot!(
@@ -7,8 +7,7 @@ use ratatui::widgets::Paragraph;
use ratatui::widgets::Widget;
use std::borrow::Cow;
use std::time::Instant;
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use unicode_segmentation::UnicodeSegmentation;
use crate::bottom_pane::popup_consts::standard_popup_hint_line;
use crate::bottom_pane::scroll_state::ScrollState;
@@ -18,7 +17,9 @@ use crate::bottom_pane::selection_popup_common::menu_surface_padding_height;
use crate::bottom_pane::selection_popup_common::render_menu_surface;
use crate::bottom_pane::selection_popup_common::render_rows;
use crate::bottom_pane::selection_popup_common::wrap_styled_line;
use crate::line_truncation::line_width;
use crate::render::renderable::Renderable;
use crate::width::display_width;
use super::DESIRED_SPACERS_BETWEEN_SECTIONS;
use super::RequestUserInputOverlay;
@@ -421,12 +422,6 @@ impl RequestUserInputOverlay {
}
}
fn line_width(line: &Line<'_>) -> usize {
line.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.sum()
}
/// Render rows into `area`, bottom-aligning the visible rows when fewer than
/// `area.height` lines are produced.
///
@@ -489,7 +484,7 @@ fn truncate_line_word_boundary_with_ellipsis(
}
let ellipsis = "…";
let ellipsis_width = UnicodeWidthStr::width(ellipsis);
let ellipsis_width = display_width(ellipsis);
if ellipsis_width >= max_width {
return Line::from(ellipsis);
}
@@ -509,19 +504,19 @@ fn truncate_line_word_boundary_with_ellipsis(
'outer: for (span_idx, span) in line.spans.iter().enumerate() {
let text = span.content.as_ref();
for (byte_idx, ch) in text.char_indices() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
if used.saturating_add(ch_width) > limit {
for (byte_idx, grapheme) in text.grapheme_indices(/*is_extended*/ true) {
let grapheme_width = display_width(grapheme);
if used.saturating_add(grapheme_width) > limit {
overflowed = true;
break 'outer;
}
used = used.saturating_add(ch_width);
used = used.saturating_add(grapheme_width);
let bp = BreakPoint {
span_idx,
byte_end: byte_idx + ch.len_utf8(),
byte_end: byte_idx + grapheme.len(),
};
last_fit = Some(bp);
if ch.is_whitespace() {
if grapheme.chars().all(char::is_whitespace) {
last_word_break = Some(bp);
}
}
@@ -576,3 +571,7 @@ fn truncate_line_word_boundary_with_ellipsis(
Line::from(spans_out).style(line_style)
}
#[cfg(test)]
#[path = "render_tests.rs"]
mod tests;
@@ -0,0 +1,37 @@
use super::truncate_line_word_boundary_with_ellipsis;
use pretty_assertions::assert_eq;
use ratatui::buffer::Buffer;
use ratatui::layout::Rect;
use ratatui::style::Stylize;
use ratatui::text::Line;
use ratatui::text::Span;
use ratatui::widgets::Paragraph;
use ratatui::widgets::Widget;
#[test]
fn halfwidth_sound_marks_are_truncated_at_a_grapheme_boundary() {
let line = Line::from("abガ tail");
assert_eq!(
truncate_line_word_boundary_with_ellipsis(line, /*max_width*/ 4),
Line::from(vec![Span::raw("ab"), Span::raw("…")])
);
}
#[test]
fn halfwidth_sound_marks_are_truncated_and_rendered_at_a_grapheme_boundary() {
let lines = [
Line::from(vec!["ab".bold(), "ガ".cyan(), " tail".dim()]),
Line::from(vec!["xy".bold(), "パ".magenta(), " tail".dim()]),
]
.map(|line| truncate_line_word_boundary_with_ellipsis(line, /*max_width*/ 5));
let area = Rect::new(0, 0, 5, 2);
let mut buf = Buffer::empty(area);
Paragraph::new(lines.to_vec()).render(area, &mut buf);
insta::assert_snapshot!(
"request_user_input_halfwidth_sound_marks_truncation",
format!("{buf:?}\n{lines:#?}")
);
}
@@ -0,0 +1,33 @@
---
source: tui/src/bottom_pane/request_user_input/render_tests.rs
expression: "format!(\"{buf:?}\\n{lines:#?}\")"
---
Buffer {
area: Rect { x: 0, y: 0, width: 5, height: 2 },
content: [
"abガ…", // hidden by multi-width symbols: [(3, " ")]
"xyパ…", // hidden by multi-width symbols: [(3, " ")]
],
styles: [
x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 2, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE,
x: 3, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 4, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 2, y: 1, fg: Magenta, bg: Reset, underline: Reset, modifier: NONE,
x: 3, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 4, y: 1, fg: Magenta, bg: Reset, underline: Reset, modifier: NONE,
]
}
[
Line::from_iter([
Span::from("ab").bold(),
Span::from("カ\u{ff9e}").cyan(),
Span::from("…").cyan(),
]),
Line::from_iter([
Span::from("xy").bold(),
Span::from("ハ\u{ff9f}").magenta(),
Span::from("…").magenta(),
]),
]
@@ -7,14 +7,15 @@ use ratatui::text::Line;
use ratatui::text::Span;
use ratatui::widgets::Block;
use ratatui::widgets::Widget;
use unicode_width::UnicodeWidthStr;
use crate::key_hint::KeyBinding;
use crate::line_truncation::line_width;
use crate::line_truncation::truncate_line_with_ellipsis_if_overflow;
use crate::render::Insets;
use crate::render::RectExt as _;
use crate::style::accent_style;
use crate::style::user_message_style;
use crate::width::display_width;
use super::scroll_state::ScrollState;
use super::selection_row_layout::SelectionDescriptionLayout;
@@ -170,7 +171,7 @@ fn compute_desc_col(
if row.disabled_reason.is_some() {
spans.push(" (disabled)".dim());
}
Line::from(spans).width()
line_width(&Line::from(spans))
})
.max()
.unwrap_or(0),
@@ -182,7 +183,7 @@ fn compute_desc_col(
if row.disabled_reason.is_some() {
spans.push(" (disabled)".dim());
}
Line::from(spans).width()
line_width(&Line::from(spans))
})
.max()
.unwrap_or(0),
@@ -225,6 +226,9 @@ fn should_wrap_name_in_column(row: &GenericDisplayRow) -> bool {
}
fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) -> Vec<Line<'static>> {
use crate::wrapping::RtOptions;
use crate::wrapping::word_wrap_lines;
let Some(description) = row.description.as_deref() else {
return Vec::new();
};
@@ -245,14 +249,13 @@ fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) ->
.unwrap_or(0)
.min(left_width.saturating_sub(1));
let name_subsequent_indent = " ".repeat(name_wrap_indent);
let name_options = textwrap::Options::new(left_width)
.initial_indent("")
.subsequent_indent(name_subsequent_indent.as_str());
let name_lines = textwrap::wrap(row.name.as_str(), name_options);
let name_options = RtOptions::new(left_width)
.initial_indent(Line::from(""))
.subsequent_indent(Line::from(" ".repeat(name_wrap_indent)));
let name_lines = word_wrap_lines(row.name.lines(), name_options);
let desc_options = textwrap::Options::new(right_width).initial_indent("");
let desc_lines = textwrap::wrap(description, desc_options);
let desc_options = RtOptions::new(right_width).initial_indent(Line::from(""));
let desc_lines = word_wrap_lines(description.lines(), desc_options);
let rows = name_lines.len().max(desc_lines.len()).max(1);
let mut out = Vec::with_capacity(rows);
@@ -265,7 +268,7 @@ fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) ->
if let Some(desc) = desc_lines.get(idx) {
let left_used = spans
.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.map(|span| display_width(span.content.as_ref()))
.sum::<usize>();
let gap = if left_used == 0 {
desc_col
@@ -797,6 +800,123 @@ mod tests {
assert_eq!(two_col.len(), 0);
}
#[test]
fn popup_name_truncation_counts_halfwidth_sound_marks() {
for (name, desc_col, match_index, expected) in
[("abガc", 6, None, "abガ…"), ("aガc", 4, Some(1), "a…")]
{
let row = GenericDisplayRow {
name: name.to_string(),
description: Some("description".to_string()),
match_indices: match_index.map(|index| vec![index]),
..Default::default()
};
let text =
build_full_line(&row, desc_col, SelectionDescriptionLayout::Columns).to_string();
assert!(text.starts_with(expected), "unexpected row: {text:?}");
}
}
#[test]
fn fuzzy_matched_emoji_graphemes_keep_description_alignment() {
let rows = [
GenericDisplayRow {
name: "👍🏻".to_string(),
match_indices: Some(vec![1]),
description: Some("description".to_string()),
..Default::default()
},
GenericDisplayRow {
name: "👨‍👩‍👧‍👦".to_string(),
match_indices: Some(vec![2]),
description: Some("description".to_string()),
..Default::default()
},
];
let area = Rect::new(0, 0, /*width*/ 20, /*height*/ 2);
let mut buf = Buffer::empty(area);
for (row_index, row) in rows.iter().enumerate() {
let line = build_full_line(
row,
/*desc_col*/ 4,
SelectionDescriptionLayout::Columns,
);
let name = line.spans.first().expect("fuzzy-matched name span");
assert_eq!(name.content.as_ref(), row.name);
assert!(name.style.add_modifier.contains(Modifier::BOLD));
let row_area = Rect::new(area.x, area.y + row_index as u16, area.width, 1);
ratatui::widgets::Widget::render(
ratatui::widgets::Paragraph::new(line),
row_area,
&mut buf,
);
assert_eq!(buf[(4, row_index as u16)].symbol(), "d");
}
insta::assert_snapshot!("popup_fuzzy_matched_emoji_graphemes", format!("{buf:?}"));
}
#[test]
fn wrapped_two_column_rows_count_halfwidth_sound_marks() {
let rows = vec![GenericDisplayRow {
name: "abガc".to_string(),
description: Some("abパc".to_string()),
wrap_indent: Some(0),
..Default::default()
}];
let area = Rect::new(0, 0, 9, 2);
let mut buf = Buffer::empty(area);
let rendered_lines = render_rows(
area,
&mut buf,
&rows,
&ScrollState::default(),
/*max_results*/ 1,
"no rows",
);
let rendered = (0..area.height)
.map(|y| {
(0..area.width)
.map(|x| buf[(x, y)].symbol())
.collect::<String>()
.trim_end()
.to_string()
})
.collect::<Vec<_>>()
.join("\n");
assert_eq!(rendered_lines, 2);
insta::assert_snapshot!(rendered, @r"
abガ ab
c パ c
");
}
#[test]
fn wrapped_two_column_rows_preserve_hard_line_breaks() {
let row = GenericDisplayRow {
name: "first\nsecond".to_string(),
description: Some("alpha\nbeta".to_string()),
wrap_indent: Some(0),
..Default::default()
};
let rendered = wrap_two_column_row(&row, /*desc_col*/ 8, /*width*/ 24)
.into_iter()
.map(|line| line.to_string())
.collect::<Vec<_>>()
.join("\n");
insta::assert_snapshot!(rendered, @r"
first alpha
second beta
");
}
#[test]
fn selected_rows_use_the_shared_accent_style() {
let rows = vec![GenericDisplayRow {
@@ -3,9 +3,11 @@ use std::borrow::Cow;
use ratatui::style::Stylize;
use ratatui::text::Line;
use ratatui::text::Span;
use unicode_width::UnicodeWidthChar;
use unicode_segmentation::UnicodeSegmentation;
use super::selection_popup_common::GenericDisplayRow;
use crate::line_truncation::line_width;
use crate::width::display_width;
use crate::wrapping::RtOptions;
use crate::wrapping::word_wrap_line;
@@ -82,35 +84,28 @@ fn build_name_spans(row: &GenericDisplayRow, name_limit: usize) -> Vec<Span<'sta
let mut used_width = 0usize;
let mut truncated = false;
if let Some(idxs) = row.match_indices.as_ref() {
let mut idx_iter = idxs.iter().peekable();
for (char_idx, ch) in row.name.chars().enumerate() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
let next_width = used_width.saturating_add(ch_width);
if next_width > name_limit {
truncated = true;
break;
}
used_width = next_width;
let mut match_indices = row.match_indices.iter().flatten().peekable();
let mut char_idx = 0usize;
for grapheme in row.name.graphemes(/*is_extended*/ true) {
let next_width = used_width.saturating_add(display_width(grapheme));
if next_width > name_limit {
truncated = true;
break;
}
used_width = next_width;
if idx_iter.peek().is_some_and(|next| **next == char_idx) {
idx_iter.next();
name_spans.push(ch.to_string().bold());
} else {
name_spans.push(ch.to_string().into());
}
}
} else {
for ch in row.name.chars() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
let next_width = used_width.saturating_add(ch_width);
if next_width > name_limit {
truncated = true;
break;
}
used_width = next_width;
name_spans.push(ch.to_string().into());
let mut matched = false;
for _ in grapheme.chars() {
matched |= match_indices.next_if(|next| **next == char_idx).is_some();
char_idx += 1;
}
let grapheme = grapheme.to_string();
name_spans.push(if matched {
grapheme.bold()
} else {
grapheme.into()
});
}
if truncated {
@@ -145,13 +140,13 @@ pub(super) fn build_full_line(
description_layout: SelectionDescriptionLayout,
) -> Line<'static> {
let description = combined_description(row, description_layout);
let name_prefix_width = Line::from(row.name_prefix_spans.clone()).width();
let name_prefix_width = line_width(&Line::from(row.name_prefix_spans.clone()));
let name_limit = description
.as_ref()
.map(|_| desc_col.saturating_sub(2).saturating_sub(name_prefix_width))
.unwrap_or(usize::MAX);
let name_spans = build_name_spans(row, name_limit);
let name_width = name_prefix_width + Line::from(name_spans.clone()).width();
let name_width = name_prefix_width + line_width(&Line::from(name_spans.clone()));
let mut spans = row.name_prefix_spans.clone();
spans.extend(name_spans);
@@ -170,8 +165,7 @@ pub(super) fn build_full_line(
/// Render a row as a full-width label followed by an indented description.
pub(super) fn wrap_stacked_row(row: &GenericDisplayRow, width: u16) -> Vec<Line<'static>> {
let width = width.max(1);
let prefix_width = Line::from(row.name_prefix_spans.clone())
.width()
let prefix_width = line_width(&Line::from(row.name_prefix_spans.clone()))
.min(width.saturating_sub(1) as usize);
let indent = " ".repeat(prefix_width);
@@ -0,0 +1,30 @@
---
source: tui/src/bottom_pane/list_selection_view.rs
expression: rendered
---
width 20:
Halfwidth sound
› 1. abガc
dakute
n
descri
ption
2. aパc
handak
uten
descri
ption
width 24:
Halfwidth sound mark
› 1. abガc dakuten
descriptio
n
2. aパc handakuten
descriptio
n
@@ -0,0 +1,21 @@
---
source: tui/src/bottom_pane/selection_popup_common.rs
expression: "format!(\"{buf:?}\")"
---
Buffer {
area: Rect { x: 0, y: 0, width: 20, height: 2 },
content: [
"👍🏻 description ", // hidden by multi-width symbols: [(1, " ")]
"👨‍👩‍👧‍👦 description ", // hidden by multi-width symbols: [(1, " ")]
],
styles: [
x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 1, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 4, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 15, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 1, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 4, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 15, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
]
}
@@ -0,0 +1,20 @@
---
source: tui/src/bottom_pane/textarea.rs
expression: "format!(\"dakuten\\n{dakuten}\\n\\nhandakuten\\n{handakuten}\\n\\nstandalone\\n{standalone}\")"
---
dakuten
cursor: Some((1, 1))
"12ガ" Hidden by multi-width symbols: [(3, " ")]
"x "
handakuten
cursor: Some((3, 1))
"ab "
"パc" Hidden by multi-width symbols: [(1, " ")]
standalone
cursor: Some((2, 1))
"a "
"゙b"
@@ -0,0 +1,6 @@
---
source: tui/src/bottom_pane/textarea.rs
expression: "format!(\"cursor: {:?}\\n{}\", t.cursor_pos(area), terminal.backend())"
---
cursor: Some((4, 0))
"****"
+98 -14
View File
@@ -17,6 +17,7 @@ use crate::keymap::RuntimeKeymap;
use crate::keymap::VimNormalKeymap;
use crate::keymap::VimOperatorKeymap;
use crate::keymap::VimTextObjectKeymap;
use crate::width::display_width;
use codex_protocol::user_input::ByteRange;
use codex_protocol::user_input::TextElement as UserTextElement;
use crossterm::event::KeyCode;
@@ -35,7 +36,6 @@ use std::cell::RefCell;
use std::ops::Range;
use textwrap::Options;
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
mod vim;
use self::vim::VimMode;
@@ -430,7 +430,7 @@ impl TextArea {
let effective_scroll = self.effective_scroll(area.height, &lines, state.scroll);
let i = Self::wrapped_line_index_by_start(&lines, self.cursor_pos)?;
let ls = &lines[i];
let col = self.text[ls.start..self.cursor_pos].width() as u16;
let col = display_width(&self.text[ls.start..self.cursor_pos]) as u16;
let screen_row = i
.saturating_sub(effective_scroll as usize)
.try_into()
@@ -444,7 +444,7 @@ impl TextArea {
fn current_display_col(&self) -> usize {
let bol = self.beginning_of_current_line();
self.text[bol..self.cursor_pos].width()
display_width(&self.text[bol..self.cursor_pos])
}
fn wrapped_line_index_by_start(lines: &[Range<usize>], pos: usize) -> Option<usize> {
@@ -462,7 +462,7 @@ impl TextArea {
) {
let mut width_so_far = 0usize;
for (i, g) in self.text[line_start..line_end].grapheme_indices(true) {
width_so_far += g.width();
width_so_far += display_width(g);
if width_so_far > target_col {
self.cursor_pos = line_start + i;
// Avoid landing inside an element; round to nearest boundary
@@ -1212,9 +1212,9 @@ impl TextArea {
let lines = &cache.lines;
if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) {
let cur_range = &lines[idx];
let target_col = self
.preferred_col
.unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width());
let target_col = self.preferred_col.unwrap_or_else(|| {
display_width(&self.text[cur_range.start..self.cursor_pos])
});
if idx > 0 {
let prev = &lines[idx - 1];
let line_start = prev.start;
@@ -1275,9 +1275,9 @@ impl TextArea {
let lines = &cache.lines;
if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) {
let cur_range = &lines[idx];
let target_col = self
.preferred_col
.unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width());
let target_col = self.preferred_col.unwrap_or_else(|| {
display_width(&self.text[cur_range.start..self.cursor_pos])
});
if idx + 1 < lines.len() {
let next = &lines[idx + 1];
let line_start = next.start;
@@ -1994,7 +1994,7 @@ impl TextArea {
continue;
}
let styled = &self.text[overlap_start..overlap_end];
let x_off = self.text[line_range.start..overlap_start].width() as u16;
let x_off = display_width(&self.text[line_range.start..overlap_start]) as u16;
let style = base_style.fg(Color::Cyan);
buf.set_string(area.x + x_off, y, text_for_display(styled), style);
}
@@ -2008,7 +2008,7 @@ impl TextArea {
continue;
}
let highlighted = &self.text[overlap_start..overlap_end];
let x_off = self.text[line_range.start..overlap_start].width() as u16;
let x_off = display_width(&self.text[line_range.start..overlap_start]) as u16;
buf.set_string(area.x + x_off, y, text_for_display(highlighted), *style);
}
}
@@ -2027,8 +2027,8 @@ impl TextArea {
let y = area.y + row as u16;
let line_range = r.start..r.end - 1;
let masked = self.text[line_range.clone()]
.chars()
.map(|_| mask_char)
.graphemes(/*is_extended*/ true)
.flat_map(|grapheme| std::iter::repeat_n(mask_char, display_width(grapheme)))
.collect::<String>();
buf.set_string(area.x, y, &masked, Style::default());
}
@@ -3518,6 +3518,90 @@ mod tests {
assert_eq!(buf[(0, 1)].symbol(), "5");
}
#[test]
fn halfwidth_sound_marks_wrap_and_align_with_the_cursor() {
use ratatui::Terminal;
use ratatui::backend::TestBackend;
let mut snapshots = Vec::new();
for (label, text, width, cursor, cells) in [
("dakuten", "12ガx", 4, (1, 1), [(2, 0, "ガ"), (0, 1, "x")]),
("handakuten", "abパc", 3, (3, 1), [(0, 1, "パ"), (2, 1, "c")]),
("standalone", "a ゙b", 2, (2, 1), [(0, 1, "゙"), (1, 1, "b")]),
] {
let mut t = ta_with(text);
t.set_cursor(text.len());
let area = Rect::new(0, 0, width, /*height*/ 2);
let mut terminal = Terminal::new(TestBackend::new(area.width, area.height)).unwrap();
terminal
.draw(|frame| {
ratatui::widgets::WidgetRef::render_ref(
&(&t),
frame.area(),
frame.buffer_mut(),
);
})
.unwrap();
assert_eq!(t.desired_height(area.width), 2);
assert_eq!(t.cursor_pos(area), Some(cursor));
for (x, y, expected) in cells {
assert_eq!(terminal.backend().buffer()[(x, y)].symbol(), expected);
}
snapshots.push(format!(
"{label}\ncursor: {:?}\n{}",
t.cursor_pos(area),
terminal.backend()
));
}
insta::assert_snapshot!(
"textarea_halfwidth_sound_marks_wrap_and_align_with_cursor",
snapshots.join("\n\n")
);
}
#[test]
fn masked_graphemes_align_with_the_cursor() {
use ratatui::Terminal;
use ratatui::backend::TestBackend;
let text = "界゙a";
let mut t = ta_with(text);
t.set_cursor(text.len());
let area = Rect::new(0, 0, /*width*/ 4, /*height*/ 1);
let mut terminal = Terminal::new(TestBackend::new(area.width, area.height)).unwrap();
let mut state = TextAreaState::default();
terminal
.draw(|frame| {
t.render_ref_masked(frame.area(), frame.buffer_mut(), &mut state, '*');
})
.unwrap();
assert_eq!(t.cursor_pos(area), Some((4, 0)));
insta::assert_snapshot!(
"textarea_masked_graphemes_align_with_cursor",
format!("cursor: {:?}\n{}", t.cursor_pos(area), terminal.backend())
);
}
#[test]
fn overwide_halfwidth_sound_marks_do_not_add_a_phantom_cursor_row() {
let area = Rect::new(0, 0, /*width*/ 2, /*height*/ 2);
for grapheme in ["ガ゙", "界゙"] {
let text = format!("{grapheme}ab");
let mut t = ta_with(&text);
t.set_cursor(text.len());
assert_eq!(t.desired_height(area.width), 2);
assert_eq!(t.cursor_pos(area), Some((2, 1)));
t.set_cursor(grapheme.len());
assert_eq!(t.cursor_pos(area), Some((0, 1)));
}
}
#[test]
fn cursor_pos_with_state_basic_and_scroll_behaviors() {
// Case 1: No wrapping needed, height fits — scroll ignored, y maps directly.
+4 -3
View File
@@ -1,6 +1,7 @@
//! Background terminal interaction and process-summary history cells.
use super::*;
use crate::width::display_width;
#[derive(Debug)]
pub(crate) struct UnifiedExecInteractionCell {
@@ -137,9 +138,9 @@ impl HistoryCell for UnifiedExecProcessesCell {
}
let prefix = " • ";
let prefix_width = UnicodeWidthStr::width(prefix);
let prefix_width = display_width(prefix);
let truncation_suffix = " [...]";
let truncation_suffix_width = UnicodeWidthStr::width(truncation_suffix);
let truncation_suffix_width = display_width(truncation_suffix);
let mut shown = 0usize;
for process in &self.processes {
if shown >= max_processes {
@@ -189,7 +190,7 @@ impl HistoryCell for UnifiedExecProcessesCell {
} else {
chunk_prefix_next
};
let chunk_prefix_width = UnicodeWidthStr::width(chunk_prefix);
let chunk_prefix_width = display_width(chunk_prefix);
if wrap_width <= chunk_prefix_width {
out.push(Line::from(chunk_prefix.dim()));
continue;
-1
View File
@@ -103,7 +103,6 @@ use std::time::Duration;
use std::time::Instant;
use tracing::error;
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
use url::Url;
const RAW_DIFF_SUMMARY_WIDTH: usize = 10_000;
+7 -16
View File
@@ -1,7 +1,9 @@
//! Session headers, onboarding guidance, and transcript cards.
use super::*;
use crate::line_truncation::line_width;
use crate::line_truncation::truncate_line_with_ellipsis_if_overflow;
use crate::width::display_width;
pub(crate) const SESSION_HEADER_MAX_INNER_WIDTH: usize = 56; // Just an eyeballed value
@@ -34,15 +36,7 @@ fn with_border_internal(
lines: Vec<Line<'static>>,
forced_inner_width: Option<usize>,
) -> Vec<Line<'static>> {
let max_line_width = lines
.iter()
.map(|line| {
line.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.sum::<usize>()
})
.max()
.unwrap_or(0);
let max_line_width = lines.iter().map(line_width).max().unwrap_or(0);
let content_width = forced_inner_width
.unwrap_or(max_line_width)
.max(max_line_width);
@@ -52,10 +46,7 @@ fn with_border_internal(
out.push(vec![format!("╭{}╮", "─".repeat(border_inner_width)).dim()].into());
for line in lines.into_iter() {
let used_width: usize = line
.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.sum();
let used_width = line_width(&line);
let span_count = line.spans.len();
let mut spans: Vec<Span<'static>> = Vec::with_capacity(span_count + 4);
spans.push(Span::from("│ ").dim());
@@ -90,7 +81,7 @@ impl TooltipHistoryCell {
impl HistoryCell for TooltipHistoryCell {
fn display_lines(&self, width: u16) -> Vec<Line<'static>> {
let indent = " ";
let indent_width = UnicodeWidthStr::width(indent);
let indent_width = display_width(indent);
let wrap_width = usize::from(width.max(1))
.saturating_sub(indent_width)
.max(1);
@@ -303,7 +294,7 @@ impl SessionHeaderHistoryCell {
if max_width == 0 {
return String::new();
}
if UnicodeWidthStr::width(formatted.as_str()) > max_width {
if display_width(formatted.as_str()) > max_width {
return crate::text_formatting::center_truncate_path(&formatted, max_width);
}
}
@@ -371,7 +362,7 @@ impl HistoryCell for SessionHeaderHistoryCell {
let dir_label = format!("{DIR_LABEL:<label_width$}");
let dir_prefix = format!("{dir_label} ");
let dir_prefix_width = UnicodeWidthStr::width(dir_prefix.as_str());
let dir_prefix_width = display_width(dir_prefix.as_str());
let dir_max_width = inner_width.saturating_sub(dir_prefix_width);
let dir = self.format_directory(Some(dir_max_width));
let dir_spans = vec![Span::from(dir_prefix).dim(), Span::from(dir)];
@@ -0,0 +1,10 @@
---
source: tui/src/history_cell/tests.rs
expression: rendered
---
/ps
Background terminals
• echo ガパガパ [...]
↳ output ガパ [...]
@@ -0,0 +1,26 @@
---
source: tui/src/history_cell/tests.rs
expression: "format!(\"{buf:?}\")"
---
Buffer {
area: Rect { x: 0, y: 0, width: 42, height: 6 },
content: [
"╭────────────────────────────────────────╮",
"│ >_ OpenAI Codex (vtest) │",
"│ │",
"│ model: gpt-5 /model to change │",
"│ directory: …パガパガパガパガパ-project │", // hidden by multi-width symbols: [(15, " "), (17, " "), (19, " "), (21, " "), (23, " "), (25, " "), (27, " "), (29, " "), (31, " ")]
"╰────────────────────────────────────────╯",
],
styles: [
x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 5, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 17, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 13, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 18, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 21, y: 3, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE,
x: 27, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 13, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 40, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
]
}
@@ -0,0 +1,37 @@
---
source: tui/src/history_cell/tests.rs
expression: "format!(\"{buf:?}\")"
---
Buffer {
area: Rect { x: 0, y: 0, width: 80, height: 6 },
content: [
"╭───────────────────────────────────────────╮ ",
"│ >_ OpenAI Codex (vtest) │ ",
"│ │ ",
"│ model: gpt-5-ガ-パ /model to change │ ", // hidden by multi-width symbols: [(20, " "), (23, " ")]
"│ directory: project │ ",
"╰───────────────────────────────────────────╯ ",
],
styles: [
x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 5, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 17, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 13, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 24, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 27, y: 3, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE,
x: 33, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 13, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 20, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 5, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 45, y: 5, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
]
}
+48
View File
@@ -783,6 +783,16 @@ fn ps_output_long_command_snapshot() {
insta::assert_snapshot!(rendered);
}
#[test]
fn ps_output_halfwidth_sound_marks_snapshot() {
let cell = new_unified_exec_processes_output(vec![UnifiedExecProcessDetails {
command_display: "echo ガパガパガパガパガパ".to_string(),
recent_chunks: vec!["output ガパガパガパガパガパ".to_string()],
}]);
let rendered = render_lines(&cell.display_lines(/*width*/ 24)).join("\n");
insta::assert_snapshot!(rendered);
}
#[test]
fn ps_output_many_sessions_snapshot() {
let cell = new_unified_exec_processes_output(
@@ -1653,6 +1663,44 @@ fn session_header_indicates_yolo_mode() {
insta::assert_snapshot!(rendered);
}
#[test]
fn session_header_aligns_halfwidth_sound_marks() {
let cell: Box<dyn HistoryCell> = Box::new(SessionHeaderHistoryCell::new(
"gpt-5-ガ-パ".to_string(),
/*reasoning_effort*/ None,
/*show_fast_status*/ false,
PathBuf::from("project"),
"test",
));
let width = 80;
let height = cell.desired_height(width);
let area = Rect::new(0, 0, width, height);
let mut buf = Buffer::empty(area);
cell.render(area, &mut buf);
insta::assert_snapshot!("session_header_halfwidth_sound_marks", format!("{buf:?}"));
}
#[test]
fn session_header_truncates_halfwidth_directory() {
let cell: Box<dyn HistoryCell> = Box::new(SessionHeaderHistoryCell::new(
"gpt-5".to_string(),
/*reasoning_effort*/ None,
/*show_fast_status*/ false,
PathBuf::from("ガパガパガパガパガパガパガパガパ-project"),
"test",
));
let width = 42;
let height = cell.desired_height(width);
let area = Rect::new(0, 0, width, height);
let mut buf = Buffer::empty(area);
cell.render(area, &mut buf);
insta::assert_snapshot!("session_header_halfwidth_directory", format!("{buf:?}"));
}
#[test]
fn yolo_mode_includes_managed_full_access_profiles() {
let permission_profile: PermissionProfile = PermissionProfile::Managed {
+14 -9
View File
@@ -1,11 +1,12 @@
use ratatui::text::Line;
use ratatui::text::Span;
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use unicode_segmentation::UnicodeSegmentation;
use crate::width::display_width;
pub(crate) fn line_width(line: &Line<'_>) -> usize {
line.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.map(|span| display_width(span.content.as_ref()))
.sum()
}
@@ -23,7 +24,7 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L
let mut spans_out: Vec<Span<'static>> = Vec::with_capacity(spans.len());
for span in spans {
let span_width = UnicodeWidthStr::width(span.content.as_ref());
let span_width = display_width(span.content.as_ref());
if span_width == 0 {
spans_out.push(span);
@@ -43,13 +44,13 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L
let style = span.style;
let text = span.content.as_ref();
let mut end_idx = 0usize;
for (idx, ch) in text.char_indices() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
if used + ch_width > max_width {
for (idx, grapheme) in text.grapheme_indices(/*is_extended*/ true) {
let grapheme_width = display_width(grapheme);
if used + grapheme_width > max_width {
break;
}
end_idx = idx + ch.len_utf8();
used += ch_width;
end_idx = idx + grapheme.len();
used += grapheme_width;
}
if end_idx > 0 {
@@ -98,3 +99,7 @@ pub(crate) fn truncate_line_with_ellipsis_if_overflow(
spans,
}
}
#[cfg(test)]
#[path = "line_truncation_tests.rs"]
mod tests;
+19
View File
@@ -0,0 +1,19 @@
use super::line_width;
use super::truncate_line_to_width;
use pretty_assertions::assert_eq;
use ratatui::text::Line;
#[test]
fn halfwidth_sound_marks_stay_with_their_grapheme_when_truncated() {
let line = Line::from("abガc");
assert_eq!(line_width(&line), 5);
assert_eq!(
truncate_line_to_width(line.clone(), /*max_width*/ 3),
Line::from("ab")
);
assert_eq!(
truncate_line_to_width(line, /*max_width*/ 4),
Line::from("abガ")
);
}
+27 -11
View File
@@ -1,5 +1,5 @@
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use crate::width::display_width;
use unicode_segmentation::UnicodeSegmentation;
/// A single visual row produced by RowBuilder.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -11,7 +11,7 @@ pub struct Row {
impl Row {
pub fn width(&self) -> usize {
self.text.width()
display_width(&self.text)
}
}
@@ -140,9 +140,9 @@ impl RowBuilder {
let (prefix, suffix, taken) =
take_prefix_by_width(&self.current_line, self.target_width);
if taken == 0 {
// Avoid infinite loop on pathological inputs; take one scalar and continue.
if let Some((i, ch)) = self.current_line.char_indices().next() {
let len = i + ch.len_utf8();
// Avoid an infinite loop on an indivisible grapheme wider than the target.
if let Some(grapheme) = self.current_line.graphemes(/*is_extended*/ true).next() {
let len = grapheme.len();
let p = self.current_line[..len].to_string();
self.rows.push(Row {
text: p,
@@ -176,13 +176,13 @@ pub fn take_prefix_by_width(text: &str, max_cols: usize) -> (String, &str, usize
}
let mut cols = 0usize;
let mut end_idx = 0usize;
for (i, ch) in text.char_indices() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
if cols.saturating_add(ch_width) > max_cols {
for (i, grapheme) in text.grapheme_indices(/*is_extended*/ true) {
let grapheme_width = display_width(grapheme);
if cols.saturating_add(grapheme_width) > max_cols {
break;
}
cols += ch_width;
end_idx = i + ch.len_utf8();
cols += grapheme_width;
end_idx = i + grapheme.len();
if cols == max_cols {
break;
}
@@ -235,6 +235,22 @@ mod tests {
);
}
#[test]
fn halfwidth_sound_marks_stay_with_their_grapheme_when_wrapping() {
assert_eq!(
take_prefix_by_width("abガc", /*max_cols*/ 3),
("ab".to_string(), "ガc", 2)
);
assert_eq!(
take_prefix_by_width("abガc", /*max_cols*/ 4),
("abガ".to_string(), "c", 4)
);
let mut row_builder = RowBuilder::new(/*target_width*/ 1);
row_builder.push_fragment("ガx");
assert_eq!(row_builder.rows()[0].text, "ガ");
}
#[test]
fn fragmentation_invariance_long_token() {
let s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // 26 chars
+23 -10
View File
@@ -49,6 +49,8 @@ use crate::terminal_hyperlinks::annotate_web_urls_in_line;
use crate::terminal_hyperlinks::remap_wrapped_line;
use crate::terminal_hyperlinks::visible_lines;
use crate::terminal_hyperlinks::web_destination;
use crate::width::char_width;
use crate::width::display_width;
use crate::wrapping::RtOptions;
use crate::wrapping::adaptive_wrap_line;
use crate::wrapping::word_wrap_line;
@@ -72,8 +74,6 @@ use std::ops::Range;
use std::path::Path;
use std::path::PathBuf;
use std::sync::LazyLock;
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use url::Url;
mod streaming;
@@ -1302,7 +1302,7 @@ where
let plain = cell.plain_text();
let mut word_count = 0usize;
for token in plain.split_whitespace() {
let token_width = token.width();
let token_width = display_width(token);
body_token_width = body_token_width.max(token_width);
long_body_token_count += usize::from(token_width >= 20);
word_count += 1;
@@ -1311,7 +1311,7 @@ where
body_token_count += word_count;
total_words += word_count;
total_cells += 1;
total_cell_width += plain.width();
total_cell_width += display_width(&plain);
}
}
@@ -1321,7 +1321,7 @@ where
total_words as f64 / total_cells as f64
};
let avg_cell_width = if total_cells == 0 {
header_plain.width() as f64
display_width(&header_plain) as f64
} else {
total_cell_width as f64 / total_cells as f64
};
@@ -1607,7 +1607,7 @@ where
} else {
current_text.push(ch);
}
column += UnicodeWidthChar::width(ch).unwrap_or(/*default*/ 0);
column += char_width(ch);
}
flush(&mut out, &mut current_text, current_destination);
}
@@ -1756,7 +1756,10 @@ where
#[inline]
fn spans_display_width(spans: &[Span<'_>]) -> usize {
spans.iter().map(|span| span.content.width()).sum()
spans
.iter()
.map(|span| display_width(span.content.as_ref()))
.sum()
}
#[inline]
@@ -1775,7 +1778,10 @@ where
#[inline]
fn longest_token_width(text: &str) -> usize {
text.split_whitespace().map(str::width).max().unwrap_or(0)
text.split_whitespace()
.map(display_width)
.max()
.unwrap_or(0)
}
fn push_inline_style(&mut self, style: Style) {
@@ -1887,7 +1893,7 @@ where
}
} else {
let mut spans = self.current_initial_indent.clone();
let shift = spans.iter().map(|span| span.content.width()).sum::<usize>();
let shift = Self::spans_display_width(&spans);
spans.append(&mut line.line.spans);
for hyperlink in &mut line.hyperlinks {
hyperlink.columns =
@@ -1926,7 +1932,7 @@ where
};
let mut spans = self.prefix_spans(pending_marker_line);
let shift = spans.iter().map(|span| span.content.width()).sum::<usize>();
let shift = Self::spans_display_width(&spans);
spans.append(&mut line.line.spans);
for hyperlink in &mut line.hyperlinks {
hyperlink.columns = hyperlink.columns.start + shift..hyperlink.columns.end + shift;
@@ -2844,6 +2850,13 @@ mod tests {
}));
}
#[test]
fn table_widths_count_halfwidth_sound_marks() {
let cell = make_cell("ガパ");
assert_eq!(W::cell_display_width(&cell), 4);
assert_eq!(W::longest_token_width("ガパtail"), 8);
}
#[test]
fn key_value_table_keeps_web_annotations() {
let destination = "https://example.com/a/very/long/path";
@@ -8,12 +8,12 @@ use crate::render::line_utils::line_to_static;
use crate::render::line_utils::push_owned_lines;
use crate::terminal_hyperlinks::HyperlinkLine;
use crate::terminal_hyperlinks::remap_wrapped_line;
use crate::width::display_width;
use crate::wrapping::RtOptions;
use crate::wrapping::word_wrap_line;
use ratatui::style::Style;
use ratatui::text::Line;
use ratatui::text::Span;
use unicode_width::UnicodeWidthStr;
const FIELD_LEADING_PADDING: usize = 1;
const FIELD_GAP: usize = 2;
@@ -48,7 +48,7 @@ pub(super) fn should_render_records(
let has_fragmented_token = cell
.plain_text()
.split_whitespace()
.any(|token| token.width() > *width);
.any(|token| display_width(token) > *width);
match metrics.kind {
TableColumnKind::Compact => has_fragmented_token,
TableColumnKind::TokenHeavy => {
@@ -105,7 +105,7 @@ pub(super) fn render_records(
) -> Vec<HyperlinkLine> {
let label_width = headers
.iter()
.map(|header| header.plain_text().width())
.map(|header| display_width(&header.plain_text()))
.max()
.unwrap_or(0);
let minimum_value_width = if metrics
@@ -167,9 +167,9 @@ fn render_aligned_field(
let label = header.plain_text();
spans.push(Span::raw(" ".repeat(FIELD_LEADING_PADDING)));
spans.push(Span::styled(label.clone(), label_style));
spans.push(Span::raw(
" ".repeat(label_width.saturating_sub(label.width()) + FIELD_GAP),
));
spans.push(Span::raw(" ".repeat(
label_width.saturating_sub(display_width(&label)) + FIELD_GAP,
)));
} else {
spans.push(Span::raw(" ".repeat(value_indent)));
}
@@ -186,7 +186,7 @@ fn render_stacked_field(
) {
let label_width = available_width
.map(|width| width.saturating_sub(FIELD_LEADING_PADDING).max(1))
.unwrap_or_else(|| header.plain_text().width().max(1));
.unwrap_or_else(|| display_width(&header.plain_text()).max(1));
let label = Line::from(Span::styled(header.plain_text(), label_style));
let mut wrapped_labels = Vec::new();
push_owned_lines(
@@ -218,7 +218,7 @@ fn push_prefixed_value_line(
) {
let shift = prefix
.iter()
.map(|span| span.content.width())
.map(|span| display_width(span.content.as_ref()))
.sum::<usize>();
prefix.append(&mut value_line.line.spans);
let mut output_line = HyperlinkLine::new(Line::from(prefix));
@@ -257,27 +257,11 @@ fn wrap_cell(cell: &TableCell, width: usize) -> Vec<HyperlinkLine> {
fn cell_width(cell: &TableCell) -> usize {
cell.lines
.iter()
.map(|line| {
line.line
.spans
.iter()
.map(|span| span.content.width())
.sum::<usize>()
})
.map(HyperlinkLine::width)
.max()
.unwrap_or(0)
}
fn widest_line_width(lines: &[HyperlinkLine]) -> usize {
lines
.iter()
.map(|line| {
line.line
.spans
.iter()
.map(|span| span.content.width())
.sum::<usize>()
})
.max()
.unwrap_or(0)
lines.iter().map(HyperlinkLine::width).max().unwrap_or(0)
}
+18
View File
@@ -1724,6 +1724,24 @@ fn table_renders_key_value_records_when_compact_fragmentation_is_systemic_snapsh
assert_snapshot!(plain_lines(&text).join("\n"));
}
#[test]
fn table_renders_halfwidth_sound_marks_at_constrained_width_snapshot() {
let md = r#"| Key | Notes |
| --- | --- |
| ガパtail | First ガ row with an escaped \| pipe. |
| パガtail | Second パ row with an escaped \| pipe. |
| short | Final ガパ row. |
"#;
let grid = render_markdown_text_with_width(md, Some(/*width*/ 23));
let records = render_markdown_text_with_width(md, Some(/*width*/ 17));
assert_snapshot!(format!(
"grid (23 cells):\n{}\n\nrecords (17 cells):\n{}",
plain_lines(&grid).join("\n"),
plain_lines(&records).join("\n")
));
}
#[test]
fn table_inside_blockquote_has_quote_prefix() {
let md = "> | A | B |\n> |---|---|\n> | 1 | 2 |\n";
@@ -0,0 +1,41 @@
---
source: tui/src/markdown_render_tests.rs
expression: "format!(\"grid (23 cells):\\n{}\\n\\nrecords (17 cells):\\n{}\", plain_lines(&grid).join(\"\\n\"), plain_lines(&records).join(\"\\n\"))"
---
grid (23 cells):
Key Notes
━━━━━━━━━━ ━━━━━━━━━━━
ガパtail First ガ
row with
an
escaped |
pipe.
────────── ───────────
パガtail Second パ
row with
an
escaped |
pipe.
────────── ───────────
short Final ガ
パ row.
records (17 cells):
Key
ガパtail
Notes
First ガ row
with an escaped
| pipe.
─────────────────
Key
パガtail
Notes
Second パ row
with an escaped
| pipe.
─────────────────
Key
short
Notes
Final ガパ row.
+7 -9
View File
@@ -4,9 +4,11 @@ use crate::history_cell::PlainHistoryCell;
use crate::history_cell::plain_lines;
use crate::history_cell::with_border_with_inner_width;
use crate::legacy_core::config::Config;
use crate::line_truncation::line_width;
use crate::token_usage::TokenUsage;
use crate::token_usage::TokenUsageInfo;
use crate::version::CODEX_CLI_VERSION;
use crate::width::display_width;
use chrono::DateTime;
use chrono::Local;
use codex_app_server_protocol::AskForApproval;
@@ -26,12 +28,10 @@ use ratatui::prelude::*;
use ratatui::style::Stylize;
use std::collections::BTreeSet;
use std::path::PathBuf;
use unicode_width::UnicodeWidthStr;
use url::Url;
use super::account::StatusAccountDisplay;
use super::format::FieldFormatter;
use super::format::line_display_width;
use super::format::push_label;
use super::format::truncate_line_to_width;
use super::helpers::compose_account_display;
@@ -482,7 +482,7 @@ impl StatusHistoryCell {
];
// On narrow terminals, keep the percentage visible rather than
// letting the fixed-width progress bar crowd out the reset time.
let value_spans = if line_display_width(&Line::from(full_value_spans.clone()))
let value_spans = if line_width(&Line::from(full_value_spans.clone()))
<= formatter.value_width(available_inner_width)
{
full_value_spans
@@ -498,9 +498,7 @@ impl StatusHistoryCell {
inline_spans.push(Span::from(" ").dim());
inline_spans.push(resets_span.clone());
if line_display_width(&Line::from(inline_spans.clone()))
<= available_inner_width
{
if line_width(&Line::from(inline_spans.clone())) <= available_inner_width {
lines.push(Line::from(inline_spans));
} else {
lines.push(base_line);
@@ -863,7 +861,7 @@ impl HistoryCell for StatusHistoryCell {
lines.extend(self.rate_limit_lines(&rate_limit_state, available_inner_width, &formatter));
let content_width = lines.iter().map(line_display_width).max().unwrap_or(0);
let content_width = lines.iter().map(line_width).max().unwrap_or(0);
let inner_width = content_width.min(available_inner_width);
let truncated_lines: Vec<Line<'static>> = lines
.into_iter()
@@ -891,10 +889,10 @@ impl HistoryCell for StatusHistoryCell {
.map(|span| span.content.as_ref())
.collect::<String>();
if let Some(start_byte) = visible.find(CHATGPT_USAGE_URL) {
let start = visible[..start_byte].width();
let start = display_width(&visible[..start_byte]);
line.hyperlinks
.push(crate::terminal_hyperlinks::TerminalHyperlink::web(
start..start + CHATGPT_USAGE_URL.width(),
start..start + display_width(CHATGPT_USAGE_URL),
CHATGPT_USAGE_URL.to_string(),
));
}
+12 -17
View File
@@ -1,8 +1,9 @@
use ratatui::prelude::*;
use ratatui::style::Stylize;
use std::collections::BTreeSet;
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use unicode_segmentation::UnicodeSegmentation;
use crate::width::display_width;
#[derive(Debug, Clone)]
pub(crate) struct FieldFormatter {
@@ -21,10 +22,10 @@ impl FieldFormatter {
{
let label_width = labels
.into_iter()
.map(|label| UnicodeWidthStr::width(label.as_ref()))
.map(|label| display_width(label.as_ref()))
.max()
.unwrap_or(0);
let indent_width = UnicodeWidthStr::width(Self::INDENT);
let indent_width = display_width(Self::INDENT);
let value_offset = indent_width + label_width + 1 + 3;
Self {
@@ -72,7 +73,7 @@ impl FieldFormatter {
buf.push_str(label);
buf.push(':');
let label_width = UnicodeWidthStr::width(label);
let label_width = display_width(label);
let padding = 3 + self.label_width.saturating_sub(label_width);
for _ in 0..padding {
buf.push(' ');
@@ -92,12 +93,6 @@ pub(crate) fn push_label(labels: &mut Vec<String>, seen: &mut BTreeSet<String>,
labels.push(owned);
}
pub(crate) fn line_display_width(line: &Line<'static>) -> usize {
line.iter()
.map(|span| UnicodeWidthStr::width(span.content.as_ref()))
.sum()
}
pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> Line<'static> {
if max_width == 0 {
return Line::from(Vec::<Span<'static>>::new());
@@ -109,7 +104,7 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L
for span in line.spans {
let text = span.content.into_owned();
let style = span.style;
let span_width = UnicodeWidthStr::width(text.as_str());
let span_width = display_width(text.as_str());
if span_width == 0 {
spans_out.push(Span::styled(text, style));
@@ -127,13 +122,13 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L
}
let mut truncated = String::new();
for ch in text.chars() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
if used + ch_width > max_width {
for grapheme in text.graphemes(/*is_extended*/ true) {
let grapheme_width = display_width(grapheme);
if used + grapheme_width > max_width {
break;
}
truncated.push(ch);
used += ch_width;
truncated.push_str(grapheme);
used += grapheme_width;
}
if !truncated.is_empty() {
+11 -2
View File
@@ -2,13 +2,13 @@ use crate::exec_command::relativize_to_home;
use crate::legacy_core::config::Config;
use crate::status::StatusAccountDisplay;
use crate::text_formatting;
use crate::width::display_width;
use chrono::DateTime;
use chrono::Local;
use codex_protocol::account::PlanType;
use codex_utils_path_uri::PathConvention;
use codex_utils_path_uri::PathUri;
use std::path::Path;
use unicode_width::UnicodeWidthStr;
fn normalize_agents_display_path(path: &Path) -> String {
dunce::simplified(path).display().to_string()
@@ -164,7 +164,7 @@ pub(crate) fn format_directory_display(directory: &Path, max_width: Option<usize
if max_width == 0 {
return String::new();
}
if UnicodeWidthStr::width(formatted.as_str()) > max_width {
if display_width(&formatted) > max_width {
return text_formatting::center_truncate_path(&formatted, max_width);
}
}
@@ -233,6 +233,15 @@ mod tests {
}
}
#[test]
fn format_directory_display_truncates_halfwidth_sound_marks() {
let directory = Path::new("workspace").join("ガ").join("project");
let max_width = display_width(directory.to_string_lossy().as_ref()) - 1;
let formatted = format_directory_display(&directory, Some(max_width));
insta::assert_snapshot!(formatted.replace('\\', "/"), @"workspace/…/project");
}
#[tokio::test]
async fn compose_agents_summary_includes_global_agents_path() {
let codex_home = TempDir::new().expect("temp codex home");
@@ -0,0 +1,24 @@
---
source: tui/src/status/tests.rs
expression: sanitized
---
/status
╭────────────────────────────────────────╮
│ >_ OpenAI Codex (v0.0.0) │
│ │
│ Visit │
│ https://chatgpt.com/codex/settings/usa │
│ for up-to-date │
│ information on rate limits and credits │
│ │
│ Model: ガパガパガパガ │
│ Directory: [[workspace]] │
│ Permissions: Custom (workspa │
│ Agents.md: <none> │
│ Account: ガパガパガパ@ex │
│ Thread name: ガパガパガパガ │
│ Collaboration mode: ガパ collaborat │
│ │
│ Limits: data not availa │
╰────────────────────────────────────────╯
+36
View File
@@ -1525,6 +1525,42 @@ async fn status_snapshot_truncates_in_narrow_terminal() {
assert_snapshot!(sanitized);
}
#[tokio::test]
async fn status_snapshot_truncates_halfwidth_kana_in_narrow_terminal() {
let temp_home = TempDir::new().expect("temp home");
let mut config = test_config(&temp_home).await;
set_workspace_cwd(&mut config, test_path_buf("/workspace/tests").abs());
let account = StatusAccountDisplay::ChatGpt {
email: Some("ガパガパガパ@example.com".to_string()),
plan: Some("ガパ plan".to_string()),
};
let usage = TokenUsage::default();
let now = chrono::Local
.with_ymd_and_hms(2024, 1, 2, 3, 4, 5)
.single()
.expect("timestamp");
let composite = new_status_output(
&config,
Some(&account),
/*token_info*/ None,
&usage,
&None,
Some("ガパガパガパガパ thread".to_string()),
/*forked_from*/ None,
/*rate_limits*/ None,
/*plan_type*/ None,
now,
"ガパガパガパガパ-model",
Some("ガパ collaboration mode"),
/*reasoning_effort_override*/ None,
);
let rendered_lines = render_lines(&composite.display_lines(/*width*/ 42));
let sanitized = sanitize_directory(rendered_lines).join("\n");
assert_snapshot!(sanitized);
}
#[tokio::test]
async fn status_snapshot_shows_missing_limits_message() {
let temp_home = TempDir::new().expect("temp home");
+29 -12
View File
@@ -20,8 +20,11 @@ use ratatui::widgets::Widget;
use ratatui::widgets::Wrap;
use url::Url;
use crate::line_truncation::line_width;
use crate::render::line_utils::line_to_borrowed;
use crate::render::line_utils::line_to_static;
use crate::width::char_width;
use crate::width::display_width;
use crate::wrapping::RtOptions;
use crate::wrapping::adaptive_wrap_line;
@@ -86,16 +89,12 @@ impl HyperlinkLine {
}
pub(crate) fn width(&self) -> usize {
self.line
.spans
.iter()
.map(|span| usize::from(span.content.as_ref().cell_width()))
.sum()
line_width(&self.line)
}
pub(crate) fn push_span(&mut self, span: Span<'static>, destination: Option<&str>) {
let start = self.width();
let end = start + usize::from(span.content.as_ref().cell_width());
let end = start + display_width(span.content.as_ref());
self.line.push_span(span);
if end > start
&& let Some(destination) = destination.and_then(web_destination)
@@ -158,7 +157,7 @@ pub(crate) fn prefix_hyperlink_lines(
} else {
subsequent_prefix.clone()
};
let shift = usize::from(prefix.content.as_ref().cell_width());
let shift = display_width(prefix.content.as_ref());
let mut spans = Vec::with_capacity(line.line.spans.len() + 1);
spans.push(prefix);
spans.extend(line.line.spans);
@@ -245,7 +244,7 @@ pub(crate) fn remap_wrapped_line(
continue;
};
let mapped = &rendered[rendered_start..];
let mut output_column = usize::from(rendered[..rendered_start].cell_width());
let mut output_column = display_width(&rendered[..rendered_start]);
for ch in mapped.chars() {
let width = char_cell_width(ch);
while source
@@ -320,8 +319,8 @@ pub(crate) fn web_links_in_text(text: &str) -> Vec<TerminalHyperlink> {
let Some(destination) = web_destination(candidate) else {
continue;
};
let start = usize::from(text[..raw_start + trimmed_start].cell_width());
let end = start + usize::from(candidate.cell_width());
let start = display_width(&text[..raw_start + trimmed_start]);
let end = start + display_width(candidate);
links.push(TerminalHyperlink::web(start..end, destination));
}
links
@@ -492,8 +491,7 @@ fn char_cell_width(ch: char) -> usize {
if ch.is_control() {
return 0;
}
let mut encoded = [0; 4];
usize::from(ch.encode_utf8(&mut encoded).cell_width())
char_width(ch)
}
pub(crate) fn mark_buffer_hyperlinks(
@@ -644,6 +642,25 @@ mod tests {
);
}
#[test]
fn hyperlink_columns_follow_a_long_prefix_without_wrapping() {
let prefix = "a".repeat(65_536);
let destination = "https://example.com/long-prefix";
let text = format!("{prefix} {destination}");
assert_eq!(
HyperlinkLine::new(Line::from(text.clone())).width(),
text.len()
);
assert_eq!(
web_links_in_text(&text),
vec![TerminalHyperlink::web(
/*columns*/ 65_537..65_537 + destination.len(),
destination.to_string(),
)]
);
}
#[test]
fn preserves_balanced_parentheses_in_bare_web_urls() {
let destination = "https://en.wikipedia.org/wiki/Function_(mathematics)";
+22 -24
View File
@@ -1,6 +1,6 @@
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use crate::width::display_width;
pub(crate) fn capitalize_first(input: &str) -> String {
let mut chars = input.chars();
@@ -33,14 +33,12 @@ pub(crate) fn format_and_truncate_tool_result(
}
}
/// Format JSON text in a compact single-line format with spaces for better Ratatui wrapping.
/// Ex: `{"a":"b",c:["d","e"]}` -> `{"a": "b", "c": ["d", "e"]}`
/// Returns the formatted JSON string if the input is valid JSON, otherwise returns None.
/// This is a little complicated, but it's necessary because Ratatui's wrapping is *very* limited
/// and can only do line breaks at whitespace. If we use the default serde_json format, we get lines
/// without spaces that Ratatui can't wrap nicely. If we use the serde_json pretty format as-is,
/// it's much too sparse and uses too many terminal rows.
/// Relevant issue: https://github.com/ratatui/ratatui/issues/293
/// Formats JSON on one line with spaces after separators for readability and natural wrap points.
///
/// Compact JSON is hard to scan, while pretty-printed JSON consumes unnecessary terminal rows.
/// Returns the formatted JSON string for valid input, or `None` otherwise.
///
/// Example: `{"a":"b","c":["d","e"]}` becomes `{"a": "b", "c": ["d", "e"]}`.
pub(crate) fn format_json_compact(text: &str) -> Option<String> {
let json = serde_json::from_str::<serde_json::Value>(text).ok()?;
let json_pretty = serde_json::to_string_pretty(&json).unwrap_or_else(|_| json.to_string());
@@ -121,7 +119,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String {
if max_width == 0 {
return String::new();
}
if UnicodeWidthStr::width(path) <= max_width {
if display_width(path) <= max_width {
return path.to_string();
}
@@ -142,7 +140,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String {
if raw_segments.is_empty() {
if has_leading_sep {
let root = sep.to_string();
if UnicodeWidthStr::width(root.as_str()) <= max_width {
if display_width(root.as_str()) <= max_width {
return root;
}
}
@@ -174,27 +172,27 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String {
if allowed_width == 0 {
return String::new();
}
if UnicodeWidthStr::width(original) <= allowed_width {
if display_width(original) <= allowed_width {
return original.to_string();
}
if allowed_width == 1 {
return "…".to_string();
}
let mut kept: Vec<char> = Vec::new();
let mut kept = Vec::new();
let mut used_width = 1; // reserve space for leading ellipsis
for ch in original.chars().rev() {
let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0);
if used_width + ch_width > allowed_width {
for grapheme in original.graphemes(/*is_extended*/ true).rev() {
let grapheme_width = display_width(grapheme);
if used_width + grapheme_width > allowed_width {
break;
}
used_width += ch_width;
kept.push(ch);
used_width += grapheme_width;
kept.push(grapheme);
}
kept.reverse();
let mut truncated = String::from("…");
for ch in kept {
truncated.push(ch);
for grapheme in kept {
truncated.push_str(grapheme);
}
truncated
};
@@ -236,7 +234,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String {
|segments: &mut Vec<Segment<'_>>, allow_front_truncate: bool| -> Option<String> {
loop {
let candidate = assemble(has_leading_sep, segments);
let width = UnicodeWidthStr::width(candidate.as_str());
let width = display_width(candidate.as_str());
if width <= max_width {
return Some(candidate);
}
@@ -263,11 +261,11 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String {
let mut changed = false;
for idx in indices {
let original_width = UnicodeWidthStr::width(segments[idx].original);
let original_width = display_width(segments[idx].original);
if original_width <= max_width && segment_count > 2 {
continue;
}
let seg_width = UnicodeWidthStr::width(segments[idx].text.as_str());
let seg_width = display_width(segments[idx].text.as_str());
let other_width = width.saturating_sub(seg_width);
let allowed_width = max_width.saturating_sub(other_width).max(1);
let new_text = front_truncate(segments[idx].original, allowed_width);
+39 -2
View File
@@ -1,16 +1,38 @@
//! Width guards for transcript rendering with fixed prefix columns.
//! Terminal display-width helpers and guards for fixed prefix columns.
//!
//! Several rendering paths reserve a fixed number of columns for bullets,
//! gutters, or labels before laying out content. When the terminal is very
//! narrow, those reserved columns can consume the entire width, leaving zero
//! or negative space for content.
//!
//! These helpers centralise the subtraction and enforce a strict-positive
//! The display-width helpers match Ratatui's terminal-cell semantics while retaining `usize`
//! precision for long lines. The guards centralise subtraction and enforce a strict-positive
//! contract: they return `Some(n)` where `n > 0`, or `None` when no usable
//! content width remains. Callers treat `None` as "render prefix-only
//! fallback" rather than attempting wrapped rendering at zero width, which
//! would produce empty or unstable output.
use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
/// Returns the display width Ratatui uses for terminal text without its `u16` limit.
pub(crate) fn display_width(text: &str) -> usize {
UnicodeWidthStr::width(text)
+ text
.chars()
.filter(|ch| matches!(ch, '\u{FF9E}' | '\u{FF9F}'))
.count()
}
/// Returns a scalar's terminal width, treating halfwidth sound marks as visible cells.
pub(crate) fn char_width(ch: char) -> usize {
if matches!(ch, '\u{FF9E}' | '\u{FF9F}') {
1
} else {
UnicodeWidthChar::width(ch).unwrap_or(0)
}
}
/// Returns usable content width after reserving fixed columns.
///
/// Guarantees a strict positive width (`Some(n)` where `n > 0`) or `None` when
@@ -36,7 +58,22 @@ pub(crate) fn usable_content_width_u16(total_width: u16, reserved_cols: u16) ->
#[cfg(test)]
mod tests {
use super::*;
use crate::line_truncation::line_width;
use pretty_assertions::assert_eq;
use ratatui::text::Line;
#[test]
fn display_width_matches_ratatui_halfwidth_sound_marks_without_overflow() {
assert_eq!(display_width("ガパ"), 4);
assert_eq!(display_width("ガ゙"), 3);
assert_eq!(display_width("界゙"), 3);
assert_eq!(char_width('\u{FF9E}'), 1);
assert_eq!(char_width('\u{FF9F}'), 1);
let text = "a".repeat(65_536);
assert_eq!(display_width(&text), 65_536);
assert_eq!(line_width(&Line::from(text)), 65_536);
}
#[test]
fn usable_content_width_returns_none_when_reserved_exhausts_width() {
+321 -16
View File
@@ -32,9 +32,198 @@ use std::ops::Range;
use textwrap::Options;
use textwrap::WordSeparator;
use textwrap::core::Word;
use textwrap::core::display_width;
use textwrap::word_splitters::split_words;
use unicode_segmentation::UnicodeSegmentation;
use crate::line_truncation::line_width;
use crate::render::line_utils::push_owned_lines;
use crate::width::display_width;
/// Projected text keeps source-offset lookup separate from legal grapheme split points.
struct ProjectedText {
text: String,
source_boundaries: Vec<(usize, usize)>,
grapheme_boundaries: Vec<usize>,
}
/// Replaces halfwidth sound-mark graphemes with equally wide, textwrap-safe placeholders.
///
/// Source boundaries recover original byte offsets, while grapheme boundaries keep placeholders
/// indivisible and preserve leading whitespace as a wrapping opportunity.
fn project_halfwidth_sound_marks(text: &str) -> Option<ProjectedText> {
if !text.contains(['\u{FF9E}', '\u{FF9F}']) {
return None;
}
let mut projected = String::with_capacity(text.len());
let mut source_boundaries = vec![(0, 0)];
let mut grapheme_boundaries = vec![0];
for (source_start, grapheme) in text.grapheme_indices(/*is_extended*/ true) {
if grapheme.contains(['\u{FF9E}', '\u{FF9F}']) {
let source_end = source_start + grapheme.len();
let content_start = grapheme
.find(|ch: char| !ch.is_whitespace())
.unwrap_or(grapheme.len());
let (whitespace, content) = grapheme.split_at(content_start);
for (offset, ch) in whitespace.char_indices() {
projected.push(ch);
source_boundaries.push((projected.len(), source_start + offset + ch.len_utf8()));
grapheme_boundaries.push(projected.len());
}
let width = display_width(content);
let projected_start = projected.len();
for _ in 0..width / 2 {
if projected.len() > projected_start {
projected.push('\u{2060}');
}
projected.push('界');
source_boundaries.push((projected.len(), source_end));
}
if width % 2 == 1 {
if projected.len() > projected_start {
projected.push('\u{2060}');
}
projected.push('a');
source_boundaries.push((projected.len(), source_end));
}
} else {
for (offset, ch) in grapheme.char_indices() {
projected.push(ch);
source_boundaries.push((projected.len(), source_start + offset + ch.len_utf8()));
}
}
grapheme_boundaries.push(projected.len());
}
Some(ProjectedText {
text: projected,
source_boundaries,
grapheme_boundaries,
})
}
/// Maps a projected byte offset back to the corresponding original-text boundary.
fn source_offset(boundaries: &[(usize, usize)], projected_offset: usize) -> usize {
boundaries
.binary_search_by_key(&projected_offset, |(offset, _)| *offset)
.map(|index| boundaries[index].1)
.unwrap_or(projected_offset)
}
/// Splits oversized projected words without separating placeholders for one source grapheme.
fn break_projected_words<'a>(
words: impl Iterator<Item = Word<'a>>,
projected: &'a ProjectedText,
line_width: usize,
) -> Vec<Word<'a>> {
let projected_start = projected.text.as_ptr() as usize;
let mut pieces = Vec::new();
for word in words {
if display_width(word.word) <= line_width {
pieces.push(word);
continue;
}
let word_start = word.word.as_ptr() as usize - projected_start;
let word_end = word_start + word.word.len();
let mut piece_start = word_start;
let mut piece_width = 0;
let mut atom_start = word_start;
let boundary_start = projected
.grapheme_boundaries
.partition_point(|atom_end| *atom_end <= word_start);
for atom_end in projected
.grapheme_boundaries
.iter()
.copied()
.skip(boundary_start)
{
if atom_end > word_end {
break;
}
let atom_width = display_width(&projected.text[atom_start..atom_end]);
if piece_width > 0 && piece_width + atom_width > line_width {
pieces.push(Word::from(&projected.text[piece_start..atom_start]));
piece_start = atom_start;
piece_width = 0;
}
piece_width += atom_width;
atom_start = atom_end;
}
let mut last = Word::from(&projected.text[piece_start..word_end]);
last.whitespace = word.whitespace;
last.penalty = word.penalty;
pieces.push(last);
}
pieces
}
/// Wraps projected text and translates the resulting ranges back to source byte offsets.
fn wrap_projected_ranges(
projected: &ProjectedText,
opts: &Options<'_>,
include_trailing_spaces: bool,
) -> Vec<Range<usize>> {
let line_widths = [
opts.width
.saturating_sub(display_width(opts.initial_indent)),
opts.width
.saturating_sub(display_width(opts.subsequent_indent)),
];
let line_ending = opts.line_ending.as_str();
let mut ranges = Vec::new();
let mut line_start = 0;
for line in projected.text.split(line_ending) {
let words = opts.word_separator.find_words(line);
let split_words = split_words(words, &opts.word_splitter);
let mut broken_words = if opts.break_words {
break_projected_words(split_words, projected, line_widths[1])
} else {
split_words.collect()
};
if opts.break_words && !opts.initial_indent.is_empty() {
broken_words.insert(0, Word::from(""));
}
let wrapped_words = opts.wrap_algorithm.wrap(&broken_words, &line_widths);
let mut cursor = line_start;
for words in wrapped_words {
let Some(last_word) = words.last() else {
let source = source_offset(&projected.source_boundaries, cursor);
ranges.push(source..source + usize::from(include_trailing_spaces));
continue;
};
let len = words
.iter()
.map(|word| word.word.len() + word.whitespace.len())
.sum::<usize>()
- last_word.whitespace.len();
let end = cursor + len;
let trailing_spaces = if include_trailing_spaces {
projected.text[end..]
.chars()
.take_while(|ch| *ch == ' ')
.count()
} else {
0
};
let source_start = source_offset(&projected.source_boundaries, cursor);
let source_end = source_offset(&projected.source_boundaries, end + trailing_spaces);
ranges.push(source_start..source_end + usize::from(include_trailing_spaces));
cursor = end + last_word.whitespace.len();
}
line_start += line.len() + line_ending.len();
}
ranges
}
/// Returns byte-ranges into `text` for each wrapped line, including
/// trailing whitespace and a +1 sentinel byte. Used by the textarea
@@ -44,6 +233,9 @@ where
O: Into<Options<'a>>,
{
let opts = width_or_options.into();
if let Some(projected) = project_halfwidth_sound_marks(text) {
return wrap_projected_ranges(&projected, &opts, /*include_trailing_spaces*/ true);
}
let mut lines: Vec<Range<usize>> = Vec::new();
let mut cursor = 0usize;
for (line_index, line) in textwrap::wrap(text, &opts).iter().enumerate() {
@@ -87,6 +279,9 @@ where
O: Into<Options<'a>>,
{
let opts = width_or_options.into();
if let Some(projected) = project_halfwidth_sound_marks(text) {
return wrap_projected_ranges(&projected, &opts, /*include_trailing_spaces*/ false);
}
let mut lines: Vec<Range<usize>> = Vec::new();
let mut cursor = 0usize;
for (line_index, line) in textwrap::wrap(text, &opts).iter().enumerate() {
@@ -684,7 +879,7 @@ fn word_wrap_flattened_line<'a>(
// Compute first line range with reduced width due to initial indent.
let initial_width_available = opts
.width
.saturating_sub(rt_opts.initial_indent.width())
.saturating_sub(line_width(&rt_opts.initial_indent))
.max(1);
let initial_wrapped = wrap_ranges_trim(flat, opts.clone().width(initial_width_available));
let Some(first_line_range) = initial_wrapped.first() else {
@@ -713,7 +908,7 @@ fn word_wrap_flattened_line<'a>(
let base = base + skip_leading_spaces;
let subsequent_width_available = opts
.width
.saturating_sub(rt_opts.subsequent_indent.width())
.saturating_sub(line_width(&rt_opts.subsequent_indent))
.max(1);
let remaining_wrapped = wrap_ranges_trim(&flat[base..], opts.width(subsequent_width_available));
for r in &remaining_wrapped {
@@ -758,11 +953,11 @@ fn mixed_url_wrap_line<'a>(
) -> Vec<Line<'a>> {
let initial_width_available = rt_opts
.width
.saturating_sub(rt_opts.initial_indent.width())
.saturating_sub(line_width(&rt_opts.initial_indent))
.max(1);
let subsequent_width_available = rt_opts
.width
.saturating_sub(rt_opts.subsequent_indent.width())
.saturating_sub(line_width(&rt_opts.subsequent_indent))
.max(1);
let ranges = mixed_url_wrap_ranges(flat, initial_width_available, subsequent_width_available);
@@ -831,12 +1026,14 @@ fn mixed_url_wrap_ranges(
0
};
let empty_line_piece_limit = line_limit.saturating_sub(empty_line_prefix_width).max(1);
let mut indivisible = false;
if line_start.is_none() && !piece.is_url && piece.width(text) > empty_line_piece_limit {
pending.splice(
pending_idx..=pending_idx,
split_mixed_url_word(text, piece, empty_line_piece_limit),
);
continue;
let split = split_mixed_url_word(text, piece.clone(), empty_line_piece_limit);
if split.len() > 1 {
pending.splice(pending_idx..=pending_idx, split);
continue;
}
indivisible = true;
}
let piece_width = piece.width(text);
@@ -845,6 +1042,7 @@ fn mixed_url_wrap_ranges(
.unwrap_or(0);
let fits = if line_start.is_none() {
piece.is_url
|| indivisible
|| empty_line_prefix_width + piece_width <= line_limit
|| empty_line_prefix_width >= line_limit
} else {
@@ -894,16 +1092,27 @@ fn split_mixed_url_word(text: &str, word: MixedUrlWord, line_limit: usize) -> Ve
return vec![word];
}
let source = Word::from(&text[word.range.clone()]);
let mut offset = word.range.start;
let mut pieces = Vec::new();
for piece in source.break_apart(line_limit.max(1)) {
let end = offset + piece.word.len();
let mut start = word.range.start;
let mut width = 0usize;
for (offset, grapheme) in text[word.range.clone()].grapheme_indices(/*is_extended*/ true) {
let grapheme_width = display_width(grapheme);
if width > 0 && width + grapheme_width > line_limit.max(1) {
let end = word.range.start + offset;
pieces.push(MixedUrlWord {
range: start..end,
is_url: false,
});
start = end;
width = 0;
}
width += grapheme_width;
}
if start < word.range.end {
pieces.push(MixedUrlWord {
range: offset..end,
range: start..word.range.end,
is_url: false,
});
offset = end;
}
pieces
}
@@ -1490,6 +1699,24 @@ them."#
);
}
#[test]
fn adaptive_wrap_line_mixed_url_counts_halfwidth_sound_marks() {
let line = Line::from("ガパtail https://x.co");
let out = adaptive_wrap_line(&line, RtOptions::new(/*width*/ 4));
let rendered = out.iter().map(concat_line).collect_vec();
assert_eq!(rendered, ["ガパ", "tail", "https://x.co"]);
}
#[test]
fn adaptive_wrap_line_mixed_url_makes_progress_for_an_indivisible_grapheme() {
let line = Line::from("ガ https://x.co");
let out = adaptive_wrap_line(&line, RtOptions::new(/*width*/ 1));
let rendered = out.iter().map(concat_line).collect_vec();
assert_eq!(rendered, ["ガ", "https://x.co"]);
}
#[test]
fn map_owned_wrapped_line_to_range_recovers_on_non_prefix_mismatch() {
// Match source chars first, then introduce a non-penalty mismatch.
@@ -1553,6 +1780,84 @@ them."#
assert_eq!(rebuilt, text);
}
#[test]
fn wrap_ranges_count_halfwidth_sound_marks_without_changing_byte_offsets() {
for (text, width, expected) in [
("abガc", 4, &["abガ", "c"][..]),
("abガc", 3, &["ab", "ガc"][..]),
("゙ab", 2, &["゙a", "b"][..]),
("a ゙b", 2, &["a", "゙b"][..]),
("a ゚b", 2, &["a", "゚b"][..]),
("ガ゙x", 3, &["ガ゙", "x"][..]),
("界゙x", 3, &["界゙", "x"][..]),
("ガ゙ab", 2, &["ガ゙", "ab"][..]),
("界゙ab", 2, &["界゙", "ab"][..]),
("abガ゙cd", 2, &["ab", "ガ゙", "cd"][..]),
("ab界゙cd", 2, &["ab", "界゙", "cd"][..]),
] {
let ranges = wrap_ranges_trim(text, Options::new(width));
let wrapped = ranges
.iter()
.map(|range| &text[range.clone()])
.collect_vec();
assert_eq!(wrapped, expected);
}
for (text, emoji, sound_mark) in [("a👨‍👩 ゙", "👨‍👩", "゙"), ("a👍🏻 ゚", "👍🏻", "゚")]
{
for options in [
Options::new(/*width*/ 2),
Options::new(/*width*/ 2).word_separator(WordSeparator::AsciiSpace),
] {
let ranges = wrap_ranges_trim(text, options);
let wrapped = ranges
.iter()
.map(|range| &text[range.clone()])
.collect_vec();
assert_eq!(wrapped, ["a", emoji, sound_mark]);
}
}
for grapheme in ["ガ゙", "界゙"] {
for width in [1, 2] {
let ranges = wrap_ranges(grapheme, Options::new(width));
assert_eq!(ranges, std::iter::once(0..grapheme.len() + 1).collect_vec());
}
}
}
#[test]
fn wrap_ranges_preserve_crlf_source_boundaries_without_splitting_graphemes() {
for prefix in ["ガ", "パ"] {
let text = format!("{prefix}\r\nnext");
for word_separator in [
WordSeparator::UnicodeBreakProperties,
WordSeparator::AsciiSpace,
] {
for line_ending in [textwrap::LineEnding::LF, textwrap::LineEnding::CRLF] {
let options = Options::new(/*width*/ 4)
.line_ending(line_ending)
.word_separator(word_separator)
.wrap_algorithm(textwrap::WrapAlgorithm::FirstFit);
let first_end =
prefix.len() + usize::from(line_ending == textwrap::LineEnding::LF);
let second_start = prefix.len() + "\r\n".len();
let ranges = wrap_ranges(&text, options.clone());
assert_eq!(ranges, [0..first_end + 1, second_start..text.len() + 1]);
for range in &ranges {
assert!(text.get(range.start..range.end - 1).is_some());
}
let trimmed = wrap_ranges_trim(&text, options);
assert_eq!(trimmed, [0..first_end, second_start..text.len()]);
}
}
}
}
#[test]
fn map_owned_wrapped_line_to_range_repro_overconsumes_repeated_prefix_patterns() {
let text = "- - foo";