From ad8a5e3a1b4d6b8dd88dd218027afeef29ed407b Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Mon, 14 Sep 2026 19:08:04 +0000 Subject: [PATCH] Render inline TeX math as Unicode in the TUI (#45501) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed Convert supported expressions inside `$...$` and `\(...\)` to readable Unicode in regular and streaming Markdown. For example, `$\alpha^2 + \beta_{10}$` renders as `α² + β₁₀`. Support a bounded TeX subset including symbols, superscripts, subscripts, square roots, and parenthesized fractions, with verbatim fallback for unsupported expressions. Exclude code, links, HTML, and display equations from conversion, and avoid interpreting common currency and shell syntax as math. Preserve file citation paths and keep streaming cache boundaries outside display equations. ## Testing Add snapshots for inline rendering and narrow wrapping, plus regression tests for unsupported input, parser limits, Markdown contexts, file citations, and streaming display boundaries across chunks and widths. GitOrigin-RevId: 828bf7b48345b05bc6c2fb5cb2798412c3b71dbe --- codex-rs/tui/src/markdown_render.rs | 6 +- .../tui/src/markdown_render/file_citations.rs | 4 +- .../markdown_render/file_citations_tests.rs | 8 + codex-rs/tui/src/markdown_render/math.rs | 206 ++++++++++++++ .../tui/src/markdown_render/math/render.rs | 265 ++++++++++++++++++ .../tui/src/markdown_render/math_tests.rs | 111 ++++++++ ...__unicode_math_inline_narrow_snapshot.snap | 7 + ...__tests__unicode_math_inline_snapshot.snap | 10 + codex-rs/tui/src/markdown_render/streaming.rs | 16 +- .../tui/src/streaming/inline_math_tests.rs | 54 ++++ codex-rs/tui/src/streaming/render.rs | 4 + 11 files changed, 685 insertions(+), 6 deletions(-) create mode 100644 codex-rs/tui/src/markdown_render/math.rs create mode 100644 codex-rs/tui/src/markdown_render/math/render.rs create mode 100644 codex-rs/tui/src/markdown_render/math_tests.rs create mode 100644 codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_narrow_snapshot.snap create mode 100644 codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_snapshot.snap create mode 100644 codex-rs/tui/src/streaming/inline_math_tests.rs diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index a9f33b1291..a80abc192d 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -70,6 +70,7 @@ use std::path::PathBuf; mod file_citations; mod local_links; +mod math; mod streaming; mod table_key_value; mod web_links; @@ -338,7 +339,10 @@ pub(crate) fn render_markdown_lines_with_width_cwd_and_hidden_link_destinations( let mut options = Options::empty(); options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_TABLES); - let parser = DecodedTextMerge::new(Parser::new_ext(input, options).into_offset_iter()); + let math = math::MathMarkdown::new(input, options); + let parser = DecodedTextMerge::new( + math.events(Parser::new_ext(&math.markdown, options).into_offset_iter()), + ); let mut w = Writer::new(input, parser, width, cwd, is_hidden_link_destination); w.run(); w.text diff --git a/codex-rs/tui/src/markdown_render/file_citations.rs b/codex-rs/tui/src/markdown_render/file_citations.rs index f14ceb3134..afac14f9b8 100644 --- a/codex-rs/tui/src/markdown_render/file_citations.rs +++ b/codex-rs/tui/src/markdown_render/file_citations.rs @@ -110,11 +110,11 @@ impl<'a> FileCitations<'a> { /// Adapt before `DecodedTextMerge`, while plain text still has exact source offsets. pub(super) fn events<'s>( &'s self, - parser: Parser<'s>, + events: impl Iterator, Range)>, cwd: Option<&'s Path>, ) -> impl Iterator, Range)> { let mut citations = self.citations.iter().peekable(); - parser.into_offset_iter().flat_map(move |(event, range)| { + events.flat_map(move |(event, range)| { while citations .next_if(|(span, _)| span.end <= range.start) .is_some() diff --git a/codex-rs/tui/src/markdown_render/file_citations_tests.rs b/codex-rs/tui/src/markdown_render/file_citations_tests.rs index 54cebcfc6b..37e580b734 100644 --- a/codex-rs/tui/src/markdown_render/file_citations_tests.rs +++ b/codex-rs/tui/src/markdown_render/file_citations_tests.rs @@ -17,6 +17,7 @@ fn rendered_text(markdown: &str, cwd: Option<&Path>) -> String { fn file_citation_paths_preserve_markdown_significant_characters() { for path in [ "/tmp/a*b*.txt", + "/tmp/$x$/report.md", "/tmp/a`b`.txt", "/tmp/a.txt", "/tmp/report#L10", @@ -158,6 +159,13 @@ fn multiple_file_citations_render_without_interpreting_encoded_source() { #[test] fn file_citations_preserve_adjacent_entities_and_escaped_punctuation() { + assert_eq!( + rendered_text( + r"$\alpha$:codex-file-citation{path=/tmp/$x$/report.md}$\beta$", + /*cwd*/ None + ), + "α/tmp/$x$/report.mdβ", + ); let cwd = std::env::temp_dir(); for (markdown, expected) in [ ( diff --git a/codex-rs/tui/src/markdown_render/math.rs b/codex-rs/tui/src/markdown_render/math.rs new file mode 100644 index 0000000000..741d88cc37 --- /dev/null +++ b/codex-rs/tui/src/markdown_render/math.rs @@ -0,0 +1,206 @@ +//! Recognize math before Markdown consumes TeX escapes, retaining exact source offsets. +//! +//! Only the rendering copy is masked. Unsupported expressions are restored verbatim; code, +//! links, HTML, and display equations stay under the ordinary Markdown renderer. + +use itertools::Either; +use pulldown_cmark::Event; +use pulldown_cmark::Options; +use pulldown_cmark::Parser; +use pulldown_cmark::Tag; +use std::borrow::Cow; +use std::ops::Range; + +mod render; + +const MAX_MATH_BYTES: usize = 4096; + +pub(super) struct MathMarkdown<'a> { + pub(super) markdown: Cow<'a, str>, + pub(super) display_ranges: Vec>, + replacements: Vec<(Range, String)>, +} + +impl<'a> MathMarkdown<'a> { + pub(super) fn new(input: &'a str, options: Options) -> Self { + let mut result = Self { + markdown: Cow::Borrowed(input), + display_ranges: Vec::new(), + replacements: Vec::new(), + }; + if !input.contains('$') && !input.contains("\\(") && !input.contains("\\[") { + return result; + } + let parser = Parser::new_ext(input, options); + let mut protected: Vec<_> = parser + .reference_definitions() + .iter() + .map(|(_, def)| def.span.clone()) + .collect(); + protected.extend(parser.into_offset_iter().filter_map(|(event, range)| { + matches!( + event, + Event::Code(_) + | Event::Html(_) + | Event::InlineHtml(_) + | Event::Start(Tag::CodeBlock(_) | Tag::Link { .. } | Tag::Image { .. }) + ) + .then_some(range) + })); + protected.sort_unstable_by_key(|range| range.start); + let mut protected = protected.into_iter().peekable(); + let mut offset = 0; + let mut scanned = 0; + let mut line_start = 0; + while offset < input.len() { + if let Some(index) = input[scanned..offset].rfind('\n') { + line_start = scanned + index + 1; + } + scanned = offset; + while protected.next_if(|range| range.end <= offset).is_some() {} + if let Some(range) = protected.peek() + && range.contains(&offset) + { + offset = range.end; + continue; + } + let rest = &input[offset..]; + let (open, close, display) = if rest.starts_with("$$") { + ("$$", "$$", true) + } else if rest.starts_with("\\[") { + ("\\[", "\\]", true) + } else if rest.starts_with("\\(") { + ("\\(", "\\)", false) + } else if rest.starts_with('$') { + ("$", "$", false) + } else { + let Some(ch) = rest.chars().next() else { + break; + }; + offset += ch.len_utf8(); + continue; + }; + let start = offset; + offset += open.len(); + if escaped(input, start) { + continue; + } + let body = &input[offset..]; + if open == "$" + && (body.starts_with(char::is_whitespace) || body.starts_with(['(', '{'])) + { + continue; + } + let limit = body + .char_indices() + .map(|(i, _)| i) + .find(|i| *i >= MAX_MATH_BYTES) + .unwrap_or(body.len()); + // Display boundaries outlive the conversion budget; a distant closer still owns its opener. + let search = if display { body } else { &body[..limit] }; + let end = search.match_indices(close).find_map(|(index, _)| { + let end = offset + index; + (!escaped(input, end)).then_some(end) + }); + if display { + if open == "$$" && end.is_none() && !input[line_start..start].trim().is_empty() { + continue; + } + // Exclude display equations from inline recognition, including unfinished ones. + offset = end.map_or(input.len(), |end| end + close.len()); + result.display_ranges.push(start..offset); + continue; + } + let Some(end) = end else { + continue; + }; + let span = start..end + close.len(); + if protected.peek().is_some_and(|range| range.start < span.end) { + continue; + } + let formula = &input[offset..end]; + if formula.contains('\n') { + continue; + } + if open == "$" { + let next = input[span.end..].chars().next(); + if formula.ends_with(char::is_whitespace) || next.is_some_and(char::is_alphanumeric) + { + continue; + } + if formula.starts_with(|ch: char| ch.is_ascii_digit()) + && !formula.contains(['\\', '^', '_', '=', '+', '-', '*', '/', '<', '>']) + || formula.len() > 1 && formula.chars().all(|ch| ch.is_ascii_uppercase()) + { + offset = span.end; + continue; + } + } + let rendered = + render::render(formula).unwrap_or_else(|| input[span.clone()].to_owned()); + // Dollars are ordinary text in the Markdown parser and cannot form an HTML tag. + result + .markdown + .to_mut() + .replace_range(span.clone(), &"$".repeat(span.len())); + offset = span.end; + result.replacements.push((span, rendered)); + } + result + } + + pub(super) fn events<'s>( + &'s self, + events: impl Iterator, Range)>, + ) -> impl Iterator, Range)> { + let mut replacements = self.replacements.iter().peekable(); + events.flat_map(move |(event, range)| { + while replacements + .next_if(|(span, _)| span.end <= range.start) + .is_some() + {} + let Event::Text(text) = event else { + return Either::Left(std::iter::once((event, range))); + }; + if replacements + .peek() + .is_none_or(|(span, _)| span.start >= range.end) + { + return Either::Left(std::iter::once((Event::Text(text), range))); + } + let mut output = Vec::new(); + let mut offset = range.start; + while let Some((span, text)) = replacements.next_if(|(span, _)| span.end <= range.end) { + if offset < span.start { + output.push(( + Event::Text(self.markdown[offset..span.start].into()), + offset..span.start, + )); + } + output.push((Event::Text(text.as_str().into()), span.clone())); + offset = span.end; + } + if offset < range.end { + output.push(( + Event::Text(self.markdown[offset..range.end].into()), + offset..range.end, + )); + } + Either::Right(output.into_iter()) + }) + } +} + +fn escaped(input: &str, offset: usize) -> bool { + input[..offset] + .bytes() + .rev() + .take_while(|byte| *byte == b'\\') + .count() + % 2 + == 1 +} + +#[cfg(test)] +#[path = "math_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/markdown_render/math/render.rs b/codex-rs/tui/src/markdown_render/math/render.rs new file mode 100644 index 0000000000..4b1ba3f29e --- /dev/null +++ b/codex-rs/tui/src/markdown_render/math/render.rs @@ -0,0 +1,265 @@ +//! Bounded Unicode layout for a deliberately small TeX math subset. + +use crate::width::display_width; + +pub(super) fn render(source: &str) -> Option { + let mut parser = MathParser { + remaining: source.trim(), + depth: 0, + }; + let result = parser.sequence(/*group*/ false)?; + (!result.trim().is_empty()).then_some(result) +} + +struct MathParser<'a> { + remaining: &'a str, + depth: usize, +} + +impl MathParser<'_> { + fn take(&mut self) -> Option { + let ch = self.remaining.chars().next()?; + self.remaining = &self.remaining[ch.len_utf8()..]; + Some(ch) + } + + fn sequence(&mut self, group: bool) -> Option { + if self.depth >= 32 { + return None; + } + self.depth += 1; + let mut result = String::new(); + let mut scripts = 0; + let mut has_base = false; + while let Some(ch) = self.remaining.chars().next() { + if ch == '}' { + if !group { + return None; + } + self.take(); + self.depth -= 1; + return Some(result); + } + if ch == '^' || ch == '_' { + let script = if ch == '^' { 1 } else { 2 }; + if !has_base || scripts & script != 0 { + return None; + } + scripts |= script; + } else if !ch.is_whitespace() { + scripts = 0; + } + let atom = self.atom()?; + if !ch.is_whitespace() && ch != '^' && ch != '_' { + // Flattening a compound base would change the scope of a following script. + has_base = atom.chars().count() == 1; + } + result.push_str(&atom); + if display_width(&result) > 256 { + return None; + } + } + self.depth -= 1; + (!group).then_some(result) + } + + fn argument(&mut self) -> Option { + self.remaining = self.remaining.trim_start(); + if self.remaining.starts_with(['^', '_']) { + return None; + } + self.atom() + } + + fn atom(&mut self) -> Option { + if self.depth >= 32 { + return None; + } + self.depth += 1; + let result = self.atom_inner(); + self.depth -= 1; + result + } + + fn atom_inner(&mut self) -> Option { + let ch = self.take()?; + match ch { + '{' => self.sequence(/*group*/ true), + '\\' => self.command(), + '^' | '_' => { + let arg = self.argument()?; + let (plain, alphabet) = if ch == '^' { + ( + "0123456789+-=()abcdefghijklmnoprstuvwxyz", + "⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖʳˢᵗᵘᵛʷˣʸᶻ", + ) + } else { + ( + "0123456789+-=()aehijklmnoprstuvx", + "₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₕᵢⱼₖₗₘₙₒₚᵣₛₜᵤᵥₓ", + ) + }; + let mut output = String::new(); + for value in arg.chars() { + let index = plain.chars().position(|ch| ch == value)?; + output.push(alphabet.chars().nth(index)?); + } + Some(output) + } + '}' | '$' | '%' | '#' | '&' | '`' => None, + ch if ch.is_whitespace() => { + self.remaining = self.remaining.trim_start(); + Some(String::from(" ")) + } + ch if ch.is_control() => None, + ch => Some(ch.to_string()), + } + } + + fn command(&mut self) -> Option { + let length = self + .remaining + .bytes() + .take_while(u8::is_ascii_alphabetic) + .count(); + if length == 0 { + return match self.take()? { + ',' | ';' | ':' | ' ' => Some(String::from(" ")), + '!' => Some(String::new()), + '{' => Some(String::from("{")), + '}' => Some(String::from("}")), + '|' => Some(String::from("‖")), + _ => None, + }; + } + let name = &self.remaining[..length]; + self.remaining = &self.remaining[length..]; + match name { + "frac" | "dfrac" | "tfrac" => { + let numerator = self.argument()?; + let denominator = self.argument()?; + Some(format!("(({numerator})/({denominator}))")) + } + "sqrt" => { + if self.remaining.trim_start().starts_with('[') { + return None; + } + let radicand = self.argument()?; + Some(format!("√({radicand})")) + } + "mathbb" => { + let arg = self.argument()?; + let text = match arg.as_str() { + "R" => "ℝ", + "C" => "ℂ", + "N" => "ℕ", + "Z" => "ℤ", + "Q" => "ℚ", + "P" => "ℙ", + _ => return None, + }; + Some(String::from(text)) + } + "mathrm" | "mathbf" | "mathit" => self.argument(), + "text" | "operatorname" => { + self.remaining = self.remaining.trim_start().strip_prefix('{')?; + let end = self.remaining.find('}')?; + let text = &self.remaining[..end]; + if text.contains(['{', '\\', '$', '%', '#', '&']) + || text.chars().any(char::is_control) + { + return None; + } + self.remaining = &self.remaining[end + 1..]; + Some(String::from(text)) + } + "left" | "right" => { + self.remaining = self.remaining.trim_start(); + match self.take()? { + '.' => Some(String::new()), + ch @ ('(' | ')' | '[' | ']' | '|') => Some(ch.to_string()), + _ => None, + } + } + "quad" | "qquad" => Some(String::from(" ")), + "sin" | "cos" | "tan" | "log" | "ln" | "exp" | "lim" | "max" | "min" => { + Some(String::from(name)) + } + _ => symbol(name).map(String::from), + } + } +} + +fn symbol(name: &str) -> Option<&'static str> { + Some(match name { + "alpha" => "α", + "beta" => "β", + "gamma" => "γ", + "delta" => "δ", + "epsilon" => "ϵ", + "varepsilon" => "ε", + "zeta" => "ζ", + "eta" => "η", + "theta" => "θ", + "vartheta" => "ϑ", + "iota" => "ι", + "kappa" => "κ", + "lambda" => "λ", + "mu" => "μ", + "nu" => "ν", + "xi" => "ξ", + "pi" => "π", + "rho" => "ρ", + "sigma" => "σ", + "tau" => "τ", + "upsilon" => "υ", + "phi" => "ϕ", + "varphi" => "φ", + "chi" => "χ", + "psi" => "ψ", + "omega" => "ω", + "Gamma" => "Γ", + "Delta" => "Δ", + "Theta" => "Θ", + "Lambda" => "Λ", + "Xi" => "Ξ", + "Pi" => "Π", + "Sigma" => "Σ", + "Upsilon" => "Υ", + "Phi" => "Φ", + "Psi" => "Ψ", + "Omega" => "Ω", + "sum" => "∑", + "prod" => "∏", + "int" => "∫", + "infty" => "∞", + "partial" => "∂", + "nabla" => "∇", + "pm" => "±", + "mp" => "∓", + "times" => "×", + "cdot" => "·", + "div" => "÷", + "le" | "leq" => "≤", + "ge" | "geq" => "≥", + "ne" | "neq" => "≠", + "approx" => "≈", + "equiv" => "≡", + "in" => "∈", + "notin" => "∉", // codespell:ignore notin + "subset" => "⊂", + "subseteq" => "⊆", + "cup" => "∪", + "cap" => "∩", + "emptyset" => "∅", + "forall" => "∀", + "exists" => "∃", + "to" | "rightarrow" => "→", + "leftarrow" => "←", + "Rightarrow" => "⇒", + "Leftrightarrow" => "⇔", + "ldots" | "dots" => "…", + "cdots" => "⋯", + _ => return None, + }) +} diff --git a/codex-rs/tui/src/markdown_render/math_tests.rs b/codex-rs/tui/src/markdown_render/math_tests.rs new file mode 100644 index 0000000000..9affeacc56 --- /dev/null +++ b/codex-rs/tui/src/markdown_render/math_tests.rs @@ -0,0 +1,111 @@ +use super::render::render; +use crate::markdown_render::render_markdown_text_with_width; +use itertools::Itertools; +use pretty_assertions::assert_eq; + +fn plain(source: &str, width: usize) -> String { + render_markdown_text_with_width(source, Some(width)) + .lines + .iter() + .map(ToString::to_string) + .join("\n") +} + +#[test] +fn unicode_math_inline_snapshot() { + insta::assert_snapshot!(plain( + r"Inline $\alpha^2 + \beta_{10}$ on $\mathbb{R}^n$. +Root $\sqrt{x^2+y^2}$ and fraction $a/\frac{b}{c}$. +Unsupported: $\unknown{x_y}$ and $\frac{a}{b}^2$. +Angles: <$x$>, <\(\alpha\)>, and <$\unknown{x}$>. +Code: `$\alpha$`; money: $5 and $10; shell: $HOME.", + /*width*/ 80 + )); +} + +#[test] +fn unicode_math_inline_narrow_snapshot() { + insta::assert_snapshot!(plain( + r"Words $\alpha^2 + \beta_1$ more words and $\sqrt{x}$.", + /*width*/ 16 + )); +} + +#[test] +fn unicode_math_preserves_markdown_contexts() { + for (source, expected) in [ + ( + r"**$\alpha_1$** & \$5.00 and $\beta^2$", + "α₁ & $5.00 and β²", + ), + ( + r"`$\alpha$` $HOME ${HOME} $(echo x) $5 and $10", + r"$\alpha$ $HOME ${HOME} $(echo x) $5 and $10", + ), + ( + r"[x](https://example.com/$HOME) and $\alpha$", + "x (https://example.com/$HOME) and α", + ), + (r"$\left. x \right|$ $\text{x_y^z}$", " x | x_y^z"), + (r"\(\alpha^2\)", "α²"), + (r"Value: <$x$> and <\(\alpha\)>.", "Value: and <α>."), + (r"$USD$+$\alpha$", "$USD$+α"), + (r"Costs $5; $\alpha$ and $x^2$.", "Costs $5; α and x²."), + (r"$HOME then $\alpha$", "$HOME then α"), + ( + r"Run echo $$ to print PID. Then $\alpha$", + "Run echo $$ to print PID. Then α", + ), + ] { + assert_eq!(plain(source, /*width*/ 80), expected); + } + // This stage leaves display equations and TeX fences to ordinary Markdown rendering. + for source in [r"$$x^2$$", "$$\nx^2\n$$", r"\[x^2\]", "```latex\nx^2\n```"] { + assert!(!plain(source, /*width*/ 80).contains('²')); + } +} + +#[test] +fn unicode_math_bounds_and_unsupported_input() { + for source in [ + r"\unknown{x}", + r"\frac{a}", + "{x", + "x}", + "x^{q}", + "^2", + "x^2^3", + "{a+b}^2", + "{x^2}^3", + r"\frac{a}{b}^2", + r"\sqrt[3]{x}", + r"\sqrt [3]{x}", + r"\left x", + r"\text{\alpha}", + r"\begin{matrix}a&b\end{matrix}", + ] { + assert_eq!(render(source), None, "{source}"); + assert_eq!( + plain(&format!("\\({source}\\)"), /*width*/ 80), + format!("\\({source}\\)") + ); + } + for source in [ + format!("{}x{}", "{".repeat(/*n*/ 40), "}".repeat(/*n*/ 40)), + "x".repeat(/*n*/ 300), + format!("{}x", "\\sqrt".repeat(/*n*/ 100)), + ] { + assert_eq!(render(&source), None); + } +} + +#[test] +fn unicode_math_oversized_display_does_not_hide_following_inline_math() { + for (open, close) in [("$$", "$$"), (r"\[", r"\]")] { + let source = format!( + "{open}\n{}\n{close}\n\nAfter $\\alpha$.\n", + "x".repeat(/*n*/ 5000) + ); + assert!(plain(&source, /*width*/ 80).ends_with("After α.")); + } +} diff --git a/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_narrow_snapshot.snap b/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_narrow_snapshot.snap new file mode 100644 index 0000000000..bcdfbfaabc --- /dev/null +++ b/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_narrow_snapshot.snap @@ -0,0 +1,7 @@ +--- +source: tui/src/markdown_render/math_tests.rs +expression: "plain(r\"Words $\\alpha^2 + \\beta_1$ more words and $\\sqrt{x}$.\", 16)" +--- +Words α² + β₁ +more words and +√(x). diff --git a/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_snapshot.snap b/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_snapshot.snap new file mode 100644 index 0000000000..4ef059160a --- /dev/null +++ b/codex-rs/tui/src/markdown_render/snapshots/codex_tui__markdown_render__math__tests__unicode_math_inline_snapshot.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/markdown_render/math_tests.rs +assertion_line: 16 +expression: "plain(r\"Inline $\\alpha^2 + \\beta_{10}$ on $\\mathbb{R}^n$.\nRoot $\\sqrt{x^2+y^2}$ and fraction $a/\\frac{b}{c}$.\nUnsupported: $\\unknown{x_y}$ and $\\frac{a}{b}^2$.\nAngles: <$x$>, <\\(\\alpha\\)>, and <$\\unknown{x}$>.\nCode: `$\\alpha$`; money: $5 and $10; shell: $HOME.\",\n80)" +--- +Inline α² + β₁₀ on ℝⁿ. +Root √(x²+y²) and fraction a/((b)/(c)). +Unsupported: $\unknown{x_y}$ and $\frac{a}{b}^2$. +Angles: , <α>, and <$\unknown{x}$>. +Code: $\alpha$; money: $5 and $10; shell: $HOME. diff --git a/codex-rs/tui/src/markdown_render/streaming.rs b/codex-rs/tui/src/markdown_render/streaming.rs index 66c55f9252..75fefdae7a 100644 --- a/codex-rs/tui/src/markdown_render/streaming.rs +++ b/codex-rs/tui/src/markdown_render/streaming.rs @@ -11,6 +11,7 @@ use super::Options; use super::Parser; use super::Tag; use super::Writer; +use super::math::MathMarkdown; use std::ops::Range; use std::path::Path; @@ -40,10 +41,11 @@ pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd( options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_TABLES); let citations = FileCitations::new(input, options); - let parser = Parser::new_ext(&citations.markdown, options); + let math = MathMarkdown::new(&citations.markdown, options); + let parser = Parser::new_ext(&math.markdown, options); let has_reference_link_definition = parser.reference_definitions().iter().next().is_some(); let parser = TopLevelBlockTracker { - iter: DecodedTextMerge::new(citations.events(parser, cwd)), + iter: DecodedTextMerge::new(citations.events(math.events(parser.into_offset_iter()), cwd)), depth: 0, block_count: 0, last_start: 0, @@ -53,7 +55,15 @@ pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd( writer.run(); StreamingMarkdownRender { lines: writer.text, - last_top_level_block_start: (writer.iter.block_count > 1).then_some(writer.iter.last_start), + last_top_level_block_start: (writer.iter.block_count > 1) + .then_some(writer.iter.last_start) + // A cached suffix must not mistake a display closer for a new opener. + .filter(|boundary| { + !math + .display_ranges + .iter() + .any(|range| range.start < *boundary && *boundary < range.end) + }), has_reference_link_definition, first_top_level_block_is_html: writer.iter.first_is_html, } diff --git a/codex-rs/tui/src/streaming/inline_math_tests.rs b/codex-rs/tui/src/streaming/inline_math_tests.rs new file mode 100644 index 0000000000..7da430c6f0 --- /dev/null +++ b/codex-rs/tui/src/streaming/inline_math_tests.rs @@ -0,0 +1,54 @@ +use super::StreamingRender; +use crate::history_cell::HistoryRenderMode; +use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd; +use pretty_assertions::assert_eq; + +#[test] +fn unicode_math_inline_stream_preserves_display_exclusion_boundaries() { + let cwd = std::env::temp_dir(); + for delimiters in [("$$", "$$"), ("\\[", "\\]")] { + let mut render = StreamingRender::new(); + let mut source = String::new(); + for chunk in [ + delimiters.0, + "\n\nx^2\n\n", + delimiters.1, + "\n\n", + "After $\\alpha$.\n", + ] { + source.push_str(chunk); + render.append( + &source, + chunk, + Some(40), + &cwd, + HistoryRenderMode::Rich, + /*inline_visualization_context*/ None, + ); + assert_eq!( + render.lines, + render_streaming_markdown_agent_with_links_and_cwd(&source, Some(40), Some(&cwd)) + .lines + ); + for width in [16, 40] { + let mut resized = StreamingRender::new(); + resized.recompute( + &source, + Some(width), + &cwd, + HistoryRenderMode::Rich, + /*inline_visualization_context*/ None, + ); + assert_eq!( + resized.lines, + render_streaming_markdown_agent_with_links_and_cwd( + &source, + Some(width), + Some(&cwd) + ) + .lines + ); + } + } + } +} diff --git a/codex-rs/tui/src/streaming/render.rs b/codex-rs/tui/src/streaming/render.rs index 9b0d289a33..1996332fff 100644 --- a/codex-rs/tui/src/streaming/render.rs +++ b/codex-rs/tui/src/streaming/render.rs @@ -218,3 +218,7 @@ pub(super) fn render_source( #[cfg(test)] #[path = "render_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "inline_math_tests.rs"] +mod inline_math_tests;