mirror of
https://github.com/openai/codex.git
synced 2026-10-21 05:22:21 +00:00
Render inline TeX math as Unicode in the TUI (#45501)
## What changed
Convert supported expressions inside `$...$` and `\(...\)` to readable Unicode in regular and streaming Markdown. For example, `$\alpha^2 + \beta_{10}$` renders as `α² + β₁₀`. Support a bounded TeX subset including symbols, superscripts, subscripts, square roots, and parenthesized fractions, with verbatim fallback for unsupported expressions.
Exclude code, links, HTML, and display equations from conversion, and avoid interpreting common currency and shell syntax as math. Preserve file citation paths and keep streaming cache boundaries outside display equations.
## Testing
Add snapshots for inline rendering and narrow wrapping, plus regression tests for unsupported input, parser limits, Markdown contexts, file citations, and streaming display boundaries across chunks and widths.
GitOrigin-RevId: 828bf7b48345b05bc6c2fb5cb2798412c3b71dbe
This commit is contained in:
Eric Traut
@@ -70,6 +70,7 @@ use std::path::PathBuf;
|
||||
|
||||
mod file_citations;
|
||||
mod local_links;
|
||||
mod math;
|
||||
mod streaming;
|
||||
mod table_key_value;
|
||||
mod web_links;
|
||||
@@ -338,7 +339,10 @@ pub(crate) fn render_markdown_lines_with_width_cwd_and_hidden_link_destinations(
|
||||
let mut options = Options::empty();
|
||||
options.insert(Options::ENABLE_STRIKETHROUGH);
|
||||
options.insert(Options::ENABLE_TABLES);
|
||||
let parser = DecodedTextMerge::new(Parser::new_ext(input, options).into_offset_iter());
|
||||
let math = math::MathMarkdown::new(input, options);
|
||||
let parser = DecodedTextMerge::new(
|
||||
math.events(Parser::new_ext(&math.markdown, options).into_offset_iter()),
|
||||
);
|
||||
let mut w = Writer::new(input, parser, width, cwd, is_hidden_link_destination);
|
||||
w.run();
|
||||
w.text
|
||||
|
||||
@@ -110,11 +110,11 @@ impl<'a> FileCitations<'a> {
|
||||
/// Adapt before `DecodedTextMerge`, while plain text still has exact source offsets.
|
||||
pub(super) fn events<'s>(
|
||||
&'s self,
|
||||
parser: Parser<'s>,
|
||||
events: impl Iterator<Item = (Event<'s>, Range<usize>)>,
|
||||
cwd: Option<&'s Path>,
|
||||
) -> impl Iterator<Item = (Event<'s>, Range<usize>)> {
|
||||
let mut citations = self.citations.iter().peekable();
|
||||
parser.into_offset_iter().flat_map(move |(event, range)| {
|
||||
events.flat_map(move |(event, range)| {
|
||||
while citations
|
||||
.next_if(|(span, _)| span.end <= range.start)
|
||||
.is_some()
|
||||
|
||||
@@ -17,6 +17,7 @@ fn rendered_text(markdown: &str, cwd: Option<&Path>) -> String {
|
||||
fn file_citation_paths_preserve_markdown_significant_characters() {
|
||||
for path in [
|
||||
"/tmp/a*b*.txt",
|
||||
"/tmp/$x$/report.md",
|
||||
"/tmp/a`b`.txt",
|
||||
"/tmp/a<b>.txt",
|
||||
"/tmp/report#L10",
|
||||
@@ -158,6 +159,13 @@ fn multiple_file_citations_render_without_interpreting_encoded_source() {
|
||||
|
||||
#[test]
|
||||
fn file_citations_preserve_adjacent_entities_and_escaped_punctuation() {
|
||||
assert_eq!(
|
||||
rendered_text(
|
||||
r"$\alpha$:codex-file-citation{path=/tmp/$x$/report.md}$\beta$",
|
||||
/*cwd*/ None
|
||||
),
|
||||
"α/tmp/$x$/report.mdβ",
|
||||
);
|
||||
let cwd = std::env::temp_dir();
|
||||
for (markdown, expected) in [
|
||||
(
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
//! Recognize math before Markdown consumes TeX escapes, retaining exact source offsets.
|
||||
//!
|
||||
//! Only the rendering copy is masked. Unsupported expressions are restored verbatim; code,
|
||||
//! links, HTML, and display equations stay under the ordinary Markdown renderer.
|
||||
|
||||
use itertools::Either;
|
||||
use pulldown_cmark::Event;
|
||||
use pulldown_cmark::Options;
|
||||
use pulldown_cmark::Parser;
|
||||
use pulldown_cmark::Tag;
|
||||
use std::borrow::Cow;
|
||||
use std::ops::Range;
|
||||
|
||||
mod render;
|
||||
|
||||
const MAX_MATH_BYTES: usize = 4096;
|
||||
|
||||
pub(super) struct MathMarkdown<'a> {
|
||||
pub(super) markdown: Cow<'a, str>,
|
||||
pub(super) display_ranges: Vec<Range<usize>>,
|
||||
replacements: Vec<(Range<usize>, String)>,
|
||||
}
|
||||
|
||||
impl<'a> MathMarkdown<'a> {
|
||||
pub(super) fn new(input: &'a str, options: Options) -> Self {
|
||||
let mut result = Self {
|
||||
markdown: Cow::Borrowed(input),
|
||||
display_ranges: Vec::new(),
|
||||
replacements: Vec::new(),
|
||||
};
|
||||
if !input.contains('$') && !input.contains("\\(") && !input.contains("\\[") {
|
||||
return result;
|
||||
}
|
||||
let parser = Parser::new_ext(input, options);
|
||||
let mut protected: Vec<_> = parser
|
||||
.reference_definitions()
|
||||
.iter()
|
||||
.map(|(_, def)| def.span.clone())
|
||||
.collect();
|
||||
protected.extend(parser.into_offset_iter().filter_map(|(event, range)| {
|
||||
matches!(
|
||||
event,
|
||||
Event::Code(_)
|
||||
| Event::Html(_)
|
||||
| Event::InlineHtml(_)
|
||||
| Event::Start(Tag::CodeBlock(_) | Tag::Link { .. } | Tag::Image { .. })
|
||||
)
|
||||
.then_some(range)
|
||||
}));
|
||||
protected.sort_unstable_by_key(|range| range.start);
|
||||
let mut protected = protected.into_iter().peekable();
|
||||
let mut offset = 0;
|
||||
let mut scanned = 0;
|
||||
let mut line_start = 0;
|
||||
while offset < input.len() {
|
||||
if let Some(index) = input[scanned..offset].rfind('\n') {
|
||||
line_start = scanned + index + 1;
|
||||
}
|
||||
scanned = offset;
|
||||
while protected.next_if(|range| range.end <= offset).is_some() {}
|
||||
if let Some(range) = protected.peek()
|
||||
&& range.contains(&offset)
|
||||
{
|
||||
offset = range.end;
|
||||
continue;
|
||||
}
|
||||
let rest = &input[offset..];
|
||||
let (open, close, display) = if rest.starts_with("$$") {
|
||||
("$$", "$$", true)
|
||||
} else if rest.starts_with("\\[") {
|
||||
("\\[", "\\]", true)
|
||||
} else if rest.starts_with("\\(") {
|
||||
("\\(", "\\)", false)
|
||||
} else if rest.starts_with('$') {
|
||||
("$", "$", false)
|
||||
} else {
|
||||
let Some(ch) = rest.chars().next() else {
|
||||
break;
|
||||
};
|
||||
offset += ch.len_utf8();
|
||||
continue;
|
||||
};
|
||||
let start = offset;
|
||||
offset += open.len();
|
||||
if escaped(input, start) {
|
||||
continue;
|
||||
}
|
||||
let body = &input[offset..];
|
||||
if open == "$"
|
||||
&& (body.starts_with(char::is_whitespace) || body.starts_with(['(', '{']))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let limit = body
|
||||
.char_indices()
|
||||
.map(|(i, _)| i)
|
||||
.find(|i| *i >= MAX_MATH_BYTES)
|
||||
.unwrap_or(body.len());
|
||||
// Display boundaries outlive the conversion budget; a distant closer still owns its opener.
|
||||
let search = if display { body } else { &body[..limit] };
|
||||
let end = search.match_indices(close).find_map(|(index, _)| {
|
||||
let end = offset + index;
|
||||
(!escaped(input, end)).then_some(end)
|
||||
});
|
||||
if display {
|
||||
if open == "$$" && end.is_none() && !input[line_start..start].trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Exclude display equations from inline recognition, including unfinished ones.
|
||||
offset = end.map_or(input.len(), |end| end + close.len());
|
||||
result.display_ranges.push(start..offset);
|
||||
continue;
|
||||
}
|
||||
let Some(end) = end else {
|
||||
continue;
|
||||
};
|
||||
let span = start..end + close.len();
|
||||
if protected.peek().is_some_and(|range| range.start < span.end) {
|
||||
continue;
|
||||
}
|
||||
let formula = &input[offset..end];
|
||||
if formula.contains('\n') {
|
||||
continue;
|
||||
}
|
||||
if open == "$" {
|
||||
let next = input[span.end..].chars().next();
|
||||
if formula.ends_with(char::is_whitespace) || next.is_some_and(char::is_alphanumeric)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if formula.starts_with(|ch: char| ch.is_ascii_digit())
|
||||
&& !formula.contains(['\\', '^', '_', '=', '+', '-', '*', '/', '<', '>'])
|
||||
|| formula.len() > 1 && formula.chars().all(|ch| ch.is_ascii_uppercase())
|
||||
{
|
||||
offset = span.end;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let rendered =
|
||||
render::render(formula).unwrap_or_else(|| input[span.clone()].to_owned());
|
||||
// Dollars are ordinary text in the Markdown parser and cannot form an HTML tag.
|
||||
result
|
||||
.markdown
|
||||
.to_mut()
|
||||
.replace_range(span.clone(), &"$".repeat(span.len()));
|
||||
offset = span.end;
|
||||
result.replacements.push((span, rendered));
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) fn events<'s>(
|
||||
&'s self,
|
||||
events: impl Iterator<Item = (Event<'s>, Range<usize>)>,
|
||||
) -> impl Iterator<Item = (Event<'s>, Range<usize>)> {
|
||||
let mut replacements = self.replacements.iter().peekable();
|
||||
events.flat_map(move |(event, range)| {
|
||||
while replacements
|
||||
.next_if(|(span, _)| span.end <= range.start)
|
||||
.is_some()
|
||||
{}
|
||||
let Event::Text(text) = event else {
|
||||
return Either::Left(std::iter::once((event, range)));
|
||||
};
|
||||
if replacements
|
||||
.peek()
|
||||
.is_none_or(|(span, _)| span.start >= range.end)
|
||||
{
|
||||
return Either::Left(std::iter::once((Event::Text(text), range)));
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let mut offset = range.start;
|
||||
while let Some((span, text)) = replacements.next_if(|(span, _)| span.end <= range.end) {
|
||||
if offset < span.start {
|
||||
output.push((
|
||||
Event::Text(self.markdown[offset..span.start].into()),
|
||||
offset..span.start,
|
||||
));
|
||||
}
|
||||
output.push((Event::Text(text.as_str().into()), span.clone()));
|
||||
offset = span.end;
|
||||
}
|
||||
if offset < range.end {
|
||||
output.push((
|
||||
Event::Text(self.markdown[offset..range.end].into()),
|
||||
offset..range.end,
|
||||
));
|
||||
}
|
||||
Either::Right(output.into_iter())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn escaped(input: &str, offset: usize) -> bool {
|
||||
input[..offset]
|
||||
.bytes()
|
||||
.rev()
|
||||
.take_while(|byte| *byte == b'\\')
|
||||
.count()
|
||||
% 2
|
||||
== 1
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "math_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,265 @@
|
||||
//! Bounded Unicode layout for a deliberately small TeX math subset.
|
||||
|
||||
use crate::width::display_width;
|
||||
|
||||
pub(super) fn render(source: &str) -> Option<String> {
|
||||
let mut parser = MathParser {
|
||||
remaining: source.trim(),
|
||||
depth: 0,
|
||||
};
|
||||
let result = parser.sequence(/*group*/ false)?;
|
||||
(!result.trim().is_empty()).then_some(result)
|
||||
}
|
||||
|
||||
struct MathParser<'a> {
|
||||
remaining: &'a str,
|
||||
depth: usize,
|
||||
}
|
||||
|
||||
impl MathParser<'_> {
|
||||
fn take(&mut self) -> Option<char> {
|
||||
let ch = self.remaining.chars().next()?;
|
||||
self.remaining = &self.remaining[ch.len_utf8()..];
|
||||
Some(ch)
|
||||
}
|
||||
|
||||
fn sequence(&mut self, group: bool) -> Option<String> {
|
||||
if self.depth >= 32 {
|
||||
return None;
|
||||
}
|
||||
self.depth += 1;
|
||||
let mut result = String::new();
|
||||
let mut scripts = 0;
|
||||
let mut has_base = false;
|
||||
while let Some(ch) = self.remaining.chars().next() {
|
||||
if ch == '}' {
|
||||
if !group {
|
||||
return None;
|
||||
}
|
||||
self.take();
|
||||
self.depth -= 1;
|
||||
return Some(result);
|
||||
}
|
||||
if ch == '^' || ch == '_' {
|
||||
let script = if ch == '^' { 1 } else { 2 };
|
||||
if !has_base || scripts & script != 0 {
|
||||
return None;
|
||||
}
|
||||
scripts |= script;
|
||||
} else if !ch.is_whitespace() {
|
||||
scripts = 0;
|
||||
}
|
||||
let atom = self.atom()?;
|
||||
if !ch.is_whitespace() && ch != '^' && ch != '_' {
|
||||
// Flattening a compound base would change the scope of a following script.
|
||||
has_base = atom.chars().count() == 1;
|
||||
}
|
||||
result.push_str(&atom);
|
||||
if display_width(&result) > 256 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
self.depth -= 1;
|
||||
(!group).then_some(result)
|
||||
}
|
||||
|
||||
fn argument(&mut self) -> Option<String> {
|
||||
self.remaining = self.remaining.trim_start();
|
||||
if self.remaining.starts_with(['^', '_']) {
|
||||
return None;
|
||||
}
|
||||
self.atom()
|
||||
}
|
||||
|
||||
fn atom(&mut self) -> Option<String> {
|
||||
if self.depth >= 32 {
|
||||
return None;
|
||||
}
|
||||
self.depth += 1;
|
||||
let result = self.atom_inner();
|
||||
self.depth -= 1;
|
||||
result
|
||||
}
|
||||
|
||||
fn atom_inner(&mut self) -> Option<String> {
|
||||
let ch = self.take()?;
|
||||
match ch {
|
||||
'{' => self.sequence(/*group*/ true),
|
||||
'\\' => self.command(),
|
||||
'^' | '_' => {
|
||||
let arg = self.argument()?;
|
||||
let (plain, alphabet) = if ch == '^' {
|
||||
(
|
||||
"0123456789+-=()abcdefghijklmnoprstuvwxyz",
|
||||
"⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖʳˢᵗᵘᵛʷˣʸᶻ",
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"0123456789+-=()aehijklmnoprstuvx",
|
||||
"₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₕᵢⱼₖₗₘₙₒₚᵣₛₜᵤᵥₓ",
|
||||
)
|
||||
};
|
||||
let mut output = String::new();
|
||||
for value in arg.chars() {
|
||||
let index = plain.chars().position(|ch| ch == value)?;
|
||||
output.push(alphabet.chars().nth(index)?);
|
||||
}
|
||||
Some(output)
|
||||
}
|
||||
'}' | '$' | '%' | '#' | '&' | '`' => None,
|
||||
ch if ch.is_whitespace() => {
|
||||
self.remaining = self.remaining.trim_start();
|
||||
Some(String::from(" "))
|
||||
}
|
||||
ch if ch.is_control() => None,
|
||||
ch => Some(ch.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn command(&mut self) -> Option<String> {
|
||||
let length = self
|
||||
.remaining
|
||||
.bytes()
|
||||
.take_while(u8::is_ascii_alphabetic)
|
||||
.count();
|
||||
if length == 0 {
|
||||
return match self.take()? {
|
||||
',' | ';' | ':' | ' ' => Some(String::from(" ")),
|
||||
'!' => Some(String::new()),
|
||||
'{' => Some(String::from("{")),
|
||||
'}' => Some(String::from("}")),
|
||||
'|' => Some(String::from("‖")),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
let name = &self.remaining[..length];
|
||||
self.remaining = &self.remaining[length..];
|
||||
match name {
|
||||
"frac" | "dfrac" | "tfrac" => {
|
||||
let numerator = self.argument()?;
|
||||
let denominator = self.argument()?;
|
||||
Some(format!("(({numerator})/({denominator}))"))
|
||||
}
|
||||
"sqrt" => {
|
||||
if self.remaining.trim_start().starts_with('[') {
|
||||
return None;
|
||||
}
|
||||
let radicand = self.argument()?;
|
||||
Some(format!("√({radicand})"))
|
||||
}
|
||||
"mathbb" => {
|
||||
let arg = self.argument()?;
|
||||
let text = match arg.as_str() {
|
||||
"R" => "ℝ",
|
||||
"C" => "ℂ",
|
||||
"N" => "ℕ",
|
||||
"Z" => "ℤ",
|
||||
"Q" => "ℚ",
|
||||
"P" => "ℙ",
|
||||
_ => return None,
|
||||
};
|
||||
Some(String::from(text))
|
||||
}
|
||||
"mathrm" | "mathbf" | "mathit" => self.argument(),
|
||||
"text" | "operatorname" => {
|
||||
self.remaining = self.remaining.trim_start().strip_prefix('{')?;
|
||||
let end = self.remaining.find('}')?;
|
||||
let text = &self.remaining[..end];
|
||||
if text.contains(['{', '\\', '$', '%', '#', '&'])
|
||||
|| text.chars().any(char::is_control)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.remaining = &self.remaining[end + 1..];
|
||||
Some(String::from(text))
|
||||
}
|
||||
"left" | "right" => {
|
||||
self.remaining = self.remaining.trim_start();
|
||||
match self.take()? {
|
||||
'.' => Some(String::new()),
|
||||
ch @ ('(' | ')' | '[' | ']' | '|') => Some(ch.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
"quad" | "qquad" => Some(String::from(" ")),
|
||||
"sin" | "cos" | "tan" | "log" | "ln" | "exp" | "lim" | "max" | "min" => {
|
||||
Some(String::from(name))
|
||||
}
|
||||
_ => symbol(name).map(String::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn symbol(name: &str) -> Option<&'static str> {
|
||||
Some(match name {
|
||||
"alpha" => "α",
|
||||
"beta" => "β",
|
||||
"gamma" => "γ",
|
||||
"delta" => "δ",
|
||||
"epsilon" => "ϵ",
|
||||
"varepsilon" => "ε",
|
||||
"zeta" => "ζ",
|
||||
"eta" => "η",
|
||||
"theta" => "θ",
|
||||
"vartheta" => "ϑ",
|
||||
"iota" => "ι",
|
||||
"kappa" => "κ",
|
||||
"lambda" => "λ",
|
||||
"mu" => "μ",
|
||||
"nu" => "ν",
|
||||
"xi" => "ξ",
|
||||
"pi" => "π",
|
||||
"rho" => "ρ",
|
||||
"sigma" => "σ",
|
||||
"tau" => "τ",
|
||||
"upsilon" => "υ",
|
||||
"phi" => "ϕ",
|
||||
"varphi" => "φ",
|
||||
"chi" => "χ",
|
||||
"psi" => "ψ",
|
||||
"omega" => "ω",
|
||||
"Gamma" => "Γ",
|
||||
"Delta" => "Δ",
|
||||
"Theta" => "Θ",
|
||||
"Lambda" => "Λ",
|
||||
"Xi" => "Ξ",
|
||||
"Pi" => "Π",
|
||||
"Sigma" => "Σ",
|
||||
"Upsilon" => "Υ",
|
||||
"Phi" => "Φ",
|
||||
"Psi" => "Ψ",
|
||||
"Omega" => "Ω",
|
||||
"sum" => "∑",
|
||||
"prod" => "∏",
|
||||
"int" => "∫",
|
||||
"infty" => "∞",
|
||||
"partial" => "∂",
|
||||
"nabla" => "∇",
|
||||
"pm" => "±",
|
||||
"mp" => "∓",
|
||||
"times" => "×",
|
||||
"cdot" => "·",
|
||||
"div" => "÷",
|
||||
"le" | "leq" => "≤",
|
||||
"ge" | "geq" => "≥",
|
||||
"ne" | "neq" => "≠",
|
||||
"approx" => "≈",
|
||||
"equiv" => "≡",
|
||||
"in" => "∈",
|
||||
"notin" => "∉", // codespell:ignore notin
|
||||
"subset" => "⊂",
|
||||
"subseteq" => "⊆",
|
||||
"cup" => "∪",
|
||||
"cap" => "∩",
|
||||
"emptyset" => "∅",
|
||||
"forall" => "∀",
|
||||
"exists" => "∃",
|
||||
"to" | "rightarrow" => "→",
|
||||
"leftarrow" => "←",
|
||||
"Rightarrow" => "⇒",
|
||||
"Leftrightarrow" => "⇔",
|
||||
"ldots" | "dots" => "…",
|
||||
"cdots" => "⋯",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
use super::render::render;
|
||||
use crate::markdown_render::render_markdown_text_with_width;
|
||||
use itertools::Itertools;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
fn plain(source: &str, width: usize) -> String {
|
||||
render_markdown_text_with_width(source, Some(width))
|
||||
.lines
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_math_inline_snapshot() {
|
||||
insta::assert_snapshot!(plain(
|
||||
r"Inline $\alpha^2 + \beta_{10}$ on $\mathbb{R}^n$.
|
||||
Root $\sqrt{x^2+y^2}$ and fraction $a/\frac{b}{c}$.
|
||||
Unsupported: $\unknown{x_y}$ and $\frac{a}{b}^2$.
|
||||
Angles: <$x$>, <\(\alpha\)>, and <$\unknown{x}$>.
|
||||
Code: `$\alpha$`; money: $5 and $10; shell: $HOME.",
|
||||
/*width*/ 80
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_math_inline_narrow_snapshot() {
|
||||
insta::assert_snapshot!(plain(
|
||||
r"Words $\alpha^2 + \beta_1$ more words and $\sqrt{x}$.",
|
||||
/*width*/ 16
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_math_preserves_markdown_contexts() {
|
||||
for (source, expected) in [
|
||||
(
|
||||
r"**$\alpha_1$** & \$5.00 and $\beta^2$",
|
||||
"α₁ & $5.00 and β²",
|
||||
),
|
||||
(
|
||||
r"`$\alpha$` $HOME ${HOME} $(echo x) $5 and $10",
|
||||
r"$\alpha$ $HOME ${HOME} $(echo x) $5 and $10",
|
||||
),
|
||||
(
|
||||
r"[x](https://example.com/$HOME) and $\alpha$",
|
||||
"x (https://example.com/$HOME) and α",
|
||||
),
|
||||
(r"$\left. x \right|$ $\text{x_y^z}$", " x | x_y^z"),
|
||||
(r"\(\alpha^2\)", "α²"),
|
||||
(r"Value: <$x$> and <\(\alpha\)>.", "Value: <x> and <α>."),
|
||||
(r"$USD$+$\alpha$", "$USD$+α"),
|
||||
(r"Costs $5; $\alpha$ and $x^2$.", "Costs $5; α and x²."),
|
||||
(r"$HOME then $\alpha$", "$HOME then α"),
|
||||
(
|
||||
r"Run echo $$ to print PID. Then $\alpha$",
|
||||
"Run echo $$ to print PID. Then α",
|
||||
),
|
||||
] {
|
||||
assert_eq!(plain(source, /*width*/ 80), expected);
|
||||
}
|
||||
// This stage leaves display equations and TeX fences to ordinary Markdown rendering.
|
||||
for source in [r"$$x^2$$", "$$\nx^2\n$$", r"\[x^2\]", "```latex\nx^2\n```"] {
|
||||
assert!(!plain(source, /*width*/ 80).contains('²'));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_math_bounds_and_unsupported_input() {
|
||||
for source in [
|
||||
r"\unknown{x}",
|
||||
r"\frac{a}",
|
||||
"{x",
|
||||
"x}",
|
||||
"x^{q}",
|
||||
"^2",
|
||||
"x^2^3",
|
||||
"{a+b}^2",
|
||||
"{x^2}^3",
|
||||
r"\frac{a}{b}^2",
|
||||
r"\sqrt[3]{x}",
|
||||
r"\sqrt [3]{x}",
|
||||
r"\left x",
|
||||
r"\text{\alpha}",
|
||||
r"\begin{matrix}a&b\end{matrix}",
|
||||
] {
|
||||
assert_eq!(render(source), None, "{source}");
|
||||
assert_eq!(
|
||||
plain(&format!("\\({source}\\)"), /*width*/ 80),
|
||||
format!("\\({source}\\)")
|
||||
);
|
||||
}
|
||||
for source in [
|
||||
format!("{}x{}", "{".repeat(/*n*/ 40), "}".repeat(/*n*/ 40)),
|
||||
"x".repeat(/*n*/ 300),
|
||||
format!("{}x", "\\sqrt".repeat(/*n*/ 100)),
|
||||
] {
|
||||
assert_eq!(render(&source), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_math_oversized_display_does_not_hide_following_inline_math() {
|
||||
for (open, close) in [("$$", "$$"), (r"\[", r"\]")] {
|
||||
let source = format!(
|
||||
"{open}\n{}\n{close}\n\nAfter $\\alpha$.\n",
|
||||
"x".repeat(/*n*/ 5000)
|
||||
);
|
||||
assert!(plain(&source, /*width*/ 80).ends_with("After α."));
|
||||
}
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
---
|
||||
source: tui/src/markdown_render/math_tests.rs
|
||||
expression: "plain(r\"Words $\\alpha^2 + \\beta_1$ more words and $\\sqrt{x}$.\", 16)"
|
||||
---
|
||||
Words α² + β₁
|
||||
more words and
|
||||
√(x).
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
---
|
||||
source: tui/src/markdown_render/math_tests.rs
|
||||
assertion_line: 16
|
||||
expression: "plain(r\"Inline $\\alpha^2 + \\beta_{10}$ on $\\mathbb{R}^n$.\nRoot $\\sqrt{x^2+y^2}$ and fraction $a/\\frac{b}{c}$.\nUnsupported: $\\unknown{x_y}$ and $\\frac{a}{b}^2$.\nAngles: <$x$>, <\\(\\alpha\\)>, and <$\\unknown{x}$>.\nCode: `$\\alpha$`; money: $5 and $10; shell: $HOME.\",\n80)"
|
||||
---
|
||||
Inline α² + β₁₀ on ℝⁿ.
|
||||
Root √(x²+y²) and fraction a/((b)/(c)).
|
||||
Unsupported: $\unknown{x_y}$ and $\frac{a}{b}^2$.
|
||||
Angles: <x>, <α>, and <$\unknown{x}$>.
|
||||
Code: $\alpha$; money: $5 and $10; shell: $HOME.
|
||||
@@ -11,6 +11,7 @@ use super::Options;
|
||||
use super::Parser;
|
||||
use super::Tag;
|
||||
use super::Writer;
|
||||
use super::math::MathMarkdown;
|
||||
use std::ops::Range;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -40,10 +41,11 @@ pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd(
|
||||
options.insert(Options::ENABLE_STRIKETHROUGH);
|
||||
options.insert(Options::ENABLE_TABLES);
|
||||
let citations = FileCitations::new(input, options);
|
||||
let parser = Parser::new_ext(&citations.markdown, options);
|
||||
let math = MathMarkdown::new(&citations.markdown, options);
|
||||
let parser = Parser::new_ext(&math.markdown, options);
|
||||
let has_reference_link_definition = parser.reference_definitions().iter().next().is_some();
|
||||
let parser = TopLevelBlockTracker {
|
||||
iter: DecodedTextMerge::new(citations.events(parser, cwd)),
|
||||
iter: DecodedTextMerge::new(citations.events(math.events(parser.into_offset_iter()), cwd)),
|
||||
depth: 0,
|
||||
block_count: 0,
|
||||
last_start: 0,
|
||||
@@ -53,7 +55,15 @@ pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd(
|
||||
writer.run();
|
||||
StreamingMarkdownRender {
|
||||
lines: writer.text,
|
||||
last_top_level_block_start: (writer.iter.block_count > 1).then_some(writer.iter.last_start),
|
||||
last_top_level_block_start: (writer.iter.block_count > 1)
|
||||
.then_some(writer.iter.last_start)
|
||||
// A cached suffix must not mistake a display closer for a new opener.
|
||||
.filter(|boundary| {
|
||||
!math
|
||||
.display_ranges
|
||||
.iter()
|
||||
.any(|range| range.start < *boundary && *boundary < range.end)
|
||||
}),
|
||||
has_reference_link_definition,
|
||||
first_top_level_block_is_html: writer.iter.first_is_html,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use super::StreamingRender;
|
||||
use crate::history_cell::HistoryRenderMode;
|
||||
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn unicode_math_inline_stream_preserves_display_exclusion_boundaries() {
|
||||
let cwd = std::env::temp_dir();
|
||||
for delimiters in [("$$", "$$"), ("\\[", "\\]")] {
|
||||
let mut render = StreamingRender::new();
|
||||
let mut source = String::new();
|
||||
for chunk in [
|
||||
delimiters.0,
|
||||
"\n\nx^2\n\n",
|
||||
delimiters.1,
|
||||
"\n\n",
|
||||
"After $\\alpha$.\n",
|
||||
] {
|
||||
source.push_str(chunk);
|
||||
render.append(
|
||||
&source,
|
||||
chunk,
|
||||
Some(40),
|
||||
&cwd,
|
||||
HistoryRenderMode::Rich,
|
||||
/*inline_visualization_context*/ None,
|
||||
);
|
||||
assert_eq!(
|
||||
render.lines,
|
||||
render_streaming_markdown_agent_with_links_and_cwd(&source, Some(40), Some(&cwd))
|
||||
.lines
|
||||
);
|
||||
for width in [16, 40] {
|
||||
let mut resized = StreamingRender::new();
|
||||
resized.recompute(
|
||||
&source,
|
||||
Some(width),
|
||||
&cwd,
|
||||
HistoryRenderMode::Rich,
|
||||
/*inline_visualization_context*/ None,
|
||||
);
|
||||
assert_eq!(
|
||||
resized.lines,
|
||||
render_streaming_markdown_agent_with_links_and_cwd(
|
||||
&source,
|
||||
Some(width),
|
||||
Some(&cwd)
|
||||
)
|
||||
.lines
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -218,3 +218,7 @@ pub(super) fn render_source(
|
||||
#[cfg(test)]
|
||||
#[path = "render_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "inline_math_tests.rs"]
|
||||
mod inline_math_tests;
|
||||
|
||||
Reference in New Issue
Block a user