mirror of
https://github.com/openai/codex.git
synced 2026-11-18 15:01:48 +00:00
Avoid rerendering streamed code fences (#39061)
## Why The streaming TUI rerendered the entire mutable Markdown block whenever a newline-bearing delta arrived. For long fenced code blocks, this repeatedly parsed and syntax-highlighted the growing fence. ## What changed - Detect open, top-level, language-tagged code fences and append newly completed lines directly to the existing render. - Preserve Syntect parser state across appended lines so multiline syntax and the active theme remain consistent. - Fall back to the canonical Markdown renderer for ambiguous fence syntax, theme changes, incomplete lines, and highlighting limit transitions. ## Testing Added coverage that compares streamed output with full renders across syntax styles, Unicode, links, blank lines, fence variants, theme changes, and highlighting limits. A long-fence test also verifies that the rendered prefix is retained as new lines arrive. GitOrigin-RevId: 036db2b7bc93607083b8001f8b4fa3cbb58c713e
This commit is contained in:
1 parent
80e925c28b
commit
386a7b629c
9 files changed
+583
-14
No files matched your search
@@ -47,6 +47,11 @@ use syntect::parsing::SyntaxSet;
|
|||||||
use syntect::util::LinesWithEndings;
|
use syntect::util::LinesWithEndings;
|
||||||
use two_face::theme::EmbeddedThemeName;
|
use two_face::theme::EmbeddedThemeName;
|
||||||
|
|
||||||
|
#[path = "highlight_streaming.rs"]
|
||||||
|
mod streaming;
|
||||||
|
|
||||||
|
pub(crate) use streaming::StreamingCodeHighlighter;
|
||||||
|
|
||||||
// -- Global singletons -------------------------------------------------------
|
// -- Global singletons -------------------------------------------------------
|
||||||
|
|
||||||
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
|
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
|
||||||
@@ -630,25 +635,30 @@ fn highlight_to_line_spans_with_theme(
|
|||||||
|
|
||||||
for line in LinesWithEndings::from(code) {
|
for line in LinesWithEndings::from(code) {
|
||||||
let ranges = h.highlight_line(line, syntax_set()).ok()?;
|
let ranges = h.highlight_line(line, syntax_set()).ok()?;
|
||||||
let mut spans: Vec<Span<'static>> = Vec::new();
|
lines.push(highlighted_line_spans(ranges));
|
||||||
for (style, text) in ranges {
|
|
||||||
// Strip trailing line endings (LF and CR) since we handle line
|
|
||||||
// breaks ourselves. CRLF inputs would otherwise leave a stray \r.
|
|
||||||
let text = text.trim_end_matches(['\n', '\r']);
|
|
||||||
if text.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
spans.push(Span::styled(text.to_string(), convert_style(style)));
|
|
||||||
}
|
|
||||||
if spans.is_empty() {
|
|
||||||
spans.push(Span::raw(String::new()));
|
|
||||||
}
|
|
||||||
lines.push(spans);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Some(lines)
|
Some(lines)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convert one highlighted line into the spans shared by full and streaming renders.
|
||||||
|
///
|
||||||
|
/// Source line endings are omitted, while empty lines retain their canonical empty span.
|
||||||
|
fn highlighted_line_spans(ranges: Vec<(SyntectStyle, &str)>) -> Vec<Span<'static>> {
|
||||||
|
let mut spans = Vec::new();
|
||||||
|
for (style, text) in ranges {
|
||||||
|
// Line breaks are represented by the surrounding Line, not its spans.
|
||||||
|
let text = text.trim_end_matches(['\n', '\r']);
|
||||||
|
if !text.is_empty() {
|
||||||
|
spans.push(Span::styled(text.to_string(), convert_style(style)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if spans.is_empty() {
|
||||||
|
spans.push(Span::raw(String::new()));
|
||||||
|
}
|
||||||
|
spans
|
||||||
|
}
|
||||||
|
|
||||||
/// Parse `code` using syntect for `lang` and return per-line styled spans.
|
/// Parse `code` using syntect for `lang` and return per-line styled spans.
|
||||||
/// Each inner Vec represents one source line. Returns None when the language
|
/// Each inner Vec represents one source line. Returns None when the language
|
||||||
/// is not recognized or the input exceeds safety limits.
|
/// is not recognized or the input exceeds safety limits.
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
//! Append-only syntax highlighting for complete lines in an open code fence.
|
||||||
|
|
||||||
|
use super::MAX_HIGHLIGHT_LINE_BYTES;
|
||||||
|
use super::exceeds_highlight_limits;
|
||||||
|
use super::find_syntax;
|
||||||
|
use super::highlighted_line_spans;
|
||||||
|
use super::syntax_set;
|
||||||
|
use super::syntax_theme_revision;
|
||||||
|
use super::theme_lock;
|
||||||
|
use ratatui::text::Line;
|
||||||
|
use syntect::easy::HighlightLines;
|
||||||
|
use syntect::highlighting::HighlightState;
|
||||||
|
use syntect::parsing::ParseState;
|
||||||
|
use syntect::util::LinesWithEndings;
|
||||||
|
|
||||||
|
/// Retains only Syntect's parser state, subject to the ordinary highlighting limits.
|
||||||
|
///
|
||||||
|
/// An absent state means the whole fence already uses the permanent plain-text fallback because
|
||||||
|
/// its language is unknown or a size limit was exceeded. The caller owns the rendered prefix.
|
||||||
|
pub(crate) struct StreamingCodeHighlighter {
|
||||||
|
state: Option<HighlightedCode>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct HighlightedCode {
|
||||||
|
bytes: usize,
|
||||||
|
lines: usize,
|
||||||
|
theme_revision: u64,
|
||||||
|
syntax: (HighlightState, ParseState),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamingCodeHighlighter {
|
||||||
|
/// Reconstruct highlighting state after the canonical renderer emitted `code`.
|
||||||
|
///
|
||||||
|
/// Unknown languages and oversized input retain a permanent plain-text fallback. A changed
|
||||||
|
/// theme or failed replay returns `None`, requiring the caller to render the whole fence again.
|
||||||
|
pub(crate) fn new(code: &str, lang: &str, theme_revision: u64) -> Option<Self> {
|
||||||
|
let theme = match theme_lock().read() {
|
||||||
|
Ok(theme) => theme,
|
||||||
|
Err(poisoned) => poisoned.into_inner(),
|
||||||
|
};
|
||||||
|
if theme_revision != syntax_theme_revision() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let lines = code.lines().count();
|
||||||
|
let syntax = find_syntax(lang).filter(|_| {
|
||||||
|
!exceeds_highlight_limits(code.len(), lines)
|
||||||
|
&& !code
|
||||||
|
.lines()
|
||||||
|
.any(|line| line.len() > MAX_HIGHLIGHT_LINE_BYTES)
|
||||||
|
});
|
||||||
|
let Some(syntax) = syntax else {
|
||||||
|
return Some(Self { state: None });
|
||||||
|
};
|
||||||
|
let mut highlighter = HighlightLines::new(syntax, &theme);
|
||||||
|
for line in LinesWithEndings::from(code) {
|
||||||
|
highlighter.highlight_line(line, syntax_set()).ok()?;
|
||||||
|
}
|
||||||
|
Some(Self {
|
||||||
|
state: Some(HighlightedCode {
|
||||||
|
bytes: code.len(),
|
||||||
|
lines,
|
||||||
|
theme_revision,
|
||||||
|
syntax: highlighter.state(),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Consume the previous state and highlight just the newly committed complete lines.
|
||||||
|
///
|
||||||
|
/// Every appended line must be newline-terminated to preserve Syntect's parser state.
|
||||||
|
/// Returning `None` discards the state: the caller must render the whole fence again, which
|
||||||
|
/// also removes old colors when the aggregate input crosses a highlighting limit.
|
||||||
|
pub(crate) fn append(mut self, appended: &str) -> Option<(Self, Vec<Line<'static>>)> {
|
||||||
|
if !appended.ends_with('\n') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let Some(mut state) = self.state.take() else {
|
||||||
|
let lines = appended
|
||||||
|
.lines()
|
||||||
|
.map(|line| Line::from(line.to_string()))
|
||||||
|
.collect();
|
||||||
|
return Some((self, lines));
|
||||||
|
};
|
||||||
|
let bytes = state.bytes.checked_add(appended.len())?;
|
||||||
|
let lines = state.lines.checked_add(appended.lines().count())?;
|
||||||
|
if exceeds_highlight_limits(bytes, lines)
|
||||||
|
|| appended
|
||||||
|
.lines()
|
||||||
|
.any(|line| line.len() > MAX_HIGHLIGHT_LINE_BYTES)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let theme = match theme_lock().read() {
|
||||||
|
Ok(theme) => theme,
|
||||||
|
Err(poisoned) => poisoned.into_inner(),
|
||||||
|
};
|
||||||
|
if state.theme_revision != syntax_theme_revision() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (highlight_state, parse_state) = state.syntax;
|
||||||
|
let mut highlighter = HighlightLines::from_state(&theme, highlight_state, parse_state);
|
||||||
|
let mut rendered = Vec::with_capacity(lines - state.lines);
|
||||||
|
for line in LinesWithEndings::from(appended) {
|
||||||
|
let ranges = highlighter.highlight_line(line, syntax_set()).ok()?;
|
||||||
|
rendered.push(Line::from(highlighted_line_spans(ranges)));
|
||||||
|
}
|
||||||
|
state.bytes = bytes;
|
||||||
|
state.lines = lines;
|
||||||
|
state.syntax = highlighter.state();
|
||||||
|
self.state = Some(state);
|
||||||
|
Some((self, rendered))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "highlight_streaming_tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
use super::super::MAX_HIGHLIGHT_BYTES;
|
||||||
|
use super::super::MAX_HIGHLIGHT_LINE_BYTES;
|
||||||
|
use super::super::MAX_HIGHLIGHT_LINES;
|
||||||
|
use super::super::highlight_code_to_lines;
|
||||||
|
use super::super::syntax_theme_revision;
|
||||||
|
use super::StreamingCodeHighlighter;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn appended_lines_preserve_multiline_syntax_state() {
|
||||||
|
let first = "fn main() {\n /* comment\n";
|
||||||
|
let appended = " continues */\n println!(\"界 e\u{301}\");\n}\n";
|
||||||
|
let code = format!("{first}{appended}");
|
||||||
|
let highlighter =
|
||||||
|
StreamingCodeHighlighter::new(first, "rust", syntax_theme_revision()).unwrap();
|
||||||
|
let (_, lines) = highlighter.append(appended).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
lines,
|
||||||
|
highlight_code_to_lines(&code, "rust")
|
||||||
|
.into_iter()
|
||||||
|
.skip(first.lines().count())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_and_unknown_language_fences_append() {
|
||||||
|
for lang in ["rust", "unknown-language"] {
|
||||||
|
let highlighter = StreamingCodeHighlighter::new("", lang, syntax_theme_revision()).unwrap();
|
||||||
|
let (_, lines) = highlighter.append("fn main() {}\n\n").unwrap();
|
||||||
|
assert_eq!(lines, highlight_code_to_lines("fn main() {}\n\n", lang));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_lines_and_theme_changes_discard_state() {
|
||||||
|
let highlighter =
|
||||||
|
StreamingCodeHighlighter::new("fn first() {}\n", "rust", syntax_theme_revision()).unwrap();
|
||||||
|
assert!(highlighter.append("fn partial()").is_none());
|
||||||
|
|
||||||
|
let mut highlighter =
|
||||||
|
StreamingCodeHighlighter::new("fn first() {}\n", "rust", syntax_theme_revision()).unwrap();
|
||||||
|
let state = highlighter.state.as_mut().unwrap();
|
||||||
|
state.theme_revision = state.theme_revision.wrapping_sub(1);
|
||||||
|
assert!(highlighter.append("fn second() {}\n").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crossing_each_highlighting_limit_requires_a_canonical_render() {
|
||||||
|
let cases = [
|
||||||
|
(
|
||||||
|
String::new(),
|
||||||
|
format!("{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1)),
|
||||||
|
),
|
||||||
|
("x\n".repeat(MAX_HIGHLIGHT_LINES), "x\n".to_string()),
|
||||||
|
(
|
||||||
|
format!("{}\n", "x".repeat(1023)).repeat(MAX_HIGHLIGHT_BYTES / 1024),
|
||||||
|
"x\n".to_string(),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
for (initial, appended) in cases {
|
||||||
|
let highlighter =
|
||||||
|
StreamingCodeHighlighter::new(&initial, "rust", syntax_theme_revision()).unwrap();
|
||||||
|
assert!(highlighter.state.is_some());
|
||||||
|
assert!(highlighter.append(&appended).is_none());
|
||||||
|
|
||||||
|
let complete = format!("{initial}{appended}");
|
||||||
|
let highlighter =
|
||||||
|
StreamingCodeHighlighter::new(&complete, "rust", syntax_theme_revision()).unwrap();
|
||||||
|
assert!(highlighter.state.is_none());
|
||||||
|
let (_, lines) = highlighter.append("next\n").unwrap();
|
||||||
|
assert_eq!(lines, highlight_code_to_lines("next\n", "unknown-language"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
//! A conservative fast path for one open, top-level, language-tagged code fence.
|
||||||
|
|
||||||
|
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||||
|
use crate::render::highlight::StreamingCodeHighlighter;
|
||||||
|
use crate::render::highlight::syntax_theme_revision;
|
||||||
|
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||||
|
use ratatui::text::Span;
|
||||||
|
|
||||||
|
/// An append-only fence whose original source is identical to pulldown-cmark's code text.
|
||||||
|
pub(super) struct OpenCodeFence {
|
||||||
|
marker: u8,
|
||||||
|
marker_len: usize,
|
||||||
|
language: String,
|
||||||
|
content_start: usize,
|
||||||
|
source_len: usize,
|
||||||
|
theme_revision: u64,
|
||||||
|
/// Initialized only if another chunk arrives, avoiding duplicate work on one-shot fences.
|
||||||
|
highlighter: Option<StreamingCodeHighlighter>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenCodeFence {
|
||||||
|
/// Recognize the final mutable top-level block after its canonical render.
|
||||||
|
///
|
||||||
|
/// `source` is the final block's suffix, while `source_len` includes the entire committed
|
||||||
|
/// Markdown source and anchors the retained fence within it.
|
||||||
|
///
|
||||||
|
/// Indentation, CR/NUL normalization, and escaped info strings stay on the canonical path.
|
||||||
|
/// Any line that could end the fence also stays there, including deliberately conservative
|
||||||
|
/// false positives, so this fast path need not duplicate CommonMark's closing-fence grammar.
|
||||||
|
pub(super) fn detect(source: &str, source_len: usize, theme_revision: u64) -> Option<Self> {
|
||||||
|
let marker = *source.as_bytes().first()?;
|
||||||
|
if marker != b'`' && marker != b'~' {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (opening, code) = source.split_once('\n')?;
|
||||||
|
let marker_len = opening.bytes().take_while(|byte| *byte == marker).count();
|
||||||
|
if marker_len < 3
|
||||||
|
|| !source.ends_with('\n')
|
||||||
|
|| theme_revision != syntax_theme_revision()
|
||||||
|
|| source.contains(['\r', '\0'])
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let info = opening[marker_len..].trim_matches([' ', '\t', '\u{b}', '\u{c}']);
|
||||||
|
if info.contains(['&', '\\']) || (marker == b'`' && info.contains('`')) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let language = info
|
||||||
|
.split([',', ' ', '\t'])
|
||||||
|
.next()
|
||||||
|
.filter(|language| !language.is_empty())?;
|
||||||
|
// Retain at most one bounded language token, never the streamed code itself.
|
||||||
|
if language.len() > MAX_HIGHLIGHT_LINE_BYTES
|
||||||
|
|| has_possible_closing_line(code, marker, marker_len)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(Self {
|
||||||
|
marker,
|
||||||
|
marker_len,
|
||||||
|
language: language.to_string(),
|
||||||
|
content_start: source_len.checked_sub(code.len())?,
|
||||||
|
source_len,
|
||||||
|
theme_revision,
|
||||||
|
highlighter: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append code only while the caller's newline-committed source extends this same fence.
|
||||||
|
///
|
||||||
|
/// Returning `None` consumes the retained state and requires a canonical whole-fence render.
|
||||||
|
/// Successful lines include the empty indent span expected by the canonical Markdown writer.
|
||||||
|
pub(super) fn append(
|
||||||
|
mut self,
|
||||||
|
raw_source: &str,
|
||||||
|
committed_source: &str,
|
||||||
|
) -> Option<(Self, Vec<HyperlinkLine>)> {
|
||||||
|
if self.source_len.checked_add(committed_source.len()) != Some(raw_source.len())
|
||||||
|
|| !raw_source.ends_with(committed_source)
|
||||||
|
|| committed_source.contains(['\r', '\0'])
|
||||||
|
|| has_possible_closing_line(committed_source, self.marker, self.marker_len)
|
||||||
|
|| self.theme_revision != syntax_theme_revision()
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let highlighter = match self.highlighter.take() {
|
||||||
|
Some(highlighter) => highlighter,
|
||||||
|
None => StreamingCodeHighlighter::new(
|
||||||
|
&raw_source[self.content_start..self.source_len],
|
||||||
|
&self.language,
|
||||||
|
self.theme_revision,
|
||||||
|
)?,
|
||||||
|
};
|
||||||
|
let (highlighter, lines) = highlighter.append(committed_source)?;
|
||||||
|
self.highlighter = Some(highlighter);
|
||||||
|
self.source_len = raw_source.len();
|
||||||
|
let lines = lines
|
||||||
|
.into_iter()
|
||||||
|
.map(|mut line| {
|
||||||
|
// The canonical writer installs an empty indent span for top-level fences.
|
||||||
|
line.spans.insert(/*index*/ 0, Span::default());
|
||||||
|
HyperlinkLine::new(line)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Some((self, lines))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Conservatively detect any line that might close the fence.
|
||||||
|
///
|
||||||
|
/// False positives intentionally fall back to CommonMark instead of duplicating its closer rules.
|
||||||
|
fn has_possible_closing_line(source: &str, marker: u8, marker_len: usize) -> bool {
|
||||||
|
source.lines().any(|line| {
|
||||||
|
line.trim_start_matches([' ', '\t'])
|
||||||
|
.bytes()
|
||||||
|
.take_while(|byte| *byte == marker)
|
||||||
|
.count()
|
||||||
|
>= marker_len
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "code_fence_tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
use super::StreamingRender;
|
||||||
|
use super::append;
|
||||||
|
use super::append_rich_and_assert_matches_full;
|
||||||
|
use super::assert_rich_stream_matches_full_render;
|
||||||
|
use super::lines_to_plain_strings;
|
||||||
|
use super::test_cwd;
|
||||||
|
use crate::history_cell::HistoryRenderMode;
|
||||||
|
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||||
|
use insta::assert_debug_snapshot;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn growing_code_fences_preserve_styles_unicode_links_and_blank_lines() {
|
||||||
|
let (_, render) = assert_rich_stream_matches_full_render(
|
||||||
|
&[
|
||||||
|
"Before [link](https://example.com).\n\n```rust,no_run\n",
|
||||||
|
"fn main() {\n",
|
||||||
|
" /* a multiline\n",
|
||||||
|
" comment */\n\n",
|
||||||
|
" println!(\"界 e\u{301} https://example.com\");\n",
|
||||||
|
"}\n",
|
||||||
|
"```\n\n~~~~python\n",
|
||||||
|
"text = \"\"\"a multiline\n",
|
||||||
|
"string\"\"\"\n",
|
||||||
|
"~~~~\n",
|
||||||
|
],
|
||||||
|
Some(18),
|
||||||
|
);
|
||||||
|
assert!(render.open_code_fence.is_none());
|
||||||
|
assert_debug_snapshot!(lines_to_plain_strings(&render.lines), @r#"
|
||||||
|
[
|
||||||
|
"Before link",
|
||||||
|
"(https://example.com).",
|
||||||
|
"",
|
||||||
|
"fn main() {",
|
||||||
|
" /* a multiline",
|
||||||
|
" comment */",
|
||||||
|
"",
|
||||||
|
" println!(\"界 e\u{301} https://example.com\");",
|
||||||
|
"}",
|
||||||
|
"",
|
||||||
|
"text = \"\"\"a multiline",
|
||||||
|
"string\"\"\"",
|
||||||
|
]
|
||||||
|
"#);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn long_open_fence_retains_its_rendered_prefix() {
|
||||||
|
let cwd = test_cwd();
|
||||||
|
let mut source = String::new();
|
||||||
|
let mut render = StreamingRender::new();
|
||||||
|
append_rich_and_assert_matches_full(
|
||||||
|
&mut render,
|
||||||
|
&mut source,
|
||||||
|
"```rust\nfn main() {\n",
|
||||||
|
Some(80),
|
||||||
|
&cwd,
|
||||||
|
);
|
||||||
|
let first_line = render.lines[0].line.spans[1].content.as_ptr();
|
||||||
|
for index in 0..512 {
|
||||||
|
append(
|
||||||
|
&mut render,
|
||||||
|
&mut source,
|
||||||
|
&format!(" let value_{index} = {index};\n"),
|
||||||
|
Some(80),
|
||||||
|
&cwd,
|
||||||
|
HistoryRenderMode::Rich,
|
||||||
|
);
|
||||||
|
assert!(render.open_code_fence.is_some());
|
||||||
|
}
|
||||||
|
assert_eq!(render.lines[0].line.spans[1].content.as_ptr(), first_line);
|
||||||
|
append_rich_and_assert_matches_full(&mut render, &mut source, "}\n```\n", Some(80), &cwd);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ambiguous_and_normalized_fences_use_canonical_semantics() {
|
||||||
|
let streams: &[&[&str]] = &[
|
||||||
|
&["```rust\n", "x\n", "``", "`\n"],
|
||||||
|
&["```rust\n", "x\n", "```\t\n", "after\n"],
|
||||||
|
&["```rust\n", "x\n", " ```` \n", "after\n"],
|
||||||
|
&["~~~rust\n", "x\n", "~~~\u{a0}\n", "after\n"],
|
||||||
|
&["```rust\r\n", "let value = 'é';\r\n", "```\r\n"],
|
||||||
|
&["```rust\n", "let value = '\0';\n", "```\n"],
|
||||||
|
&["```rust,no_run\n", "fn main() {}\n", "```\n"],
|
||||||
|
&[" ```rust\n", " fn main() {}\n", " ```\n"],
|
||||||
|
&["```unknown-language\n", "first\n", "second\n", "```\n"],
|
||||||
|
&[
|
||||||
|
"```markdown\n| A | B |\n",
|
||||||
|
"| --- | --- |\n",
|
||||||
|
"| 1 | 2 |\n```\n\n```rust\nfn first() {}\n",
|
||||||
|
"fn second() {}\n",
|
||||||
|
"```\n",
|
||||||
|
],
|
||||||
|
];
|
||||||
|
for chunks in streams {
|
||||||
|
assert_rich_stream_matches_full_render(chunks, Some(40));
|
||||||
|
}
|
||||||
|
for indent in 0..=3 {
|
||||||
|
let closing = format!("{}~~~~~\t\n", " ".repeat(indent));
|
||||||
|
assert_rich_stream_matches_full_render(
|
||||||
|
&["~~~~rust\n", "x\n", &closing, "after\n"],
|
||||||
|
Some(40),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crossing_a_highlight_limit_removes_previously_highlighted_colors() {
|
||||||
|
let cwd = test_cwd();
|
||||||
|
let mut source = String::new();
|
||||||
|
let mut render = StreamingRender::new();
|
||||||
|
for chunk in [
|
||||||
|
"```rust\nlet first = 1;\n".to_string(),
|
||||||
|
format!("{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1)),
|
||||||
|
"let second = 2;\n".to_string(),
|
||||||
|
"```\n".to_string(),
|
||||||
|
] {
|
||||||
|
append_rich_and_assert_matches_full(&mut render, &mut source, &chunk, Some(80), &cwd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recomputing_after_resize_or_mode_change_drops_fence_state() {
|
||||||
|
let (mut source, mut render) =
|
||||||
|
assert_rich_stream_matches_full_render(&["```rust\nfn first() {}\n"], Some(80));
|
||||||
|
assert!(render.open_code_fence.is_some());
|
||||||
|
let cwd = test_cwd();
|
||||||
|
render.recompute(
|
||||||
|
&source,
|
||||||
|
Some(20),
|
||||||
|
&cwd,
|
||||||
|
HistoryRenderMode::Rich,
|
||||||
|
/*inline_visualization_context*/ None,
|
||||||
|
);
|
||||||
|
assert!(render.open_code_fence.is_none());
|
||||||
|
append_rich_and_assert_matches_full(
|
||||||
|
&mut render,
|
||||||
|
&mut source,
|
||||||
|
"fn second() {}\n",
|
||||||
|
Some(20),
|
||||||
|
&cwd,
|
||||||
|
);
|
||||||
|
assert!(render.open_code_fence.is_some());
|
||||||
|
render.clear();
|
||||||
|
assert!(render.open_code_fence.is_none());
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
use super::OpenCodeFence;
|
||||||
|
use super::has_possible_closing_line;
|
||||||
|
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||||
|
use crate::render::highlight::syntax_theme_revision;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_only_source_preserving_open_fences() {
|
||||||
|
for source in [
|
||||||
|
"```rust,no_run\nfn main() {}\n",
|
||||||
|
"~~~~python title=demo\nvalue = 1\n",
|
||||||
|
"```\u{b}rust\u{c}\nfn main() {}\n",
|
||||||
|
"```unknown-language\ntext\n",
|
||||||
|
] {
|
||||||
|
let fence =
|
||||||
|
OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).expect(source);
|
||||||
|
assert!(fence.highlighter.is_none());
|
||||||
|
}
|
||||||
|
for source in [
|
||||||
|
" ```rust\ncode\n",
|
||||||
|
"```\ncode\n",
|
||||||
|
"```rust\r\ncode\r\n",
|
||||||
|
"```rust\n'\0'\n",
|
||||||
|
"```rust\ncode\n",
|
||||||
|
"```rust\\,no_run\ncode\n",
|
||||||
|
"```rust\npartial",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).is_none(),
|
||||||
|
"{source:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let source = format!("```{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1));
|
||||||
|
assert!(OpenCodeFence::detect(&source, source.len(), syntax_theme_revision()).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_possible_closer_returns_to_the_canonical_parser() {
|
||||||
|
let opening = "~~~~rust\n";
|
||||||
|
assert!(!has_possible_closing_line(
|
||||||
|
"~~~\n```\n",
|
||||||
|
/*marker*/ b'~',
|
||||||
|
/*marker_len*/ 4
|
||||||
|
));
|
||||||
|
for line in [
|
||||||
|
"~~~~\n",
|
||||||
|
" ~~~~~~ \n",
|
||||||
|
"~~~~\t\n",
|
||||||
|
"~~~~\u{a0}\n",
|
||||||
|
"\t~~~~\n",
|
||||||
|
" ~~~~\n",
|
||||||
|
"~~~~ trailing text\n",
|
||||||
|
] {
|
||||||
|
let source = format!("{opening}{line}");
|
||||||
|
assert!(OpenCodeFence::detect(&source, source.len(), syntax_theme_revision()).is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lazy_initialization_rejects_a_changed_theme() {
|
||||||
|
let source = "```rust\nfn first() {}\n";
|
||||||
|
let mut fence = OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).unwrap();
|
||||||
|
fence.theme_revision = fence.theme_revision.wrapping_sub(1);
|
||||||
|
let appended = "fn second() {}\n";
|
||||||
|
assert!(
|
||||||
|
fence
|
||||||
|
.append(&format!("{source}{appended}"), appended)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ use std::time::Instant;
|
|||||||
use crate::markdown_stream::MarkdownStreamCollector;
|
use crate::markdown_stream::MarkdownStreamCollector;
|
||||||
use crate::terminal_hyperlinks::HyperlinkLine;
|
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||||
pub(crate) mod chunking;
|
pub(crate) mod chunking;
|
||||||
|
mod code_fence;
|
||||||
pub(crate) mod commit_tick;
|
pub(crate) mod commit_tick;
|
||||||
pub(crate) mod controller;
|
pub(crate) mod controller;
|
||||||
mod render;
|
mod render;
|
||||||
|
|||||||
@@ -3,12 +3,14 @@
|
|||||||
//! Completed top-level blocks are retained while the final block stays mutable, avoiding repeated
|
//! Completed top-level blocks are retained while the final block stays mutable, avoiding repeated
|
||||||
//! rendering of the stable prefix as newline-bearing deltas arrive.
|
//! rendering of the stable prefix as newline-bearing deltas arrive.
|
||||||
|
|
||||||
|
use super::code_fence::OpenCodeFence;
|
||||||
use crate::history_cell::HistoryRenderMode;
|
use crate::history_cell::HistoryRenderMode;
|
||||||
use crate::history_cell::raw_lines_from_source;
|
use crate::history_cell::raw_lines_from_source;
|
||||||
use crate::inline_visualization::InlineVisualizationContext;
|
use crate::inline_visualization::InlineVisualizationContext;
|
||||||
use crate::inline_visualization::contains_inline_visualization;
|
use crate::inline_visualization::contains_inline_visualization;
|
||||||
use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
|
use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
|
||||||
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
|
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
|
||||||
|
use crate::render::highlight::syntax_theme_revision;
|
||||||
use crate::terminal_hyperlinks::HyperlinkLine;
|
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||||
use crate::terminal_hyperlinks::plain_hyperlink_lines;
|
use crate::terminal_hyperlinks::plain_hyperlink_lines;
|
||||||
use ratatui::text::Line;
|
use ratatui::text::Line;
|
||||||
@@ -28,6 +30,8 @@ pub(super) struct StreamingRender {
|
|||||||
has_reference_link_definition: bool,
|
has_reference_link_definition: bool,
|
||||||
/// Inline visualization directives require source-wide rewriting once one is committed.
|
/// Inline visualization directives require source-wide rewriting once one is committed.
|
||||||
has_inline_visualization_directive: bool,
|
has_inline_visualization_directive: bool,
|
||||||
|
/// Parser state for a directly appendable, open top-level code fence.
|
||||||
|
open_code_fence: Option<OpenCodeFence>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StreamingRender {
|
impl StreamingRender {
|
||||||
@@ -38,6 +42,7 @@ impl StreamingRender {
|
|||||||
stable_rendered_len: 0,
|
stable_rendered_len: 0,
|
||||||
has_reference_link_definition: false,
|
has_reference_link_definition: false,
|
||||||
has_inline_visualization_directive: false,
|
has_inline_visualization_directive: false,
|
||||||
|
open_code_fence: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,6 +52,7 @@ impl StreamingRender {
|
|||||||
self.stable_rendered_len = 0;
|
self.stable_rendered_len = 0;
|
||||||
self.has_reference_link_definition = false;
|
self.has_reference_link_definition = false;
|
||||||
self.has_inline_visualization_directive = false;
|
self.has_inline_visualization_directive = false;
|
||||||
|
self.open_code_fence = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Re-render the full source and reset both stable-prefix boundaries.
|
/// Re-render the full source and reset both stable-prefix boundaries.
|
||||||
@@ -61,6 +67,7 @@ impl StreamingRender {
|
|||||||
render_mode: HistoryRenderMode,
|
render_mode: HistoryRenderMode,
|
||||||
inline_visualization_context: Option<&InlineVisualizationContext>,
|
inline_visualization_context: Option<&InlineVisualizationContext>,
|
||||||
) {
|
) {
|
||||||
|
self.open_code_fence = None;
|
||||||
self.has_inline_visualization_directive = contains_inline_visualization(source);
|
self.has_inline_visualization_directive = contains_inline_visualization(source);
|
||||||
self.lines = match (render_mode, inline_visualization_context) {
|
self.lines = match (render_mode, inline_visualization_context) {
|
||||||
(HistoryRenderMode::Rich, None) if !self.has_inline_visualization_directive => {
|
(HistoryRenderMode::Rich, None) if !self.has_inline_visualization_directive => {
|
||||||
@@ -130,7 +137,16 @@ impl StreamingRender {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(fence) = self.open_code_fence.take()
|
||||||
|
&& let Some((fence, lines)) = fence.append(raw_source, committed_source)
|
||||||
|
{
|
||||||
|
self.lines.extend(lines);
|
||||||
|
self.open_code_fence = Some(fence);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let pending_source = &raw_source[self.stable_source_len..];
|
let pending_source = &raw_source[self.stable_source_len..];
|
||||||
|
let theme_revision = syntax_theme_revision();
|
||||||
let pending =
|
let pending =
|
||||||
render_streaming_markdown_agent_with_links_and_cwd(pending_source, width, Some(cwd));
|
render_streaming_markdown_agent_with_links_and_cwd(pending_source, width, Some(cwd));
|
||||||
if pending.has_reference_link_definition {
|
if pending.has_reference_link_definition {
|
||||||
@@ -145,6 +161,13 @@ impl StreamingRender {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let final_block_start = pending.last_top_level_block_start.unwrap_or(/*default*/ 0);
|
||||||
|
self.open_code_fence = OpenCodeFence::detect(
|
||||||
|
&pending_source[final_block_start..],
|
||||||
|
raw_source.len(),
|
||||||
|
theme_revision,
|
||||||
|
);
|
||||||
|
|
||||||
let mut newly_stable_rendered_len = None;
|
let mut newly_stable_rendered_len = None;
|
||||||
if let Some(boundary) = pending.last_top_level_block_start {
|
if let Some(boundary) = pending.last_top_level_block_start {
|
||||||
let newly_stable_source = &pending_source[..boundary];
|
let newly_stable_source = &pending_source[..boundary];
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ use pretty_assertions::assert_eq;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[path = "code_fence_render_tests.rs"]
|
||||||
|
mod code_fence_tests;
|
||||||
|
|
||||||
fn test_cwd() -> PathBuf {
|
fn test_cwd() -> PathBuf {
|
||||||
std::env::temp_dir()
|
std::env::temp_dir()
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user