github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-20 08:19:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Render streamed Markdown incrementally (#34045)

Browse Source 
## Why

Streaming TUI responses repeatedly rendered the entire accumulated Markdown source for each committed delta, including blocks whose output could no longer change.

## What changed

- Retain rendered output for completed top-level Markdown blocks and rerender only the final mutable block as new source arrives.
- Track block boundaries during the existing Markdown parser pass and map them through table-fence normalization.
- Preserve canonical full rendering when source-wide state may affect earlier output, including reference link definitions and inline visualizations, and when the width or render mode changes.
- Append raw-mode lines directly while preserving blank lines.

## Testing

Added coverage that compares incremental output with full rendering across paragraphs, headings, lists, block quotes, code fences, tables, HTML blocks, links, visualizations, render-mode changes, and normalized table fences.

GitOrigin-RevId: 748171c4ed58923329b59c48ea3c2afa4819ec88
This commit is contained in:
Charlie Marsh
2026-07-18 18:00:51 +00:00
committed by copyberry
parent b8b61bc692
commit 537e69ab66
10 changed files with 874 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/tui/src/inline_visualization.rs
+1 -1
View File
@@ -24,7 +24,7 @@ use uuid::Uuid;
use self::viewer::materialize_document;
const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{";
pub(crate) const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{";
const MAX_FRAGMENT_BYTES: u64 = 2 * 1024 * 1024;
#[derive(Clone, Debug)]
codex-rs/tui/src/markdown.rs
+28
View File
@@ -106,6 +106,34 @@ pub(crate) fn render_markdown_agent_with_links_cwd_and_visualizations(
lines
}
/// Render an agent message and collect the block metadata needed for incremental rendering.
///
/// Block offsets are mapped back to `markdown_source` after Markdown table fences are unwrapped.
/// If a normalized boundary cannot be expressed as a raw-source suffix, it is discarded so the
/// transformed block remains mutable.
pub(crate) fn render_streaming_markdown_agent_with_links_and_cwd(
markdown_source: &str,
width: Option<usize>,
cwd: Option<&Path>,
) -> crate::markdown_render::StreamingMarkdownRender {
let normalized = unwrap_markdown_fences(markdown_source);
let mut rendered = crate::markdown_render::render_streaming_markdown_lines_with_width_and_cwd(
&normalized,
width,
cwd,
);
if normalized != markdown_source {
// Fence unwrapping removes opening/closing lines. A normalized tail that is still a raw
// suffix necessarily begins after those removed lines, so its boundary can safely be
// mapped back to the raw source; otherwise leave the transformed block mutable.
rendered.last_top_level_block_start = rendered
.last_top_level_block_start
.and_then(|boundary| markdown_source.strip_suffix(&normalized[boundary..]))
.map(str::len);
}
rendered
}
/// Strip `` ```md ``/`` ```markdown `` fences that contain tables, emitting their content as bare
/// markdown so `pulldown-cmark` parses the tables natively.
///
codex-rs/tui/src/markdown_render.rs
+4
View File
@@ -77,8 +77,12 @@ use unicode_width::UnicodeWidthChar;
use unicode_width::UnicodeWidthStr;
use url::Url;
mod streaming;
mod table_key_value;
pub(crate) use streaming::StreamingMarkdownRender;
pub(crate) use streaming::render_streaming_markdown_lines_with_width_and_cwd;
const TABLE_COLUMN_GAP: usize = 2;
const TABLE_CELL_PADDING: usize = 1;
const TABLE_HEADER_SEPARATOR_CHAR: char = '━';
codex-rs/tui/src/markdown_render/streaming.rs
+92
View File
@@ -0,0 +1,92 @@
//! Streaming markdown render metadata collected during the writer's single parse pass.
//!
//! Top-level block offsets always refer to the exact source passed to this renderer; callers that
//! normalize source before rendering must not apply those offsets to the original source.
use super::DecodedTextMerge;
use super::Event;
use super::HyperlinkLine;
use super::Options;
use super::Parser;
use super::Tag;
use super::Writer;
use super::never_hide_link_destination;
use std::ops::Range;
use std::path::Path;
/// Rendered lines and the block metadata needed to keep only the final block mutable.
pub(crate) struct StreamingMarkdownRender {
/// Styled output produced by the same parser pass that collected the metadata below.
pub(crate) lines: Vec<HyperlinkLine>,
/// Byte offset of the final top-level block when at least one earlier block exists.
pub(crate) last_top_level_block_start: Option<usize>,
/// Whether a reference definition can retroactively change another block's rendering.
pub(crate) has_reference_link_definition: bool,
/// Whether the first block is raw HTML, which joins a retained prefix without a separator.
pub(crate) first_top_level_block_is_html: bool,
}
/// Render `input` while tracking the final mutable top-level block.
///
/// Every reported byte offset indexes the exact `input` passed here. Callers that transform source
/// before rendering must map the offset back to their original source before retaining a prefix.
pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd(
input: &str,
width: Option<usize>,
cwd: Option<&Path>,
) -> StreamingMarkdownRender {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
options.insert(Options::ENABLE_TABLES);
let parser = Parser::new_ext(input, options);
let has_reference_link_definition = parser.reference_definitions().iter().next().is_some();
let parser = TopLevelBlockTracker {
iter: DecodedTextMerge::new(parser.into_offset_iter()),
depth: 0,
block_count: 0,
last_start: 0,
first_is_html: false,
};
let mut writer = Writer::new(input, parser, width, cwd, &never_hide_link_destination);
writer.run();
StreamingMarkdownRender {
lines: writer.text,
last_top_level_block_start: (writer.iter.block_count > 1).then_some(writer.iter.last_start),
has_reference_link_definition,
first_top_level_block_is_html: writer.iter.first_is_html,
}
}
/// Records top-level block boundaries without adding a second parser traversal.
struct TopLevelBlockTracker<I> {
iter: I,
depth: usize,
block_count: usize,
last_start: usize,
first_is_html: bool,
}
impl<'a, I> Iterator for TopLevelBlockTracker<I>
where
I: Iterator<Item = (Event<'a>, Range<usize>)>,
{
type Item = (Event<'a>, Range<usize>);
fn next(&mut self) -> Option<Self::Item> {
let (event, range) = self.iter.next()?;
if self.depth == 0 && matches!(&event, Event::Start(_) | Event::Rule | Event::Html(_)) {
self.block_count += 1;
self.last_start = range.start;
if self.block_count == 1 {
self.first_is_html =
matches!(&event, Event::Start(Tag::HtmlBlock) | Event::Html(_));
}
}
match event {
Event::Start(_) => self.depth += 1,
Event::End(_) => self.depth = self.depth.saturating_sub(1),
_ => {}
}
Some((event, range))
}
}
codex-rs/tui/src/streaming/controller.rs
+79 -44
View File
@@ -29,7 +29,7 @@
//!
//! ## Invariants
//!
//! - `emitted_stable_len <= enqueued_stable_len <= rendered_lines.len()`.
//! - `emitted_stable_len <= enqueued_stable_len <= render.lines.len()`.
//! - `raw_source` is append-only until `reset()`; never modified mid-stream.
//! - Tail starts exactly at `enqueued_stable_len`.
//! - During confirmed table streaming, only lines from the table header onward
@@ -37,13 +37,11 @@
use crate::history_cell::HistoryCell;
use crate::history_cell::HistoryRenderMode;
use crate::history_cell::raw_lines_from_source;
use crate::history_cell::{self};
use crate::inline_visualization::InlineVisualizationContext;
use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
use crate::style::proposed_plan_style;
use crate::terminal_hyperlinks::HyperlinkLine;
use crate::terminal_hyperlinks::plain_hyperlink_lines;
use crate::terminal_hyperlinks::prefix_hyperlink_lines;
use ratatui::prelude::Stylize;
use ratatui::text::Line;
@@ -53,6 +51,8 @@ use std::time::Duration;
use std::time::Instant;
use super::StreamState;
use super::render::StreamingRender;
use super::render::render_source;
use super::table_holdback::TableHoldbackScanner;
use super::table_holdback::TableHoldbackState;
#[cfg(test)]
@@ -77,8 +77,8 @@ struct StreamCore {
width: Option<usize>,
/// Accumulated raw markdown source for the current stream.
raw_source: String,
/// Full re-render of `raw_source` at `width`. Rebuilt on every committed delta.
rendered_lines: Vec<HyperlinkLine>,
/// Incremental render of `raw_source` at `width`.
render: StreamingRender,
/// Lines enqueued into the commit-animation queue.
enqueued_stable_len: usize,
/// Lines actually emitted to scrollback.
@@ -116,7 +116,7 @@ impl StreamCore {
state: StreamState::new(width, cwd),
width,
raw_source: String::with_capacity(1024),
rendered_lines: Vec::with_capacity(64),
render: StreamingRender::new(),
enqueued_stable_len: 0,
emitted_stable_len: 0,
cwd: cwd.to_path_buf(),
@@ -146,7 +146,14 @@ impl StreamCore {
{
self.raw_source.push_str(&committed_source);
self.holdback_scanner.push_source_chunk(&committed_source);
self.recompute_streaming_render();
self.render.append(
&self.raw_source,
&committed_source,
self.width,
self.cwd.as_path(),
self.render_mode,
self.inline_visualization_context.as_ref(),
);
enqueued = self.sync_stable_queue();
}
enqueued
@@ -165,7 +172,13 @@ impl StreamCore {
self.raw_source.push_str(&remainder_source);
self.holdback_scanner.push_source_chunk(&remainder_source);
}
let rendered = self.render_source(&self.raw_source);
let rendered = render_source(
&self.raw_source,
self.width,
self.cwd.as_path(),
self.render_mode,
self.inline_visualization_context.as_ref(),
);
if self.emitted_stable_len >= rendered.len() {
Vec::new()
} else {
@@ -220,13 +233,13 @@ impl StreamCore {
/// reappear in the active cell and duplicate content on screen.
#[inline]
fn current_tail_lines(&self) -> Vec<HyperlinkLine> {
let start = self.enqueued_stable_len.min(self.rendered_lines.len());
self.rendered_lines[start..].to_vec()
let start = self.enqueued_stable_len.min(self.render.lines.len());
self.render.lines[start..].to_vec()
}
#[inline]
fn has_tail(&self) -> bool {
self.enqueued_stable_len < self.rendered_lines.len()
self.enqueued_stable_len < self.render.lines.len()
}
/// Update rendering width and rebuild queued stable lines for the new layout.
@@ -250,10 +263,16 @@ impl StreamCore {
return;
}
self.recompute_streaming_render();
self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len());
self.render.recompute(
&self.raw_source,
self.width,
self.cwd.as_path(),
self.render_mode,
self.inline_visualization_context.as_ref(),
);
self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len());
if had_pending_queue
&& self.emitted_stable_len == self.rendered_lines.len()
&& self.emitted_stable_len == self.render.lines.len()
&& self.emitted_stable_len > 0
{
// If wrapped remainder compresses into fewer lines at the new width,
@@ -266,7 +285,7 @@ impl StreamCore {
// Avoid replaying already-emitted content after resize when no
// stable lines were waiting in the queue and there was no mutable
// tail to preserve.
self.enqueued_stable_len = self.rendered_lines.len();
self.enqueued_stable_len = self.render.lines.len();
return;
}
self.rebuild_stable_queue_from_render();
@@ -276,29 +295,13 @@ impl StreamCore {
fn reset(&mut self) {
self.state.clear();
self.raw_source.clear();
self.rendered_lines.clear();
self.render.clear();
self.enqueued_stable_len = 0;
self.emitted_stable_len = 0;
self.stable_prefix_len_cache = None;
self.holdback_scanner.reset();
}
fn render_source(&self, source: &str) -> Vec<HyperlinkLine> {
match self.render_mode {
HistoryRenderMode::Rich => render_markdown_agent_with_links_cwd_and_visualizations(
source,
self.width,
Some(self.cwd.as_path()),
self.inline_visualization_context.as_ref(),
),
HistoryRenderMode::Raw => plain_hyperlink_lines(raw_lines_from_source(source)),
}
}
fn recompute_streaming_render(&mut self) {
self.rendered_lines = self.render_source(&self.raw_source);
}
fn set_render_mode(&mut self, render_mode: HistoryRenderMode) {
if self.render_mode == render_mode {
return;
@@ -311,17 +314,23 @@ impl StreamCore {
return;
}
self.recompute_streaming_render();
self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len());
self.render.recompute(
&self.raw_source,
self.width,
self.cwd.as_path(),
self.render_mode,
self.inline_visualization_context.as_ref(),
);
self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len());
if had_pending_queue
&& self.emitted_stable_len == self.rendered_lines.len()
&& self.emitted_stable_len == self.render.lines.len()
&& self.emitted_stable_len > 0
{
self.emitted_stable_len -= 1;
}
self.state.clear_queue();
if self.emitted_stable_len > 0 && !had_pending_queue && !had_live_tail {
self.enqueued_stable_len = self.rendered_lines.len();
self.enqueued_stable_len = self.render.lines.len();
return;
}
self.rebuild_stable_queue_from_render();
@@ -330,7 +339,8 @@ impl StreamCore {
/// Compute how many rendered lines should be in the stable region.
fn compute_target_stable_len(&mut self) -> usize {
let tail_budget = self.active_tail_budget_lines();
self.rendered_lines
self.render
.lines
.len()
.saturating_sub(tail_budget)
.max(self.emitted_stable_len)
@@ -347,7 +357,7 @@ impl StreamCore {
self.state.clear_queue();
if self.emitted_stable_len < target_stable_len {
self.state.enqueue(
self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec(),
self.render.lines[self.emitted_stable_len..target_stable_len].to_vec(),
);
}
self.enqueued_stable_len = target_stable_len;
@@ -359,7 +369,7 @@ impl StreamCore {
}
self.state
.enqueue(self.rendered_lines[self.enqueued_stable_len..target_stable_len].to_vec());
.enqueue(self.render.lines[self.enqueued_stable_len..target_stable_len].to_vec());
self.enqueued_stable_len = target_stable_len;
true
}
@@ -374,7 +384,7 @@ impl StreamCore {
self.state.clear_queue();
if self.emitted_stable_len < target_stable_len {
self.state
.enqueue(self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec());
.enqueue(self.render.lines[self.emitted_stable_len..target_stable_len].to_vec());
}
self.enqueued_stable_len = target_stable_len;
}
@@ -416,11 +426,11 @@ impl StreamCore {
/// the only place where those coordinate systems are bridged.
fn tail_budget_from_source_start(&mut self, source_start: usize) -> usize {
if source_start == 0 {
return self.rendered_lines.len();
return self.render.lines.len();
}
let source_start = source_start.min(self.raw_source.len());
let stable_prefix_len = self.stable_prefix_len_for_source_start(source_start);
self.rendered_lines.len().saturating_sub(stable_prefix_len)
self.render.lines.len().saturating_sub(stable_prefix_len)
}
/// Render the stable prefix before `source_start` and return its line count.
@@ -816,6 +826,31 @@ mod tests {
.collect()
}
#[test]
fn queued_heading_is_emitted_once_after_incremental_append() {
let mut ctrl = stream_controller(Some(80));
assert!(ctrl.push("Paragraph.\n\n# Heading\n\n"));
ctrl.push("Next paragraph.\n");
let (cell, _) = ctrl.on_commit_tick_batch(usize::MAX);
let mut streamed = cell
.into_iter()
.flat_map(|cell| cell.transcript_lines(u16::MAX))
.collect::<Vec<_>>();
if let (Some(cell), _source) = ctrl.finalize() {
streamed.extend(cell.transcript_lines(u16::MAX));
}
let streamed = lines_to_plain_strings(&streamed);
assert_eq!(
streamed
.iter()
.filter(|line| line.contains("# Heading"))
.count(),
1,
"expected the streamed heading to be emitted once: {streamed:?}",
);
}
fn collect_plan_streamed_lines(deltas: &[&str], width: Option<usize>) -> Vec<String> {
let mut ctrl = plan_stream_controller(width);
let mut lines = Vec::new();
@@ -896,7 +931,7 @@ mod tests {
let mut ctrl = stream_controller(Some(80));
assert!(!ctrl.has_live_tail());
ctrl.core.rendered_lines = vec![Line::from("tail line").into()];
ctrl.core.render.lines = vec![Line::from("tail line").into()];
ctrl.core.enqueued_stable_len = 0;
assert!(ctrl.has_live_tail());
@@ -909,7 +944,7 @@ mod tests {
let mut ctrl = plan_stream_controller(Some(80));
assert!(!ctrl.has_live_tail());
ctrl.core.rendered_lines = vec![Line::from("tail line").into()];
ctrl.core.render.lines = vec![Line::from("tail line").into()];
ctrl.core.enqueued_stable_len = 0;
assert!(ctrl.has_live_tail());
codex-rs/tui/src/streaming/mod.rs
+1
View File
@@ -19,6 +19,7 @@ use crate::terminal_hyperlinks::HyperlinkLine;
pub(crate) mod chunking;
pub(crate) mod commit_tick;
pub(crate) mod controller;
mod render;
mod table_holdback;
struct QueuedLine {
codex-rs/tui/src/streaming/render.rs
+191
View File
@@ -0,0 +1,191 @@
//! Incremental markdown rendering for in-flight transcript streams.
//!
//! Completed top-level blocks are retained while the final block stays mutable, avoiding repeated
//! rendering of the stable prefix as newline-bearing deltas arrive.
use crate::history_cell::HistoryRenderMode;
use crate::history_cell::raw_lines_from_source;
use crate::inline_visualization::DIRECTIVE_PREFIX;
use crate::inline_visualization::InlineVisualizationContext;
use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
use crate::terminal_hyperlinks::HyperlinkLine;
use crate::terminal_hyperlinks::plain_hyperlink_lines;
use ratatui::text::Line;
use std::path::Path;
/// Incremental render state split at source and rendered-line boundaries.
///
/// The prefix before both boundaries is immutable; only the final top-level Markdown block is
/// re-rendered as committed source arrives.
pub(super) struct StreamingRender {
pub(super) lines: Vec<HyperlinkLine>,
/// Source prefix containing only completed top-level markdown blocks.
stable_source_len: usize,
/// Rendered-line boundary corresponding to `stable_source_len`.
stable_rendered_len: usize,
/// Reference-style link definitions can affect any earlier or later markdown block.
has_reference_link_definition: bool,
}
impl StreamingRender {
pub(super) fn new() -> Self {
Self {
lines: Vec::with_capacity(64),
stable_source_len: 0,
stable_rendered_len: 0,
has_reference_link_definition: false,
}
}
pub(super) fn clear(&mut self) {
self.lines.clear();
self.stable_source_len = 0;
self.stable_rendered_len = 0;
self.has_reference_link_definition = false;
}
/// Re-render the full source and reset both stable-prefix boundaries.
///
/// This is used when width or render mode changes, and whenever source-wide rendering state
/// makes retaining previously rendered blocks unsafe.
pub(super) fn recompute(
&mut self,
source: &str,
width: Option<usize>,
cwd: &Path,
render_mode: HistoryRenderMode,
inline_visualization_context: Option<&InlineVisualizationContext>,
) {
self.lines = match (render_mode, inline_visualization_context) {
(HistoryRenderMode::Rich, None) if !source.contains(DIRECTIVE_PREFIX) => {
let rendered =
render_streaming_markdown_agent_with_links_and_cwd(source, width, Some(cwd));
self.has_reference_link_definition = rendered.has_reference_link_definition;
rendered.lines
}
_ => {
self.has_reference_link_definition = false;
render_source(
source,
width,
cwd,
render_mode,
inline_visualization_context,
)
}
};
self.stable_source_len = 0;
self.stable_rendered_len = 0;
}
/// Append newly committed source while retaining only the final markdown block as mutable.
///
/// The final top-level block can still change meaning when another line arrives (for example,
/// list tightness, a setext heading, a fenced block, or a table). Earlier top-level blocks are
/// rendered once and retained. Reference-style link definitions and inline visualization
/// rewriting fall back to a full render because they can affect source-wide rendering state.
pub(super) fn append(
&mut self,
raw_source: &str,
committed_source: &str,
width: Option<usize>,
cwd: &Path,
render_mode: HistoryRenderMode,
inline_visualization_context: Option<&InlineVisualizationContext>,
) {
if render_mode == HistoryRenderMode::Raw {
self.lines
.extend(plain_hyperlink_lines(raw_lines_from_source(
committed_source,
)));
return;
}
if inline_visualization_context.is_some() || raw_source.contains(DIRECTIVE_PREFIX) {
self.recompute(
raw_source,
width,
cwd,
render_mode,
inline_visualization_context,
);
return;
}
if self.has_reference_link_definition {
self.recompute(
raw_source,
width,
cwd,
render_mode,
inline_visualization_context,
);
return;
}
let pending_source = &raw_source[self.stable_source_len..];
let pending =
render_streaming_markdown_agent_with_links_and_cwd(pending_source, width, Some(cwd));
if pending.has_reference_link_definition {
self.has_reference_link_definition = true;
self.recompute(
raw_source,
width,
cwd,
render_mode,
inline_visualization_context,
);
return;
}
let mut newly_stable_rendered_len = None;
if let Some(boundary) = pending.last_top_level_block_start {
let newly_stable_source = &pending_source[..boundary];
let newly_stable = render_source(
newly_stable_source,
width,
cwd,
render_mode,
inline_visualization_context,
);
self.stable_source_len += boundary;
newly_stable_rendered_len = Some(newly_stable.len());
}
self.lines.truncate(self.stable_rendered_len);
if !self.lines.is_empty()
&& (!pending.lines.is_empty() || !pending_source.trim().is_empty())
&& !pending.first_top_level_block_is_html
{
self.lines.push(HyperlinkLine::new(Line::default()));
}
let pending_render_start = self.lines.len();
self.lines.extend(pending.lines);
if let Some(newly_stable_rendered_len) = newly_stable_rendered_len {
self.stable_rendered_len = pending_render_start + newly_stable_rendered_len;
}
}
}
pub(super) fn render_source(
source: &str,
width: Option<usize>,
cwd: &Path,
render_mode: HistoryRenderMode,
inline_visualization_context: Option<&InlineVisualizationContext>,
) -> Vec<HyperlinkLine> {
match render_mode {
HistoryRenderMode::Rich => render_markdown_agent_with_links_cwd_and_visualizations(
source,
width,
Some(cwd),
inline_visualization_context,
),
HistoryRenderMode::Raw => plain_hyperlink_lines(raw_lines_from_source(source)),
}
}
#[cfg(test)]
#[path = "render_tests.rs"]
mod tests;
codex-rs/tui/src/streaming/render_tests.rs
+333
View File
@@ -0,0 +1,333 @@
use super::StreamingRender;
use super::render_source;
use crate::history_cell::HistoryRenderMode;
use crate::inline_visualization::InlineVisualizationContext;
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
use crate::terminal_hyperlinks::HyperlinkLine;
use crate::terminal_hyperlinks::visible_lines;
use codex_protocol::ThreadId;
use insta::assert_debug_snapshot;
use pretty_assertions::assert_eq;
use std::path::Path;
use std::path::PathBuf;
fn test_cwd() -> PathBuf {
std::env::temp_dir()
}
fn append(
render: &mut StreamingRender,
source: &mut String,
chunk: &str,
width: Option<usize>,
cwd: &Path,
render_mode: HistoryRenderMode,
) {
source.push_str(chunk);
render.append(
source,
chunk,
width,
cwd,
render_mode,
/*inline_visualization_context*/ None,
);
}
fn append_rich_and_assert_matches_full(
render: &mut StreamingRender,
source: &mut String,
chunk: &str,
width: Option<usize>,
cwd: &Path,
) {
append(render, source, chunk, width, cwd, HistoryRenderMode::Rich);
assert_eq!(
render.lines,
render_source(
source,
width,
cwd,
HistoryRenderMode::Rich,
/*inline_visualization_context*/ None,
),
"incremental render diverged after chunk {chunk:?}",
);
}
fn lines_to_plain_strings(lines: &[HyperlinkLine]) -> Vec<String> {
visible_lines(lines.to_vec())
.iter()
.map(|line| {
line.spans
.iter()
.map(|span| span.content.as_ref())
.collect::<String>()
})
.collect()
}
fn assert_rich_stream_matches_full_render(
chunks: &[&str],
width: Option<usize>,
) -> (String, StreamingRender) {
let cwd = test_cwd();
let mut source = String::new();
let mut render = StreamingRender::new();
for chunk in chunks {
append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd);
}
(source, render)
}
#[test]
fn incremental_render_keeps_final_block_mutable_and_matches_full_render() {
let chunks = [
"# Heading\n",
"\n",
"First paragraph with a [link](https://example.com).\n",
"continued on the next line.\n\n",
"1. First item\n",
"2. Second item\n\n",
"> Quoted paragraph\n\n",
"```rust\n",
"fn main() {}\n",
"```\n\n",
"| Key | Value |\n",
"| --- | --- |\n",
"| alpha | beta |\n",
];
let (source, render) = assert_rich_stream_matches_full_render(&chunks, Some(48));
assert!(render.stable_source_len > 0);
assert!(render.stable_source_len < source.len());
assert_debug_snapshot!("incremental_render_representative_stream", render.lines);
}
#[test]
fn growing_single_top_level_blocks_render_and_scan_in_one_pass() {
let streams: &[&[&str]] = &[
&[
"A paragraph that keeps growing\n",
"without a blank line between chunks.\n",
"It stays one top-level block.\n",
],
&[
"| Key | Value |\n",
"| --- | --- |\n",
"| alpha | beta |\n",
"| gamma | delta |\n",
],
];
let cwd = test_cwd();
let width = Some(80);
for chunks in streams {
let mut source = String::new();
let mut render = StreamingRender::new();
for chunk in *chunks {
append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd);
let pending = render_streaming_markdown_agent_with_links_and_cwd(
&source,
width,
Some(cwd.as_path()),
);
assert_eq!(pending.last_top_level_block_start, None);
assert_eq!(render.lines, pending.lines);
assert_eq!(render.stable_source_len, 0);
}
}
}
#[test]
fn incremental_raw_render_preserves_blank_lines() {
let cwd = test_cwd();
let width = Some(80);
let mut source = String::new();
let mut render = StreamingRender::new();
for chunk in ["alpha\n", "\n", "beta\n", "\n"] {
append(
&mut render,
&mut source,
chunk,
width,
&cwd,
HistoryRenderMode::Raw,
);
}
assert_eq!(
lines_to_plain_strings(&render.lines),
vec!["alpha", "", "beta", ""],
);
}
#[test]
fn inline_visualizations_use_canonical_full_render() {
let cwd = test_cwd();
let context = InlineVisualizationContext::new(&cwd, ThreadId::new())
.expect("UUIDv7 thread id should provide a timestamp");
let width = Some(80);
let mut source = String::new();
let mut render = StreamingRender::new();
for chunk in ["Before.\n\n", "::codex-inline-vis{file=\"missing.html\"}\n"] {
source.push_str(chunk);
render.append(
&source,
chunk,
width,
&cwd,
HistoryRenderMode::Rich,
Some(&context),
);
assert_eq!(
render.lines,
render_source(
&source,
width,
&cwd,
HistoryRenderMode::Rich,
Some(&context),
),
);
assert_eq!(render.stable_source_len, 0);
}
}
#[test]
fn inline_visualizations_without_context_use_canonical_full_render() {
let (_, render) = assert_rich_stream_matches_full_render(
&["Before.\n\n", "::codex-inline-vis{file=\"missing.html\"}\n"],
Some(80),
);
assert_eq!(render.stable_source_len, 0);
assert_debug_snapshot!(
"inline_visualizations_without_context_use_canonical_full_render",
render.lines
);
}
#[test]
fn reference_link_definition_recomputes_earlier_and_later_blocks() {
let streams: &[&[&str]] = &[
&[
"Earlier [reference][id].\n\n",
"An unrelated paragraph.\n\n",
"[id]: https://example.com/reference\n",
"\n",
"Later [reference][id].\n",
],
&[
"Earlier [reference][id].\n\n",
"Another paragraph.\n\n",
"```markdown\n",
"| Key | Value |\n",
"| --- | --- |\n",
"| alpha | beta |\n",
"\n",
"[id]: https://example.com/reference\n",
"```\n",
],
];
for chunks in streams {
let (_, render) = assert_rich_stream_matches_full_render(chunks, Some(80));
assert!(render.has_reference_link_definition);
}
}
#[test]
fn reference_link_definition_survives_raw_to_rich_render_mode_switch() {
let cwd = test_cwd();
let width = Some(80);
let mut source = String::new();
let mut render = StreamingRender::new();
append(
&mut render,
&mut source,
"[id]: https://example.com/reference\n\n",
width,
&cwd,
HistoryRenderMode::Raw,
);
render.recompute(
&source,
width,
&cwd,
HistoryRenderMode::Rich,
/*inline_visualization_context*/ None,
);
for chunk in ["First [reference][id].\n\n", "Later [reference][id].\n"] {
append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd);
}
assert!(render.has_reference_link_definition);
}
#[test]
fn incremental_render_does_not_add_blank_line_before_html_block() {
let streams: &[&[&str]] = &[
&[
"Paragraph.\n\n",
"<div>x</div>\n\n",
"Following paragraph.\n",
],
&["Paragraph.\n\n<div>x</div>\n\n", "Following paragraph.\n"],
];
for chunks in streams {
let (_, render) = assert_rich_stream_matches_full_render(chunks, Some(80));
assert_eq!(
lines_to_plain_strings(&render.lines),
vec!["Paragraph.", "<div>x</div>", "", "Following paragraph.",],
);
}
}
#[test]
fn incremental_render_preserves_heading_and_normalized_html_seams() {
let streams: &[&[&str]] = &[
&["Paragraph.\n\n# Heading\n\n", "Next paragraph.\n"],
&[
"Paragraph.\n\n",
"<div>x</div>\n\n",
"```markdown\n",
"| Key | Value |\n",
"| --- | --- |\n",
"| alpha | beta |\n",
"\n",
"Following paragraph.\n",
"```\n",
],
];
for chunks in streams {
assert_rich_stream_matches_full_render(chunks, Some(80));
}
}
#[test]
fn paragraphs_after_unwrapped_table_fence_advance_stable_source() {
let cwd = test_cwd();
let width = Some(80);
let (mut source, mut render) = assert_rich_stream_matches_full_render(
&[
"```markdown\n",
"| Key | Value |\n",
"| --- | --- |\n",
"| alpha | beta |\n",
"```\n\n",
],
width,
);
let mut previous_stable_source_len = render.stable_source_len;
for block in ["<div>Post-fence HTML.</div>\n\n", "First paragraph.\n\n"] {
append_rich_and_assert_matches_full(&mut render, &mut source, block, width, &cwd);
assert!(render.stable_source_len > previous_stable_source_len);
previous_stable_source_len = render.stable_source_len;
}
}
codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__incremental_render_representative_stream.snap
+127
View File
@@ -0,0 +1,127 @@
---
source: tui/src/streaming/render_tests.rs
expression: render.lines
---
[
HyperlinkLine {
line: Line::from_iter([
Span::from("# ").bold().underlined(),
Span::from("Heading").bold().underlined(),
]),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from("First paragraph with a "),
Span::from("link"),
]),
hyperlinks: [
TerminalHyperlink {
columns: 23..27,
destination: "https://example.com",
destination_kind: Web,
},
],
},
HyperlinkLine {
line: Line::from_iter([
Span::from("("),
Span::from("https://example.com").cyan().underlined(),
Span::from(")"),
Span::from("."),
]),
hyperlinks: [
TerminalHyperlink {
columns: 1..20,
destination: "https://example.com",
destination_kind: Web,
},
],
},
HyperlinkLine {
line: Line::from("continued on the next line."),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from("1. ").light_blue(),
Span::from("First item"),
]),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from("2. ").light_blue(),
Span::from("Second item"),
]),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from("> "),
Span::from("Quoted paragraph"),
]).green(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::default(),
Span::from("fn").fg(Color::Rgb(203, 166, 247)),
Span::from(" ").fg(Color::Rgb(205, 214, 244)),
Span::from("main").fg(Color::Rgb(137, 180, 250)),
Span::from("(").fg(Color::Rgb(147, 153, 178)),
Span::from(")").fg(Color::Rgb(147, 153, 178)),
Span::from(" ").fg(Color::Rgb(205, 214, 244)),
Span::from("{").fg(Color::Rgb(147, 153, 178)),
Span::from("}").fg(Color::Rgb(147, 153, 178)),
]),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from(" "),
Span::from("Key"),
Span::from(" "),
Span::from(" "),
Span::from(" "),
Span::from(" "),
Span::from("Value"),
]).fg(Color::Rgb(249, 226, 175)).bold(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from(Span::from("━━━━━━━ ━━━━━━━").dim()),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from_iter([
Span::from(" "),
Span::from("alpha"),
Span::from(" "),
Span::from(" "),
Span::from(" "),
Span::from("beta"),
]),
hyperlinks: [],
},
]
codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__inline_visualizations_without_context_use_canonical_full_render.snap
+18
View File
@@ -0,0 +1,18 @@
---
source: tui/src/streaming/render_tests.rs
expression: render.lines
---
[
HyperlinkLine {
line: Line::from("Before."),
hyperlinks: [],
},
HyperlinkLine {
line: Line::default(),
hyperlinks: [],
},
HyperlinkLine {
line: Line::from(Span::from("Visualization unavailable on this device.").italic()),
hyperlinks: [],
},
]