Extract reverse JSONL scanning from session indexing (#32246)

## What changed

- Add a generic `ReverseJsonlScanner` that reads seekable JSONL data from the
  end in bounded chunks, skips blank records, and reports malformed records
  without stopping the scan.
- Use the scanner for reverse session-index lookups while preserving the
  existing behavior of ignoring invalid entries.

## Testing

- Add coverage for unterminated final records, malformed JSON, blank lines,
  chunk boundaries, and records spanning multiple chunks.
- Verify session-index lookup can skip an invalid record and still find valid
  entries on either side of it.

GitOrigin-RevId: 66255f48492f74726d4ac4831cdc82f748de17d6
This commit is contained in:
Owen Lin authored and copyberry committed 2026-07-10 19:18:39 +00:00
1 parent 414217dc8a
commit a328f30172
5 files changed
+288 -57

No files matched your search

+9 -57
View File
@@ -2,15 +2,14 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::fs::File;
use std::io::ErrorKind;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::sync::LazyLock;
use std::sync::Mutex;
use crate::reverse_jsonl_scanner::ReverseJsonlScanner;
use crate::reverse_jsonl_scanner::ScanOutcome;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionMetaLine;
use serde::Deserialize;
@@ -18,7 +17,6 @@ use serde::Serialize;
use tokio::io::AsyncBufReadExt;
const SESSION_INDEX_FILE: &str = "session_index.jsonl";
const READ_CHUNK_SIZE: usize = 8192;
static SESSION_INDEX_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -245,64 +243,18 @@ fn scan_index_from_end_for_each<F>(
where
F: FnMut(&SessionIndexEntry) -> std::io::Result<Option<SessionIndexEntry>>,
{
let mut file = File::open(path)?;
let mut remaining = file.metadata()?.len();
let mut line_rev: Vec<u8> = Vec::new();
let mut buf = vec![0u8; READ_CHUNK_SIZE];
while remaining > 0 {
let read_size = usize::try_from(remaining.min(READ_CHUNK_SIZE as u64))
.map_err(std::io::Error::other)?;
remaining -= read_size as u64;
file.seek(SeekFrom::Start(remaining))?;
file.read_exact(&mut buf[..read_size])?;
for &byte in buf[..read_size].iter().rev() {
if byte == b'\n' {
if let Some(entry) = parse_line_from_rev(&mut line_rev, &mut visit_entry)? {
return Ok(Some(entry));
}
continue;
}
line_rev.push(byte);
let mut scanner = ReverseJsonlScanner::new(File::open(path)?)?;
while let Some(outcome) = scanner.scan_next::<SessionIndexEntry>()? {
let ScanOutcome::Parsed(entry) = outcome else {
continue;
};
if let Some(entry) = visit_entry(&entry)? {
return Ok(Some(entry));
}
}
if let Some(entry) = parse_line_from_rev(&mut line_rev, &mut visit_entry)? {
return Ok(Some(entry));
}
Ok(None)
}
fn parse_line_from_rev<F>(
line_rev: &mut Vec<u8>,
visit_entry: &mut F,
) -> std::io::Result<Option<SessionIndexEntry>>
where
F: FnMut(&SessionIndexEntry) -> std::io::Result<Option<SessionIndexEntry>>,
{
if line_rev.is_empty() {
return Ok(None);
}
line_rev.reverse();
let line = std::mem::take(line_rev);
let Ok(mut line) = String::from_utf8(line) else {
return Ok(None);
};
if line.ends_with('\r') {
line.pop();
}
let trimmed = line.trim();
if trimmed.is_empty() {
return Ok(None);
}
let Ok(entry) = serde_json::from_str::<SessionIndexEntry>(trimmed) else {
return Ok(None);
};
visit_entry(&entry)
}
#[cfg(test)]
#[path = "session_index_tests.rs"]
mod tests;