mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Extract reverse JSONL scanning from session indexing (#32246)
## What changed - Add a generic `ReverseJsonlScanner` that reads seekable JSONL data from the end in bounded chunks, skips blank records, and reports malformed records without stopping the scan. - Use the scanner for reverse session-index lookups while preserving the existing behavior of ignoring invalid entries. ## Testing - Add coverage for unterminated final records, malformed JSON, blank lines, chunk boundaries, and records spanning multiple chunks. - Verify session-index lookup can skip an invalid record and still find valid entries on either side of it. GitOrigin-RevId: 66255f48492f74726d4ac4831cdc82f748de17d6
This commit is contained in:
1 parent
414217dc8a
commit
a328f30172
5 files changed
+288
-57
No files matched your search
@@ -2,15 +2,14 @@ use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::fs::File;
|
||||
use std::io::ErrorKind;
|
||||
use std::io::Read;
|
||||
use std::io::Seek;
|
||||
use std::io::SeekFrom;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::reverse_jsonl_scanner::ReverseJsonlScanner;
|
||||
use crate::reverse_jsonl_scanner::ScanOutcome;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use serde::Deserialize;
|
||||
@@ -18,7 +17,6 @@ use serde::Serialize;
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
|
||||
const SESSION_INDEX_FILE: &str = "session_index.jsonl";
|
||||
const READ_CHUNK_SIZE: usize = 8192;
|
||||
static SESSION_INDEX_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
@@ -245,64 +243,18 @@ fn scan_index_from_end_for_each<F>(
|
||||
where
|
||||
F: FnMut(&SessionIndexEntry) -> std::io::Result<Option<SessionIndexEntry>>,
|
||||
{
|
||||
let mut file = File::open(path)?;
|
||||
let mut remaining = file.metadata()?.len();
|
||||
let mut line_rev: Vec<u8> = Vec::new();
|
||||
let mut buf = vec![0u8; READ_CHUNK_SIZE];
|
||||
|
||||
while remaining > 0 {
|
||||
let read_size = usize::try_from(remaining.min(READ_CHUNK_SIZE as u64))
|
||||
.map_err(std::io::Error::other)?;
|
||||
remaining -= read_size as u64;
|
||||
file.seek(SeekFrom::Start(remaining))?;
|
||||
file.read_exact(&mut buf[..read_size])?;
|
||||
|
||||
for &byte in buf[..read_size].iter().rev() {
|
||||
if byte == b'\n' {
|
||||
if let Some(entry) = parse_line_from_rev(&mut line_rev, &mut visit_entry)? {
|
||||
return Ok(Some(entry));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
line_rev.push(byte);
|
||||
let mut scanner = ReverseJsonlScanner::new(File::open(path)?)?;
|
||||
while let Some(outcome) = scanner.scan_next::<SessionIndexEntry>()? {
|
||||
let ScanOutcome::Parsed(entry) = outcome else {
|
||||
continue;
|
||||
};
|
||||
if let Some(entry) = visit_entry(&entry)? {
|
||||
return Ok(Some(entry));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(entry) = parse_line_from_rev(&mut line_rev, &mut visit_entry)? {
|
||||
return Ok(Some(entry));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn parse_line_from_rev<F>(
|
||||
line_rev: &mut Vec<u8>,
|
||||
visit_entry: &mut F,
|
||||
) -> std::io::Result<Option<SessionIndexEntry>>
|
||||
where
|
||||
F: FnMut(&SessionIndexEntry) -> std::io::Result<Option<SessionIndexEntry>>,
|
||||
{
|
||||
if line_rev.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
line_rev.reverse();
|
||||
let line = std::mem::take(line_rev);
|
||||
let Ok(mut line) = String::from_utf8(line) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if line.ends_with('\r') {
|
||||
line.pop();
|
||||
}
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Ok(entry) = serde_json::from_str::<SessionIndexEntry>(trimmed) else {
|
||||
return Ok(None);
|
||||
};
|
||||
visit_entry(&entry)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "session_index_tests.rs"]
|
||||
mod tests;
|
||||
Reference in new issue
Block a user