Read compressed rollouts and materialize before append (#25087)

## Why

Local rollout compression needs a cold `.jsonl.zst` representation
without letting compressed physical paths leak into append-mode writers.
The unsafe case is resume or metadata update code successfully reading a
compressed rollout and then appending raw JSONL bytes to the zstd file.

This PR folds the former #25088 materialization slice into the
read-support PR so the reader changes and append-safety invariant land
together.

## What Changed

- Teach rollout readers, discovery, listing, search, and ID lookup to
understand compressed `.jsonl.zst` rollouts.
- Keep `.jsonl` as the logical/stored rollout path while allowing read
paths to open either plain or compressed storage.
- Materialize compressed rollouts back to plain `.jsonl` before
append-mode writes, including resume and direct metadata append paths.
- Preserve compressed-file permissions when materializing back to plain
JSONL.
- Refresh thread-store resolved rollout paths after compatibility
metadata writes so reconciliation follows the materialized file.
- Avoid treating transient compression temp files as real rollout lookup
results.

## Remaining Stack

#25089 remains the separate worker PR. It is based directly on this PR
and stays behind the disabled `local_thread_store_compression` feature
flag.

The worker still has a broader coordination question: a resume or
metadata update can race with background compression while a plain file
is being replaced by `.jsonl.zst`. This PR handles the read and
materialize-before-append primitives; it does not make the worker
production-ready.

## Validation

- `just test -p codex-rollout`
- `just test -p codex-thread-store`
- `just fix -p codex-rollout`
- `just fix -p codex-thread-store`
- `just bazel-lock-check`
This commit is contained in:
jif-oai authored and GitHub committed 2026-06-01 15:14:19 +02:00
1 parent f27bbbd49c
commit a8a6071279
16 files changed
+924 -131

No files matched your search

+26 -19
View File
@@ -30,6 +30,7 @@ use tracing::warn;
use super::ARCHIVED_SESSIONS_SUBDIR;
use super::SESSIONS_SUBDIR;
use super::compression;
use super::list::Cursor;
use super::list::SortDirection;
use super::list::ThreadItem;
@@ -699,17 +700,20 @@ impl RolloutRecorder {
(None, Some(log_file_info), path, Some(session_meta))
}
RolloutRecorderParams::Resume { path } => (
Some(
tokio::fs::OpenOptions::new()
.append(true)
.open(&path)
.await?,
),
None,
path,
None,
),
RolloutRecorderParams::Resume { path } => {
let path = compression::materialize_rollout_for_append(path.as_path()).await?;
(
Some(
tokio::fs::OpenOptions::new()
.append(true)
.open(&path)
.await?,
),
None,
path,
None,
)
}
};
// Clone the cwd for the spawned task to collect git info asynchronously
@@ -820,19 +824,17 @@ impl RolloutRecorder {
path: &Path,
) -> std::io::Result<(Vec<RolloutItem>, Option<ThreadId>, usize)> {
trace!("Resuming rollout from {path:?}");
let text = tokio::fs::read_to_string(path).await?;
if text.trim().is_empty() {
return Err(IoError::other("empty session file"));
}
let mut items: Vec<RolloutItem> = Vec::new();
let mut thread_id: Option<ThreadId> = None;
let mut parse_errors = 0usize;
for line in text.lines() {
let mut reader = compression::open_rollout_line_reader(path).await?;
let mut saw_non_empty_line = false;
while let Some(line) = reader.next_line().await? {
if line.trim().is_empty() {
continue;
}
let mut v: Value = match serde_json::from_str(line) {
saw_non_empty_line = true;
let mut v: Value = match serde_json::from_str(&line) {
Ok(v) => v,
Err(e) => {
warn!("failed to parse line as JSON: {line:?}, error: {e}");
@@ -875,6 +877,9 @@ impl RolloutRecorder {
}
}
}
if !saw_non_empty_line {
return Err(IoError::other("empty session file"));
}
tracing::debug!(
"Resumed rollout with {} items, thread ID: {:?}, parse errors: {}",
@@ -898,7 +903,7 @@ impl RolloutRecorder {
Ok(InitialHistory::Resumed(ResumedHistory {
conversation_id,
history: items,
rollout_path: Some(path.to_path_buf()),
rollout_path: Some(compression::plain_rollout_path(path)),
}))
}
@@ -1366,6 +1371,7 @@ fn precompute_log_file_info(
}
fn open_log_file(path: &Path) -> std::io::Result<File> {
let path = compression::materialize_rollout_for_append_blocking(path)?;
let Some(parent) = path.parent() else {
return Err(IoError::other(format!(
"rollout path has no parent: {}",
@@ -1625,6 +1631,7 @@ pub async fn append_rollout_item_to_path(
rollout_path: &Path,
item: &RolloutItem,
) -> std::io::Result<()> {
let rollout_path = compression::materialize_rollout_for_append(rollout_path).await?;
let file = tokio::fs::OpenOptions::new()
.append(true)
.open(rollout_path)