mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Read compressed rollouts and materialize before append (#25087)
## Why Local rollout compression needs a cold `.jsonl.zst` representation without letting compressed physical paths leak into append-mode writers. The unsafe case is resume or metadata update code successfully reading a compressed rollout and then appending raw JSONL bytes to the zstd file. This PR folds the former #25088 materialization slice into the read-support PR so the reader changes and append-safety invariant land together. ## What Changed - Teach rollout readers, discovery, listing, search, and ID lookup to understand compressed `.jsonl.zst` rollouts. - Keep `.jsonl` as the logical/stored rollout path while allowing read paths to open either plain or compressed storage. - Materialize compressed rollouts back to plain `.jsonl` before append-mode writes, including resume and direct metadata append paths. - Preserve compressed-file permissions when materializing back to plain JSONL. - Refresh thread-store resolved rollout paths after compatibility metadata writes so reconciliation follows the materialized file. - Avoid treating transient compression temp files as real rollout lookup results. ## Remaining Stack #25089 remains the separate worker PR. It is based directly on this PR and stays behind the disabled `local_thread_store_compression` feature flag. The worker still has a broader coordination question: a resume or metadata update can race with background compression while a plain file is being replaced by `.jsonl.zst`. This PR handles the read and materialize-before-append primitives; it does not make the worker production-ready. ## Validation - `just test -p codex-rollout` - `just test -p codex-thread-store` - `just fix -p codex-rollout` - `just fix -p codex-thread-store` - `just bazel-lock-check`
This commit is contained in:
1 parent
f27bbbd49c
commit
a8a6071279
16 files changed
+924
-131
No files matched your search
@@ -30,6 +30,7 @@ use tracing::warn;
|
||||
|
||||
use super::ARCHIVED_SESSIONS_SUBDIR;
|
||||
use super::SESSIONS_SUBDIR;
|
||||
use super::compression;
|
||||
use super::list::Cursor;
|
||||
use super::list::SortDirection;
|
||||
use super::list::ThreadItem;
|
||||
@@ -699,17 +700,20 @@ impl RolloutRecorder {
|
||||
|
||||
(None, Some(log_file_info), path, Some(session_meta))
|
||||
}
|
||||
RolloutRecorderParams::Resume { path } => (
|
||||
Some(
|
||||
tokio::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.await?,
|
||||
),
|
||||
None,
|
||||
path,
|
||||
None,
|
||||
),
|
||||
RolloutRecorderParams::Resume { path } => {
|
||||
let path = compression::materialize_rollout_for_append(path.as_path()).await?;
|
||||
(
|
||||
Some(
|
||||
tokio::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.await?,
|
||||
),
|
||||
None,
|
||||
path,
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
// Clone the cwd for the spawned task to collect git info asynchronously
|
||||
@@ -820,19 +824,17 @@ impl RolloutRecorder {
|
||||
path: &Path,
|
||||
) -> std::io::Result<(Vec<RolloutItem>, Option<ThreadId>, usize)> {
|
||||
trace!("Resuming rollout from {path:?}");
|
||||
let text = tokio::fs::read_to_string(path).await?;
|
||||
if text.trim().is_empty() {
|
||||
return Err(IoError::other("empty session file"));
|
||||
}
|
||||
|
||||
let mut items: Vec<RolloutItem> = Vec::new();
|
||||
let mut thread_id: Option<ThreadId> = None;
|
||||
let mut parse_errors = 0usize;
|
||||
for line in text.lines() {
|
||||
let mut reader = compression::open_rollout_line_reader(path).await?;
|
||||
let mut saw_non_empty_line = false;
|
||||
while let Some(line) = reader.next_line().await? {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut v: Value = match serde_json::from_str(line) {
|
||||
saw_non_empty_line = true;
|
||||
let mut v: Value = match serde_json::from_str(&line) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!("failed to parse line as JSON: {line:?}, error: {e}");
|
||||
@@ -875,6 +877,9 @@ impl RolloutRecorder {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !saw_non_empty_line {
|
||||
return Err(IoError::other("empty session file"));
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
"Resumed rollout with {} items, thread ID: {:?}, parse errors: {}",
|
||||
@@ -898,7 +903,7 @@ impl RolloutRecorder {
|
||||
Ok(InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id,
|
||||
history: items,
|
||||
rollout_path: Some(path.to_path_buf()),
|
||||
rollout_path: Some(compression::plain_rollout_path(path)),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -1366,6 +1371,7 @@ fn precompute_log_file_info(
|
||||
}
|
||||
|
||||
fn open_log_file(path: &Path) -> std::io::Result<File> {
|
||||
let path = compression::materialize_rollout_for_append_blocking(path)?;
|
||||
let Some(parent) = path.parent() else {
|
||||
return Err(IoError::other(format!(
|
||||
"rollout path has no parent: {}",
|
||||
@@ -1625,6 +1631,7 @@ pub async fn append_rollout_item_to_path(
|
||||
rollout_path: &Path,
|
||||
item: &RolloutItem,
|
||||
) -> std::io::Result<()> {
|
||||
let rollout_path = compression::materialize_rollout_for_append(rollout_path).await?;
|
||||
let file = tokio::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(rollout_path)
|
||||
|
||||
Reference in new issue
Block a user