Read compressed rollouts and materialize before append (#25087)

## Why

Local rollout compression needs a cold `.jsonl.zst` representation
without letting compressed physical paths leak into append-mode writers.
The unsafe case is resume or metadata update code successfully reading a
compressed rollout and then appending raw JSONL bytes to the zstd file.

This PR folds the former #25088 materialization slice into the
read-support PR so the reader changes and append-safety invariant land
together.

## What Changed

- Teach rollout readers, discovery, listing, search, and ID lookup to
understand compressed `.jsonl.zst` rollouts.
- Keep `.jsonl` as the logical/stored rollout path while allowing read
paths to open either plain or compressed storage.
- Materialize compressed rollouts back to plain `.jsonl` before
append-mode writes, including resume and direct metadata append paths.
- Preserve compressed-file permissions when materializing back to plain
JSONL.
- Refresh thread-store resolved rollout paths after compatibility
metadata writes so reconciliation follows the materialized file.
- Avoid treating transient compression temp files as real rollout lookup
results.

## Remaining Stack

#25089 remains the separate worker PR. It is based directly on this PR
and stays behind the disabled `local_thread_store_compression` feature
flag.

The worker still has a broader coordination question: a resume or
metadata update can race with background compression while a plain file
is being replaced by `.jsonl.zst`. This PR handles the read and
materialize-before-append primitives; it does not make the worker
production-ready.

## Validation

- `just test -p codex-rollout`
- `just test -p codex-thread-store`
- `just fix -p codex-rollout`
- `just fix -p codex-thread-store`
- `just bazel-lock-check`

codex-rs/rollout/src/compression_tests.rs

@@ -0,0 +1,294 @@
+use std::fs;
+#[cfg(unix)]
+use std::os::unix::fs::PermissionsExt;
+
+use codex_protocol::ThreadId;
+use codex_protocol::protocol::EventMsg;
+use codex_protocol::protocol::InitialHistory;
+use codex_protocol::protocol::RolloutItem;
+use codex_protocol::protocol::RolloutLine;
+use codex_protocol::protocol::SessionMeta;
+use codex_protocol::protocol::SessionMetaLine;
+use codex_protocol::protocol::SessionSource;
+use codex_protocol::protocol::UserMessageEvent;
+use pretty_assertions::assert_eq;
+use tempfile::TempDir;
+use uuid::Uuid;
+
+use super::*;
+use crate::RolloutConfig;
+use crate::RolloutRecorder;
+use crate::RolloutRecorderParams;
+use crate::append_rollout_item_to_path;
+
+#[tokio::test]
+async fn load_rollout_items_reads_compressed_rollout() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(1);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "hello compressed")?;
+    compress_now(&rollout_path)?;
+
+    let (items, loaded_thread_id, parse_errors) =
+        RolloutRecorder::load_rollout_items(&rollout_path).await?;
+
+    assert_eq!(loaded_thread_id, Some(thread_id));
+    assert_eq!(parse_errors, 0);
+    assert_eq!(items.len(), 2);
+    assert!(!rollout_path.exists());
+    assert!(compressed_rollout_path(&rollout_path).exists());
+    Ok(())
+}
+
+#[test]
+fn rollout_file_from_path_normalizes_compressed_file_names() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(7);
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    let compressed_path = compressed_rollout_path(&rollout_path);
+
+    assert_eq!(
+        RolloutFile::from_path(compressed_path.clone()),
+        Some(RolloutFile {
+            path: compressed_path,
+            plain_file_name: format!("rollout-2025-01-03T12-00-00-{uuid}.jsonl"),
+        })
+    );
+    Ok(())
+}
+
+#[test]
+fn rollout_file_from_path_hides_compressed_sibling_when_plain_exists() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(8);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "plain wins")?;
+
+    assert_eq!(
+        RolloutFile::from_path(compressed_rollout_path(&rollout_path)),
+        None
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn append_rollout_item_materializes_compressed_rollout() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(2);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "hello before append")?;
+    compress_now(&rollout_path)?;
+
+    append_rollout_item_to_path(
+        &rollout_path,
+        &RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
+            message: "hello after append".to_string(),
+            ..Default::default()
+        })),
+    )
+    .await?;
+
+    assert!(rollout_path.exists());
+    assert!(!compressed_rollout_path(&rollout_path).exists());
+    let (items, loaded_thread_id, parse_errors) =
+        RolloutRecorder::load_rollout_items(&rollout_path).await?;
+    assert_eq!(loaded_thread_id, Some(thread_id));
+    assert_eq!(parse_errors, 0);
+    assert_eq!(items.len(), 3);
+    Ok(())
+}
+
+#[tokio::test]
+async fn resume_materializes_compressed_rollout_path() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let config = RolloutConfig {
+        codex_home: home.path().to_path_buf(),
+        sqlite_home: home.path().to_path_buf(),
+        cwd: home.path().to_path_buf(),
+        model_provider_id: "test-provider".to_string(),
+        generate_memories: true,
+    };
+    let uuid = Uuid::from_u128(3);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "hello before resume")?;
+    compress_now(&rollout_path)?;
+    let compressed_path = compressed_rollout_path(&rollout_path);
+
+    let InitialHistory::Resumed(history) =
+        RolloutRecorder::get_rollout_history(compressed_path.as_path()).await?
+    else {
+        panic!("expected compressed rollout to load as resumed history");
+    };
+    assert_eq!(history.rollout_path, Some(rollout_path.clone()));
+
+    let recorder = RolloutRecorder::new(
+        &config,
+        RolloutRecorderParams::resume(compressed_path.clone()),
+    )
+    .await?;
+
+    assert_eq!(recorder.rollout_path(), rollout_path.as_path());
+    assert!(rollout_path.exists());
+    assert!(!compressed_path.exists());
+    recorder
+        .record_canonical_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
+            UserMessageEvent {
+                message: "hello after resume".to_string(),
+                ..Default::default()
+            },
+        ))])
+        .await?;
+    recorder.flush().await?;
+    recorder.shutdown().await?;
+
+    let (items, loaded_thread_id, parse_errors) =
+        RolloutRecorder::load_rollout_items(&rollout_path).await?;
+    assert_eq!(loaded_thread_id, Some(thread_id));
+    assert_eq!(parse_errors, 0);
+    assert_eq!(items.len(), 3);
+    Ok(())
+}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn append_materialization_preserves_compressed_rollout_permissions() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(6);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "restricted transcript")?;
+    compress_now(&rollout_path)?;
+    let compressed_path = compressed_rollout_path(&rollout_path);
+    fs::set_permissions(&compressed_path, fs::Permissions::from_mode(0o600))?;
+
+    append_rollout_item_to_path(
+        &rollout_path,
+        &RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
+            message: "materialize restricted transcript".to_string(),
+            ..Default::default()
+        })),
+    )
+    .await?;
+
+    assert!(rollout_path.exists());
+    assert!(!compressed_path.exists());
+    assert_eq!(
+        fs::metadata(&rollout_path)?.permissions().mode() & 0o777,
+        0o600
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn find_thread_path_by_id_handles_compressed_rollout_filenames() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(8);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid);
+    write_rollout(&rollout_path, thread_id, "compressed filename lookup")?;
+    compress_now(&rollout_path)?;
+    let compressed_path = compressed_rollout_path(&rollout_path);
+
+    assert_eq!(
+        crate::find_thread_path_by_id_str(
+            home.path(),
+            &uuid.to_string(),
+            /*state_db_ctx*/ None
+        )
+        .await?,
+        Some(compressed_path)
+    );
+    assert_eq!(
+        crate::find_thread_path_by_id_str(home.path(), "not-a-uuid", /*state_db_ctx*/ None).await?,
+        None
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn find_thread_path_by_id_ignores_compression_temp_matches() -> anyhow::Result<()> {
+    let home = TempDir::new()?;
+    let uuid = Uuid::from_u128(9);
+    let thread_id = ThreadId::from_string(&uuid.to_string())?;
+    let temp_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid).with_file_name(format!(
+        "rollout-2025-01-03T12-00-00-{uuid}.jsonl.zst.compress.1.0.tmp"
+    ));
+    write_rollout(&temp_path, thread_id, "temporary file should not resolve")?;
+
+    assert_eq!(
+        crate::find_thread_path_by_id_str(
+            home.path(),
+            &uuid.to_string(),
+            /*state_db_ctx*/ None
+        )
+        .await?,
+        None
+    );
+    Ok(())
+}
+
+fn rollout_path(home: &std::path::Path, ts: &str, uuid: Uuid) -> std::path::PathBuf {
+    home.join("sessions/2025/01/03")
+        .join(format!("rollout-{ts}-{uuid}.jsonl"))
+}
+
+fn write_rollout(path: &std::path::Path, thread_id: ThreadId, message: &str) -> anyhow::Result<()> {
+    let parent = path.parent().expect("rollout path should have parent");
+    fs::create_dir_all(parent)?;
+    let session_meta_line = SessionMetaLine {
+        meta: SessionMeta {
+            id: thread_id,
+            forked_from_id: None,
+            timestamp: "2025-01-03T12:00:00Z".to_string(),
+            cwd: parent.to_path_buf(),
+            originator: "test".to_string(),
+            cli_version: "test".to_string(),
+            source: SessionSource::Cli,
+            thread_source: None,
+            agent_path: None,
+            agent_nickname: None,
+            agent_role: None,
+            model_provider: None,
+            base_instructions: None,
+            dynamic_tools: None,
+            memory_mode: None,
+        },
+        git: None,
+    };
+    let lines = [
+        RolloutLine {
+            timestamp: "2025-01-03T12:00:00Z".to_string(),
+            item: RolloutItem::SessionMeta(session_meta_line),
+        },
+        RolloutLine {
+            timestamp: "2025-01-03T12:00:01Z".to_string(),
+            item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
+                message: message.to_string(),
+                ..Default::default()
+            })),
+        },
+    ];
+    let jsonl = lines
+        .iter()
+        .map(serde_json::to_string)
+        .collect::<Result<Vec<_>, _>>()?
+        .join("\n");
+    fs::write(path, format!("{jsonl}\n"))?;
+    Ok(())
+}
+
+fn compress_now(path: &std::path::Path) -> anyhow::Result<()> {
+    let compressed_path = compressed_rollout_path(path);
+    let input = fs::File::open(path)?;
+    let output = fs::File::create(compressed_path)?;
+    let mut encoder = zstd::stream::write::Encoder::new(output, 3)?;
+    let mut input = std::io::BufReader::new(input);
+    std::io::copy(&mut input, &mut encoder)?;
+    encoder.finish()?;
+    fs::remove_file(path)?;
+    Ok(())
+}