Support compression for shared rollout lineages (#41357)

## Why

Rollout compression skips files that participate in paginated fork lineages because lineage readers rely on byte offsets into the original JSONL. This prevents shared rollout history from being compressed.

## What changed

- Add a seekable rollout reader that preserves logical JSONL offsets for plain and zstd-compressed files without materializing compressed data beside the rollout.
- Use the reader for lineage validation, model-context replay, and SQLite history projection, including when compression races with a read.
- Add the opt-in `local_thread_store_shared_compression` feature. The default mode continues to leave shared lineages uncompressed for compatibility, while the new mode includes referenced sources and fork pointers.
- Record the uncompressed size in new zstd frames so common prefix-bound checks can avoid decoding the full file.

## Testing

Add coverage for seekable compressed reads, prefix validation across zstd frame shapes, compressed lineage replay and projection, concurrent forks, and compression-mode behavior.

GitOrigin-RevId: 83e519f7eabf7448977eff37c55899858cc17a07
This commit is contained in:
jif authored and copyberry committed 2026-08-28 16:04:26 +00:00
1 parent 39507eea53
commit 1cc81ca89a
14 files changed
+493 -111

No files matched your search

+30 -9
View File
@@ -21,13 +21,22 @@ const OPEN_ROLLOUT_LINE_READER_RETRY_DELAY: Duration = Duration::from_millis(50)
const TEMP_SUFFIX: &str = ".tmp";
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
/// Whether cold files belonging to shared rollout lineages may be compressed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RolloutCompressionMode {
/// Preserve compatibility with readers that require shared lineages to remain plain JSONL.
Standalone,
/// Requires every reader of this Codex home to support compressed shared lineages.
IncludeShared,
}
/// Starts a best-effort background job that compresses cold local rollout files.
///
/// The worker is fire-and-forget: failures are logged, startup is not blocked,
/// and a run marker under `codex_home` prevents overlapping or too-frequent
/// compression runs from the same local store.
pub fn spawn_rollout_compression_worker(codex_home: PathBuf) {
worker::spawn(codex_home)
pub fn spawn_rollout_compression_worker(codex_home: PathBuf, mode: RolloutCompressionMode) {
worker::spawn(codex_home, mode)
}
/// Returns the modified time for the existing plain or compressed rollout file.
@@ -247,6 +256,7 @@ mod worker {
use crate::RolloutReferenceIndex;
use crate::SESSIONS_SUBDIR;
use super::RolloutCompressionMode;
use super::RolloutFile;
use super::metrics;
use super::path;
@@ -324,7 +334,7 @@ mod worker {
}
}
pub(super) fn spawn(codex_home: PathBuf) {
pub(super) fn spawn(codex_home: PathBuf, mode: RolloutCompressionMode) {
let Ok(handle) = tokio::runtime::Handle::try_current() else {
metrics::run("skipped_no_runtime");
warn!(
@@ -334,7 +344,7 @@ mod worker {
return;
};
handle.spawn(async move {
if let Err(err) = run(codex_home.clone()).await {
if let Err(err) = run(codex_home.clone(), mode).await {
warn!(
"rollout compression worker failed for {}: {err}",
codex_home.display()
@@ -343,7 +353,7 @@ mod worker {
});
}
pub(super) async fn run(codex_home: PathBuf) -> io::Result<()> {
pub(super) async fn run(codex_home: PathBuf, mode: RolloutCompressionMode) -> io::Result<()> {
let Some(_maintenance_guard) =
crate::try_acquire_rollout_maintenance_lock(codex_home.as_path())?
else {
@@ -391,8 +401,14 @@ mod worker {
if started_at.elapsed() >= WORKER_MAX_RUNTIME {
break;
}
compress_rollouts_in_root(root.as_path(), started_at, &reference_index, &mut stats)
.await?;
compress_rollouts_in_root(
root.as_path(),
started_at,
&reference_index,
mode,
&mut stats,
)
.await?;
}
Ok::<_, io::Error>(stats)
}
@@ -433,6 +449,7 @@ mod worker {
root: &Path,
started_at: Instant,
reference_index: &RolloutReferenceIndex,
mode: RolloutCompressionMode,
stats: &mut CompressionStats,
) -> io::Result<()> {
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
@@ -501,12 +518,14 @@ mod worker {
metrics::file("skipped_unreadable_meta");
continue;
};
if reference_index.reference_count(rollout_id) > 0 {
if mode == RolloutCompressionMode::Standalone
&& reference_index.reference_count(rollout_id) > 0
{
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_referenced");
continue;
}
if meta.meta.history_base.is_some() {
if mode == RolloutCompressionMode::Standalone && meta.meta.history_base.is_some() {
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_fork_pointer");
continue;
@@ -745,6 +764,8 @@ mod worker {
fn encode_zstd_to_writer(source: &Path, output: impl Write) -> io::Result<()> {
let mut input = File::open(source)?;
let mut encoder = zstd::stream::write::Encoder::new(output, COMPRESSION_LEVEL)?;
// Preserve fast byte-bound checks for paginated history without decoding the whole file.
encoder.set_pledged_src_size(Some(input.metadata()?.len()))?;
io::copy(&mut input, &mut encoder)?;
encoder.finish()?;
Ok(())