Include shared histories in rollout compression (#42039)

## What changed

- Make `local_thread_store_compression` compress cold rollout files across shared and forked histories without a separate compression mode.
- Retire `local_thread_store_shared_compression` while continuing to accept it in strict configuration without changing compression behavior.
- Read rollout files through the compressed-rollout reader when `codex exec resume` determines the latest turn's working directory.

## Testing

- Cover compression and restoration of an archived fork chain.
- Cover cwd-based resume selection when only the compressed rollout remains.
- Cover the removed compatibility key in strict configuration.

GitOrigin-RevId: 8edebf61eccc1f79cabb8e641f8982c76f337bf1
This commit is contained in:
jif authored and copyberry committed 2026-09-01 12:23:20 +00:00
1 parent 2e5ee418ad
commit 9d0eae74cd
13 files changed
+137 -305

No files matched your search

+9 -50
View File
@@ -21,22 +21,13 @@ const OPEN_ROLLOUT_LINE_READER_RETRY_DELAY: Duration = Duration::from_millis(50)
const TEMP_SUFFIX: &str = ".tmp";
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
/// Whether cold files belonging to shared rollout lineages may be compressed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RolloutCompressionMode {
/// Preserve compatibility with readers that require shared lineages to remain plain JSONL.
Standalone,
/// Requires every reader of this Codex home to support compressed shared lineages.
IncludeShared,
}
/// Starts a best-effort background job that compresses cold local rollout files.
///
/// The worker is fire-and-forget: failures are logged, startup is not blocked,
/// and a run marker under `codex_home` prevents overlapping or too-frequent
/// compression runs from the same local store.
pub fn spawn_rollout_compression_worker(codex_home: PathBuf, mode: RolloutCompressionMode) {
worker::spawn(codex_home, mode)
pub fn spawn_rollout_compression_worker(codex_home: PathBuf) {
worker::spawn(codex_home)
}
/// Returns the modified time for the existing plain or compressed rollout file.
@@ -253,10 +244,8 @@ mod worker {
use tokio::task::JoinSet;
use crate::ARCHIVED_SESSIONS_SUBDIR;
use crate::RolloutReferenceIndex;
use crate::SESSIONS_SUBDIR;
use super::RolloutCompressionMode;
use super::RolloutFile;
use super::metrics;
use super::path;
@@ -334,7 +323,7 @@ mod worker {
}
}
pub(super) fn spawn(codex_home: PathBuf, mode: RolloutCompressionMode) {
pub(super) fn spawn(codex_home: PathBuf) {
let Ok(handle) = tokio::runtime::Handle::try_current() else {
metrics::run("skipped_no_runtime");
warn!(
@@ -344,7 +333,7 @@ mod worker {
return;
};
handle.spawn(async move {
if let Err(err) = run(codex_home.clone(), mode).await {
if let Err(err) = run(codex_home.clone()).await {
warn!(
"rollout compression worker failed for {}: {err}",
codex_home.display()
@@ -353,7 +342,7 @@ mod worker {
});
}
pub(super) async fn run(codex_home: PathBuf, mode: RolloutCompressionMode) -> io::Result<()> {
pub(super) async fn run(codex_home: PathBuf) -> io::Result<()> {
let Some(_maintenance_guard) =
crate::try_acquire_rollout_maintenance_lock(codex_home.as_path())?
else {
@@ -384,15 +373,6 @@ mod worker {
let started_at = Instant::now();
let result = async {
cleanup_stale_temps(codex_home.as_path()).await?;
let Some(reference_index) = RolloutReferenceIndex::scan_until(
codex_home.as_path(),
started_at,
WORKER_MAX_RUNTIME,
)
.await?
else {
return Ok(CompressionStats::default());
};
let mut stats = CompressionStats::default();
for root in [
codex_home.join(ARCHIVED_SESSIONS_SUBDIR),
@@ -401,14 +381,7 @@ mod worker {
if started_at.elapsed() >= WORKER_MAX_RUNTIME {
break;
}
compress_rollouts_in_root(
root.as_path(),
started_at,
&reference_index,
mode,
&mut stats,
)
.await?;
compress_rollouts_in_root(root.as_path(), started_at, &mut stats).await?;
}
Ok::<_, io::Error>(stats)
}
@@ -448,8 +421,6 @@ mod worker {
async fn compress_rollouts_in_root(
root: &Path,
started_at: Instant,
reference_index: &RolloutReferenceIndex,
mode: RolloutCompressionMode,
stats: &mut CompressionStats,
) -> io::Result<()> {
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
@@ -508,26 +479,14 @@ mod worker {
continue;
}
let path = rollout_file.into_path();
let Some(rollout_id) = crate::rollout_id_from_path(path.as_path()) else {
if crate::rollout_id_from_path(path.as_path()).is_none() {
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_unreadable_meta");
continue;
};
let Ok(meta) = crate::read_session_meta_line(path.as_path()).await else {
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_unreadable_meta");
continue;
};
if mode == RolloutCompressionMode::Standalone
&& reference_index.reference_count(rollout_id) > 0
{
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_referenced");
continue;
}
if mode == RolloutCompressionMode::Standalone && meta.meta.history_base.is_some() {
if crate::read_session_meta_line(path.as_path()).await.is_err() {
stats.skipped = stats.skipped.saturating_add(1);
metrics::file("skipped_fork_pointer");
metrics::file("skipped_unreadable_meta");
continue;
}
stats.scanned = stats.scanned.saturating_add(1);