mirror of
https://github.com/openai/codex.git
synced 2026-11-20 07:06:24 +00:00
Compress cold local rollouts (#25089)
## Rollout compression stack This stack splits #24941 into reviewable steps for local rollout compression. The design is intentionally staged: 1. Teach readers, listing, search, and lookup to understand compressed rollouts. 2. Make append and resume paths materialize compressed rollouts back to plain JSONL before writing. 3. Add a disabled-by-default worker that can compress cold archived rollouts behind `local_thread_store_compression`. The key invariant is that writers append to plain `.jsonl`. A `.jsonl.zst` file is a cold/read representation; if a write is needed, the compressed file is materialized back to plain JSONL first. Readers prefer plain `.jsonl` when both forms exist and can fall back to the compressed sibling during transitions. The worker is deliberately the last PR and remains behind an under-development feature flag. It currently scans only `archived_sessions`, not active `sessions`, because active sessions have the highest resume/append race risk. That means this stack does not yet compress most unarchived local history. ## Known race / follow-up The remaining unresolved design question is writer/compressor coordination. Even for archived rollouts, a resume or metadata update can append while the worker is replacing the plain file with `.jsonl.zst`; the current double-stat checks narrow but do not fully eliminate the window where a writer has opened the plain file before unlink. Do not treat the worker PR as production-ready until we either: - prevent append/resume paths from racing archived compression, or - introduce a shared representation/append lock or equivalent coordination. The first two PRs are useful independently: they make compressed rollouts readable and make append paths safely recover back to plain JSONL. The third PR isolates the worker behavior so that coordination issue is reviewable separately. ## Validation Focused local validation for the stack includes: - `just test -p codex-rollout` - `just test -p codex-thread-store` where thread-store paths were touched - `just test -p codex-features` for the feature flag slice - `just bazel-lock-check` after dependency graph changes - scoped `just fix -p ...` passes for changed crates CI is still the source of truth for the full platform matrix. ## This PR in the stack This is PR 3/3, based on #25088. It adds the under-development feature flag and starts the best-effort background worker when enabled. The worker currently compresses only cold archived rollouts, skips active sessions, verifies compressed output, preserves mtime and permissions, keeps a store-level lock heartbeat, and cleans stale temp files. Stack order: 1. #25087: read compressed local rollouts. 2. #25088: materialize compressed rollouts before append. 3. This PR: add the disabled local compression worker.
This commit is contained in:
1 parent
3cdce52865
commit
01cb97851b
6 files changed
+599
-4
No files matched your search
@@ -21,6 +21,15 @@ const OPEN_ROLLOUT_LINE_READER_RETRY_DELAY: Duration = Duration::from_millis(50)
|
||||
const TEMP_SUFFIX: &str = ".tmp";
|
||||
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Starts a best-effort background job that compresses cold local rollout files.
|
||||
///
|
||||
/// The worker is fire-and-forget: failures are logged, startup is not blocked,
|
||||
/// and a process-wide lock under `codex_home` prevents overlapping compression
|
||||
/// runs from the same local store.
|
||||
pub fn spawn_rollout_compression_worker(codex_home: PathBuf) {
|
||||
worker::spawn(codex_home)
|
||||
}
|
||||
|
||||
/// Returns the modified time for the existing plain or compressed rollout file.
|
||||
pub(crate) async fn file_modified_time(path: &Path) -> io::Result<Option<time::OffsetDateTime>> {
|
||||
let Some(path) = path::existing_rollout_path(path).await else {
|
||||
@@ -209,6 +218,393 @@ impl RolloutLineReader {
|
||||
|
||||
type BlockingLineReader = std::io::Lines<std::io::BufReader<Box<dyn Read + Send>>>;
|
||||
|
||||
mod worker {
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::File;
|
||||
use std::fs::FileTimes;
|
||||
use std::fs::Permissions;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use tracing::debug;
|
||||
use tracing::info;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::ARCHIVED_SESSIONS_SUBDIR;
|
||||
use crate::SESSIONS_SUBDIR;
|
||||
|
||||
use super::RolloutFile;
|
||||
use super::path;
|
||||
|
||||
const TEMP_SUFFIX: &str = ".tmp";
|
||||
const COMPRESSION_LEVEL: i32 = 3;
|
||||
const MIN_ROLLOUT_AGE: Duration = Duration::from_secs(7 * 24 * 60 * 60);
|
||||
const GLOBAL_LOCK_STALE_AFTER: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
const TEMP_FILE_STALE_AFTER: Duration = GLOBAL_LOCK_STALE_AFTER;
|
||||
const WORKER_MAX_RUNTIME: Duration = Duration::from_secs(5 * 60 * 60);
|
||||
const LOCK_FILE_NAME: &str = "rollout-compression.lock";
|
||||
|
||||
#[derive(Default)]
|
||||
struct CompressionStats {
|
||||
scanned: usize,
|
||||
compressed: usize,
|
||||
skipped: usize,
|
||||
failed: usize,
|
||||
}
|
||||
|
||||
struct CompressionLock {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl CompressionLock {
|
||||
fn try_acquire(codex_home: &Path) -> io::Result<Option<Self>> {
|
||||
let lock_dir = codex_home.join(".tmp");
|
||||
std::fs::create_dir_all(lock_dir.as_path())?;
|
||||
let path = lock_dir.join(LOCK_FILE_NAME);
|
||||
match create_lock_file(path.as_path()) {
|
||||
Ok(()) => return Ok(Some(Self { path })),
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let stale = std::fs::metadata(path.as_path())
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.ok()
|
||||
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
|
||||
.is_some_and(|age| age >= GLOBAL_LOCK_STALE_AFTER);
|
||||
if !stale {
|
||||
return Ok(None);
|
||||
}
|
||||
match std::fs::remove_file(path.as_path()) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
match create_lock_file(path.as_path()) {
|
||||
Ok(()) => Ok(Some(Self { path })),
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CompressionLock {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(self.path.as_path());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn(codex_home: PathBuf) {
|
||||
let Ok(handle) = tokio::runtime::Handle::try_current() else {
|
||||
warn!(
|
||||
"failed to start rollout compression worker for {}: no Tokio runtime",
|
||||
codex_home.display()
|
||||
);
|
||||
return;
|
||||
};
|
||||
handle.spawn(async move {
|
||||
if let Err(err) = run(codex_home.clone()).await {
|
||||
warn!(
|
||||
"rollout compression worker failed for {}: {err}",
|
||||
codex_home.display()
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) async fn run(codex_home: PathBuf) -> io::Result<()> {
|
||||
let Some(_lock) = CompressionLock::try_acquire(codex_home.as_path())? else {
|
||||
debug!(
|
||||
"rollout compression worker already running for {}",
|
||||
codex_home.display()
|
||||
);
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let started_at = Instant::now();
|
||||
cleanup_stale_temps(codex_home.as_path()).await?;
|
||||
let mut stats = CompressionStats::default();
|
||||
if started_at.elapsed() < WORKER_MAX_RUNTIME {
|
||||
let archived_root = codex_home.join(ARCHIVED_SESSIONS_SUBDIR);
|
||||
compress_rollouts_in_root(archived_root.as_path(), started_at, &mut stats).await?;
|
||||
}
|
||||
info!(
|
||||
"rollout compression worker finished: scanned={}, compressed={}, skipped={}, failed={}",
|
||||
stats.scanned, stats.compressed, stats.skipped, stats.failed
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_lock_file(path: &Path) -> io::Result<()> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path)?;
|
||||
writeln!(
|
||||
file,
|
||||
"pid={} started_at={:?}",
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compress_rollouts_in_root(
|
||||
root: &Path,
|
||||
started_at: Instant,
|
||||
stats: &mut CompressionStats,
|
||||
) -> io::Result<()> {
|
||||
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(dir) = stack.pop() {
|
||||
if started_at.elapsed() >= WORKER_MAX_RUNTIME {
|
||||
break;
|
||||
}
|
||||
let mut read_dir = match tokio::fs::read_dir(dir.as_path()).await {
|
||||
Ok(read_dir) => read_dir,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read rollout compression directory {}: {err}",
|
||||
dir.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
while let Some(entry) = read_dir.next_entry().await? {
|
||||
if started_at.elapsed() >= WORKER_MAX_RUNTIME {
|
||||
break;
|
||||
}
|
||||
let path = entry.path();
|
||||
let file_type = match entry.file_type().await {
|
||||
Ok(file_type) => file_type,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read rollout compression file type {}: {err}",
|
||||
path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
stack.push(path);
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(rollout_file) = RolloutFile::from_path(path) else {
|
||||
continue;
|
||||
};
|
||||
if rollout_file.is_compressed() {
|
||||
continue;
|
||||
}
|
||||
let path = rollout_file.into_path();
|
||||
stats.scanned = stats.scanned.saturating_add(1);
|
||||
match compress_rollout_if_cold(path.as_path()).await {
|
||||
Ok(true) => stats.compressed = stats.compressed.saturating_add(1),
|
||||
Ok(false) => stats.skipped = stats.skipped.saturating_add(1),
|
||||
Err(err) => {
|
||||
stats.failed = stats.failed.saturating_add(1);
|
||||
warn!("failed to compress rollout {}: {err}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compress_rollout_if_cold(path: &Path) -> io::Result<bool> {
|
||||
let path = path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || compress_rollout_if_cold_blocking(path.as_path()))
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
|
||||
fn compress_rollout_if_cold_blocking(path: &Path) -> io::Result<bool> {
|
||||
let before = match cold_file_state(path)? {
|
||||
Some(state) => state,
|
||||
None => return Ok(false),
|
||||
};
|
||||
let compressed_path = path::compressed_rollout_path(path);
|
||||
if compressed_path.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let temp_dir = compressed_path
|
||||
.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
.unwrap_or_else(|| Path::new("."));
|
||||
std::fs::create_dir_all(temp_dir)?;
|
||||
let mut temp_file = tempfile::Builder::new()
|
||||
.prefix("rollout-compress-")
|
||||
.suffix(TEMP_SUFFIX)
|
||||
.tempfile_in(temp_dir)?;
|
||||
encode_zstd_to_writer(path, temp_file.as_file_mut())?;
|
||||
temp_file.as_file_mut().flush()?;
|
||||
verify_zstd(temp_file.path())?;
|
||||
if !same_file_state(path, &before)? {
|
||||
return Ok(false);
|
||||
}
|
||||
set_file_metadata(temp_file.as_file(), before.modified, &before.permissions)?;
|
||||
temp_file.as_file().sync_all()?;
|
||||
|
||||
match temp_file.persist_noclobber(compressed_path.as_path()) {
|
||||
Ok(_) => {}
|
||||
Err(err) if err.error.kind() == io::ErrorKind::AlreadyExists => return Ok(false),
|
||||
Err(err) => return Err(err.error),
|
||||
}
|
||||
if !same_file_state(path, &before)? {
|
||||
let _ = std::fs::remove_file(compressed_path.as_path());
|
||||
return Ok(false);
|
||||
}
|
||||
std::fs::remove_file(path)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
struct FileState {
|
||||
len: u64,
|
||||
modified: SystemTime,
|
||||
permissions: Permissions,
|
||||
}
|
||||
|
||||
fn cold_file_state(path: &Path) -> io::Result<Option<FileState>> {
|
||||
let metadata = match std::fs::metadata(path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !metadata.is_file() {
|
||||
return Ok(None);
|
||||
}
|
||||
let modified = metadata.modified()?;
|
||||
let age = SystemTime::now()
|
||||
.duration_since(modified)
|
||||
.unwrap_or(Duration::ZERO);
|
||||
if age < MIN_ROLLOUT_AGE {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(FileState {
|
||||
len: metadata.len(),
|
||||
modified,
|
||||
permissions: metadata.permissions(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn same_file_state(path: &Path, expected: &FileState) -> io::Result<bool> {
|
||||
match std::fs::metadata(path) {
|
||||
Ok(metadata) => Ok(metadata.len() == expected.len
|
||||
&& metadata.modified()? == expected.modified
|
||||
&& metadata.permissions() == expected.permissions),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_zstd_to_writer(source: &Path, output: impl Write) -> io::Result<()> {
|
||||
let mut input = File::open(source)?;
|
||||
let mut encoder = zstd::stream::write::Encoder::new(output, COMPRESSION_LEVEL)?;
|
||||
io::copy(&mut input, &mut encoder)?;
|
||||
encoder.finish()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn verify_zstd(path: &Path) -> io::Result<()> {
|
||||
let input = File::open(path)?;
|
||||
let mut decoder = zstd::stream::read::Decoder::new(input)?;
|
||||
let mut sink = io::sink();
|
||||
io::copy(&mut decoder, &mut sink)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_file_metadata(
|
||||
file: &File,
|
||||
modified: SystemTime,
|
||||
permissions: &Permissions,
|
||||
) -> io::Result<()> {
|
||||
file.set_times(FileTimes::new().set_modified(modified))?;
|
||||
file.set_permissions(permissions.clone())
|
||||
}
|
||||
|
||||
async fn cleanup_stale_temps(codex_home: &Path) -> io::Result<()> {
|
||||
for root in [
|
||||
codex_home.join(SESSIONS_SUBDIR),
|
||||
codex_home.join(ARCHIVED_SESSIONS_SUBDIR),
|
||||
] {
|
||||
cleanup_stale_temps_in_root(root.as_path()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_stale_temps_in_root(root: &Path) -> io::Result<()> {
|
||||
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(dir) = stack.pop() {
|
||||
let mut read_dir = match tokio::fs::read_dir(dir.as_path()).await {
|
||||
Ok(read_dir) => read_dir,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read rollout temp cleanup directory {}: {err}",
|
||||
dir.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
while let Some(entry) = read_dir.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let file_type = match entry.file_type().await {
|
||||
Ok(file_type) => file_type,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"failed to read rollout temp cleanup file type {}: {err}",
|
||||
path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
stack.push(path);
|
||||
continue;
|
||||
}
|
||||
if file_type.is_file()
|
||||
&& path
|
||||
.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_some_and(|name| name.ends_with(TEMP_SUFFIX))
|
||||
{
|
||||
let stale = entry
|
||||
.metadata()
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|metadata| metadata.modified().ok())
|
||||
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
|
||||
.is_some_and(|age| age >= TEMP_FILE_STALE_AFTER);
|
||||
if !stale {
|
||||
continue;
|
||||
}
|
||||
match tokio::fs::remove_file(path.as_path()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => warn!(
|
||||
"failed to remove stale rollout temp {}: {err}",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the existing rollout path, preferring the plain `.jsonl` file over
|
||||
/// its `.jsonl.zst` compressed sibling.
|
||||
pub async fn existing_rollout_path(path: &Path) -> Option<PathBuf> {
|
||||
|
||||
Reference in new issue
Block a user