mirror of
https://github.com/openai/codex.git
synced 2026-11-20 07:06:24 +00:00
Include shared histories in rollout compression (#42039)
## What changed - Make `local_thread_store_compression` compress cold rollout files across shared and forked histories without a separate compression mode. - Retire `local_thread_store_shared_compression` while continuing to accept it in strict configuration without changing compression behavior. - Read rollout files through the compressed-rollout reader when `codex exec resume` determines the latest turn's working directory. ## Testing - Cover compression and restoration of an archived fork chain. - Cover cwd-based resume selection when only the compressed rollout remains. - Cover the removed compatibility key in strict configuration. GitOrigin-RevId: 8edebf61eccc1f79cabb8e641f8982c76f337bf1
This commit is contained in:
1 parent
2e5ee418ad
commit
9d0eae74cd
13 files changed
+137
-305
No files matched your search
@@ -21,22 +21,13 @@ const OPEN_ROLLOUT_LINE_READER_RETRY_DELAY: Duration = Duration::from_millis(50)
|
||||
const TEMP_SUFFIX: &str = ".tmp";
|
||||
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Whether cold files belonging to shared rollout lineages may be compressed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RolloutCompressionMode {
|
||||
/// Preserve compatibility with readers that require shared lineages to remain plain JSONL.
|
||||
Standalone,
|
||||
/// Requires every reader of this Codex home to support compressed shared lineages.
|
||||
IncludeShared,
|
||||
}
|
||||
|
||||
/// Starts a best-effort background job that compresses cold local rollout files.
|
||||
///
|
||||
/// The worker is fire-and-forget: failures are logged, startup is not blocked,
|
||||
/// and a run marker under `codex_home` prevents overlapping or too-frequent
|
||||
/// compression runs from the same local store.
|
||||
pub fn spawn_rollout_compression_worker(codex_home: PathBuf, mode: RolloutCompressionMode) {
|
||||
worker::spawn(codex_home, mode)
|
||||
pub fn spawn_rollout_compression_worker(codex_home: PathBuf) {
|
||||
worker::spawn(codex_home)
|
||||
}
|
||||
|
||||
/// Returns the modified time for the existing plain or compressed rollout file.
|
||||
@@ -253,10 +244,8 @@ mod worker {
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
use crate::ARCHIVED_SESSIONS_SUBDIR;
|
||||
use crate::RolloutReferenceIndex;
|
||||
use crate::SESSIONS_SUBDIR;
|
||||
|
||||
use super::RolloutCompressionMode;
|
||||
use super::RolloutFile;
|
||||
use super::metrics;
|
||||
use super::path;
|
||||
@@ -334,7 +323,7 @@ mod worker {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn(codex_home: PathBuf, mode: RolloutCompressionMode) {
|
||||
pub(super) fn spawn(codex_home: PathBuf) {
|
||||
let Ok(handle) = tokio::runtime::Handle::try_current() else {
|
||||
metrics::run("skipped_no_runtime");
|
||||
warn!(
|
||||
@@ -344,7 +333,7 @@ mod worker {
|
||||
return;
|
||||
};
|
||||
handle.spawn(async move {
|
||||
if let Err(err) = run(codex_home.clone(), mode).await {
|
||||
if let Err(err) = run(codex_home.clone()).await {
|
||||
warn!(
|
||||
"rollout compression worker failed for {}: {err}",
|
||||
codex_home.display()
|
||||
@@ -353,7 +342,7 @@ mod worker {
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) async fn run(codex_home: PathBuf, mode: RolloutCompressionMode) -> io::Result<()> {
|
||||
pub(super) async fn run(codex_home: PathBuf) -> io::Result<()> {
|
||||
let Some(_maintenance_guard) =
|
||||
crate::try_acquire_rollout_maintenance_lock(codex_home.as_path())?
|
||||
else {
|
||||
@@ -384,15 +373,6 @@ mod worker {
|
||||
let started_at = Instant::now();
|
||||
let result = async {
|
||||
cleanup_stale_temps(codex_home.as_path()).await?;
|
||||
let Some(reference_index) = RolloutReferenceIndex::scan_until(
|
||||
codex_home.as_path(),
|
||||
started_at,
|
||||
WORKER_MAX_RUNTIME,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(CompressionStats::default());
|
||||
};
|
||||
let mut stats = CompressionStats::default();
|
||||
for root in [
|
||||
codex_home.join(ARCHIVED_SESSIONS_SUBDIR),
|
||||
@@ -401,14 +381,7 @@ mod worker {
|
||||
if started_at.elapsed() >= WORKER_MAX_RUNTIME {
|
||||
break;
|
||||
}
|
||||
compress_rollouts_in_root(
|
||||
root.as_path(),
|
||||
started_at,
|
||||
&reference_index,
|
||||
mode,
|
||||
&mut stats,
|
||||
)
|
||||
.await?;
|
||||
compress_rollouts_in_root(root.as_path(), started_at, &mut stats).await?;
|
||||
}
|
||||
Ok::<_, io::Error>(stats)
|
||||
}
|
||||
@@ -448,8 +421,6 @@ mod worker {
|
||||
async fn compress_rollouts_in_root(
|
||||
root: &Path,
|
||||
started_at: Instant,
|
||||
reference_index: &RolloutReferenceIndex,
|
||||
mode: RolloutCompressionMode,
|
||||
stats: &mut CompressionStats,
|
||||
) -> io::Result<()> {
|
||||
if !tokio::fs::try_exists(root).await.unwrap_or(false) {
|
||||
@@ -508,26 +479,14 @@ mod worker {
|
||||
continue;
|
||||
}
|
||||
let path = rollout_file.into_path();
|
||||
let Some(rollout_id) = crate::rollout_id_from_path(path.as_path()) else {
|
||||
if crate::rollout_id_from_path(path.as_path()).is_none() {
|
||||
stats.skipped = stats.skipped.saturating_add(1);
|
||||
metrics::file("skipped_unreadable_meta");
|
||||
continue;
|
||||
};
|
||||
let Ok(meta) = crate::read_session_meta_line(path.as_path()).await else {
|
||||
stats.skipped = stats.skipped.saturating_add(1);
|
||||
metrics::file("skipped_unreadable_meta");
|
||||
continue;
|
||||
};
|
||||
if mode == RolloutCompressionMode::Standalone
|
||||
&& reference_index.reference_count(rollout_id) > 0
|
||||
{
|
||||
stats.skipped = stats.skipped.saturating_add(1);
|
||||
metrics::file("skipped_referenced");
|
||||
continue;
|
||||
}
|
||||
if mode == RolloutCompressionMode::Standalone && meta.meta.history_base.is_some() {
|
||||
if crate::read_session_meta_line(path.as_path()).await.is_err() {
|
||||
stats.skipped = stats.skipped.saturating_add(1);
|
||||
metrics::file("skipped_fork_pointer");
|
||||
metrics::file("skipped_unreadable_meta");
|
||||
continue;
|
||||
}
|
||||
stats.scanned = stats.scanned.saturating_add(1);
|
||||
|
||||
Reference in new issue
Block a user