Add rollout migration tooling and background migration (#37348)

## What changed

- Add `codex migrate-rollouts` with dry-run inspection by default, explicit `--apply`, thread filtering, optional I/O throttling, progress output, and JSON or verbose reports.
- Add the disabled-by-default `background_paginated_rollout_migration` feature to migrate legacy local sessions at startup before rollout compression begins.
- Persist a migration cursor and skipped-file fingerprints so later startups avoid full rescans while retrying changed files and recovering pending migrations.
- Coordinate migration with live writers and emit metrics for manual and startup runs.

## Testing

- Cover startup cursor advancement and lookback, pending migration recovery, live-writer coordination, changed empty rollouts, and progress reporting.

GitOrigin-RevId: 276ac506c50ebec5140fd319faca1bb998172061
This commit is contained in:
Owen Lin
2026-08-07 01:26:42 +00:00
committed by copyberry
parent 69b6152c17
commit 4bb7ee3472
21 changed files with 1569 additions and 24 deletions
+23 -7
View File
@@ -349,16 +349,32 @@ pub fn thread_store_from_config(
) -> Arc<dyn ThreadStore> {
match &config.experimental_thread_store {
ThreadStoreConfig::Local => {
if config
let compression_enabled = config
.features
.enabled(Feature::LocalThreadStoreCompression)
{
codex_rollout::spawn_rollout_compression_worker(config.codex_home.to_path_buf());
}
Arc::new(LocalThreadStore::new(
.enabled(Feature::LocalThreadStoreCompression);
let background_migration_enabled = config
.features
.enabled(Feature::BackgroundPaginatedRolloutMigration);
let has_state_db = state_db.is_some();
let store = Arc::new(LocalThreadStore::new(
LocalThreadStoreConfig::from_config(config),
state_db,
))
));
if has_state_db && background_migration_enabled {
let startup_store = Arc::clone(&store);
let codex_home = config.codex_home.to_path_buf();
tokio::spawn(async move {
if let Err(err) = startup_store.migrate_rollouts_on_startup().await {
warn!("failed to migrate legacy rollouts on startup: {err}");
}
if compression_enabled {
codex_rollout::spawn_rollout_compression_worker(codex_home);
}
});
} else if compression_enabled {
codex_rollout::spawn_rollout_compression_worker(config.codex_home.to_path_buf());
}
store
}
ThreadStoreConfig::InMemory { id } => InMemoryThreadStore::for_id(id),
}