Add rollout migration tooling and background migration (#37348)

## What changed

- Add `codex migrate-rollouts` with dry-run inspection by default, explicit `--apply`, thread filtering, optional I/O throttling, progress output, and JSON or verbose reports.
- Add the disabled-by-default `background_paginated_rollout_migration` feature to migrate legacy local sessions at startup before rollout compression begins.
- Persist a migration cursor and skipped-file fingerprints so later startups avoid full rescans while retrying changed files and recovering pending migrations.
- Coordinate migration with live writers and emit metrics for manual and startup runs.

## Testing

- Cover startup cursor advancement and lookback, pending migration recovery, live-writer coordination, changed empty rollouts, and progress reporting.

GitOrigin-RevId: 276ac506c50ebec5140fd319faca1bb998172061
This commit is contained in:
Owen Lin authored and copyberry committed 2026-08-07 01:26:42 +00:00
1 parent 69b6152c17
commit 4bb7ee3472
21 files changed
+1569 -24

No files matched your search

@@ -0,0 +1,133 @@
//! Records low-cardinality migration metrics shared by manual and startup runs.
//!
//! Migration owns the outcome stream, so it is the one place that can count successful,
//! skipped, and failed rollouts consistently. Callers only tell it why the run started.
use std::time::Instant;
use super::RolloutMigrationMode;
use super::RolloutMigrationOptions;
use super::RolloutMigrationReport;
use super::RolloutMigrationStatus;
use crate::ThreadStoreResult;
const RUN_METRIC: &str = "codex.rollout_migration.run";
const RUN_DURATION_METRIC: &str = "codex.rollout_migration.run.duration_ms";
const RUN_IO_BYTES_METRIC: &str = "codex.rollout_migration.run.io_bytes";
const THREAD_METRIC: &str = "codex.rollout_migration.thread";
#[derive(Clone, Copy)]
pub(super) enum RolloutMigrationTrigger {
Manual,
Startup,
}
#[derive(Clone, Copy)]
enum RolloutMigrationScope {
All,
Selected,
}
pub(super) struct RolloutMigrationTelemetry {
trigger: RolloutMigrationTrigger,
mode: RolloutMigrationMode,
scope: RolloutMigrationScope,
started_at: Instant,
}
impl RolloutMigrationTelemetry {
pub(super) fn new(trigger: RolloutMigrationTrigger, options: &RolloutMigrationOptions) -> Self {
Self {
trigger,
mode: options.mode,
scope: if options.thread_ids.is_empty() {
RolloutMigrationScope::All
} else {
RolloutMigrationScope::Selected
},
started_at: Instant::now(),
}
}
pub(super) fn finish(&self, result: &ThreadStoreResult<RolloutMigrationReport>) {
let Some(metrics) = codex_otel::global() else {
return;
};
let mut io_bytes = 0_u64;
if let Ok(report) = result {
for outcome in &report.outcomes {
let tags = [
("trigger", self.trigger.tag()),
("mode", self.mode.tag()),
("scope", self.scope.tag()),
("status", outcome.status.tag()),
];
let _ = metrics.counter(THREAD_METRIC, /*inc*/ 1, &tags);
io_bytes = io_bytes.saturating_add(outcome.bytes_processed);
}
}
let result = match result {
Err(_) => "error",
Ok(report)
if report
.outcomes
.iter()
.any(|outcome| outcome.status == RolloutMigrationStatus::Failed) =>
{
"partial_failure"
}
Ok(_) => "success",
};
let tags = [
("trigger", self.trigger.tag()),
("mode", self.mode.tag()),
("scope", self.scope.tag()),
("result", result),
];
let duration_ms = i64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(i64::MAX);
let io_bytes = i64::try_from(io_bytes).unwrap_or(i64::MAX);
let _ = metrics.counter(RUN_METRIC, /*inc*/ 1, &tags);
let _ = metrics.histogram(RUN_DURATION_METRIC, duration_ms, &tags);
let _ = metrics.histogram(RUN_IO_BYTES_METRIC, io_bytes, &tags);
}
}
impl RolloutMigrationTrigger {
fn tag(self) -> &'static str {
match self {
Self::Manual => "manual",
Self::Startup => "startup",
}
}
}
impl RolloutMigrationMode {
fn tag(self) -> &'static str {
match self {
Self::DryRun => "dry_run",
Self::Apply => "apply",
}
}
}
impl RolloutMigrationScope {
fn tag(self) -> &'static str {
match self {
Self::All => "all",
Self::Selected => "selected",
}
}
}
impl RolloutMigrationStatus {
fn tag(self) -> &'static str {
match self {
Self::Eligible => "eligible",
Self::Migrated => "migrated",
Self::AlreadyPaginated => "already_paginated",
Self::SkippedEmpty => "skipped_empty",
Self::SkippedBusy => "skipped_busy",
Self::Failed => "failed",
}
}
}