Preserve legacy semantics during rollout migration (#37191)

## Why

Legacy rollouts can contain historical rollbacks, compaction checkpoints, and
subagent copies of parent history. Migrating those records verbatim can change
the visible conversation or the model context used when a thread resumes, and
can retain large amounts of duplicated subagent history.

## What changed

- Plan and replay historical rollbacks by logical turn, preserving surviving
  lifecycle events and the correct compaction anchor for future resumes.
- Migrate subagent rollouts from a safe bounded replay when possible and record
  the history boundary so copied parent context is not projected as child turns.
- Include archived and compressed rollouts while keeping their storage form and
  location intact.
- Tolerate malformed, partial, oversized, and empty JSONL records during the
  migration scan.

## Testing

Add coverage for rollback and compaction combinations, subagent replay bounds,
archived and compressed rollouts, recovery, concurrent maintenance, and damaged
JSONL input.

GitOrigin-RevId: 3948e16d43f75df8571b0a92af12a181d1d08fdf
This commit is contained in:
Owen Lin authored and copyberry committed 2026-08-06 00:54:14 +00:00
1 parent f141dc77f0
commit aac9f84247
11 files changed
+2752 -251

No files matched your search

@@ -15,8 +15,11 @@ use std::time::Duration;
use chrono::DateTime;
use codex_app_server_protocol::project_rollout_line;
use codex_protocol::ThreadId;
use codex_protocol::protocol::InternalSessionSource;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::ThreadSource;
use serde::Serialize;
@@ -39,6 +42,10 @@ mod canonicalizer;
mod legacy_event;
mod line_parser;
mod publish;
mod rollback;
mod rollback_plan;
mod rollback_replay;
mod subagent;
use canonicalizer::LegacyRolloutCanonicalizer;
use publish::compress_rollout_to_path;
@@ -48,13 +55,25 @@ use publish::decompressed_staged_rollout_path;
use publish::migration_journal_path;
use publish::pending_migration_thread_ids;
use publish::remove_file_if_present;
use publish::rewrite_subagent_history_boundary;
use publish::rewritten_staged_rollout_path;
use publish::staged_rollout_path;
use publish::sync_parent_directory;
use publish::write_migration_journal;
use rollback_plan::RollbackPlan;
use rollback_plan::RollbackPlanner;
const PROJECTION_BATCH_BYTES: u64 = 256 * 1024;
const MAX_ROLLOUT_LINE_BYTES: usize = 16 * 1024 * 1024;
enum CanonicalizationAttempt {
Complete {
expected_length: u64,
expected_ordinal: u64,
},
NeedsRollbackPlan,
}
struct CanonicalizationSource<'a> {
thread_id: ThreadId,
source_path: &'a Path,
@@ -99,7 +118,7 @@ pub enum RolloutMigrationStatus {
Eligible,
Migrated,
AlreadyPaginated,
SkippedSubagent,
SkippedEmpty,
SkippedBusy,
Failed,
}
@@ -132,6 +151,12 @@ struct RolloutRecord {
byte_count: u64,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RolloutMigrationKind {
Ordinary,
Subagent,
}
impl RolloutMigrationRateLimiter {
fn new(max_mib_per_second: u64) -> ThreadStoreResult<Self> {
let bytes_per_second = max_mib_per_second
@@ -166,7 +191,7 @@ impl RolloutMigrationRateLimiter {
}
impl LocalThreadStore {
/// Inspect or migrate eligible legacy rollout files beneath active sessions.
/// Inspect or migrate eligible legacy rollout files beneath active and archived sessions.
pub async fn migrate_rollouts(
&self,
options: RolloutMigrationOptions,
@@ -186,6 +211,15 @@ impl LocalThreadStore {
let mut paths =
find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR))
.await?;
paths.extend(
find_rollout_paths(
&self
.config
.codex_home
.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
)
.await?,
);
if options.mode == RolloutMigrationMode::Apply {
let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;
paths.sort_by_key(|path| {
@@ -196,145 +230,167 @@ impl LocalThreadStore {
let mut report = RolloutMigrationReport::default();
for path in paths {
let metadata = match codex_rollout::read_session_meta_line(&path).await {
Ok(metadata) => metadata,
Err(error) => {
let thread_id = thread_id_from_rollout_filename(&path);
if matches_selection(&options.thread_ids, thread_id) {
report.outcomes.push(RolloutMigrationOutcome {
thread_id,
rollout_path: path,
status: RolloutMigrationStatus::Failed,
bytes_processed: 0,
message: Some(error.to_string()),
});
}
continue;
}
};
let thread_id = metadata.meta.id;
if !matches_selection(&options.thread_ids, Some(thread_id)) {
continue;
}
if metadata.meta.source.is_non_root_agent()
|| matches!(
metadata.meta.thread_source,
Some(ThreadSource::Subagent | ThreadSource::MemoryConsolidation)
)
if let Some(outcome) = self
.migrate_rollout_path(path, &options, &mut limiter)
.await?
{
report.outcomes.push(RolloutMigrationOutcome {
thread_id: Some(thread_id),
rollout_path: path,
status: RolloutMigrationStatus::SkippedSubagent,
bytes_processed: 0,
message: None,
});
continue;
report.outcomes.push(outcome);
}
let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
let bytes_before = limiter.bytes_processed;
let result = if options.mode == RolloutMigrationMode::Apply
&& tokio::fs::try_exists(&journal_path)
.await
.map_err(migration_error)?
{
match self
.recover_published_migration(thread_id, &path, &journal_path, &mut limiter)
.await
{
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
Err(ThreadStoreError::Conflict { message }) => {
let bytes_processed =
limiter.bytes_processed.saturating_sub(bytes_before);
report.outcomes.push(skipped_busy_outcome(
thread_id,
path,
message,
bytes_processed,
));
continue;
}
Err(error) => Err(error),
}
} else {
Ok(RolloutMigrationStatus::AlreadyPaginated)
};
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
report
.outcomes
.push(migration_outcome(thread_id, path, result, bytes_processed));
continue;
}
if options.mode == RolloutMigrationMode::DryRun {
report.outcomes.push(RolloutMigrationOutcome {
thread_id: Some(thread_id),
rollout_path: path,
status: RolloutMigrationStatus::Eligible,
bytes_processed: 0,
message: None,
});
continue;
}
let writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
Ok(guard) => guard,
Err(ThreadStoreError::Conflict { message }) => {
report.outcomes.push(skipped_busy_outcome(
thread_id, path, message, /*bytes_processed*/ 0,
));
continue;
}
Err(error) => {
report.outcomes.push(migration_outcome(
thread_id,
path,
Err(error),
/*bytes_processed*/ 0,
));
continue;
}
};
let bytes_before = limiter.bytes_processed;
let result = match self
.migrate_one_rollout(thread_id, &path, &journal_path, &mut limiter)
.await
{
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
Err(error) => {
if let Err(cleanup_error) = self
.cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
.await
{
Err(migration_error(format!(
"{error}; failed to clean up unpublished migration: {cleanup_error}"
)))
} else {
Err(error)
}
}
};
drop(writer_guard);
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
let outcome = match result {
Err(ThreadStoreError::Conflict { message }) => {
skipped_busy_outcome(thread_id, path, message, bytes_processed)
}
result => migration_outcome(thread_id, path, result, bytes_processed),
};
report.outcomes.push(outcome);
}
Ok(report)
}
async fn migrate_rollout_path(
&self,
mut path: PathBuf,
options: &RolloutMigrationOptions,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<Option<RolloutMigrationOutcome>> {
let metadata = match codex_rollout::read_session_meta_line(&path).await {
Ok(metadata) => metadata,
Err(error) => {
let thread_id = thread_id_from_rollout_filename(&path);
if !matches_selection(&options.thread_ids, thread_id) {
return Ok(None);
}
let empty = tokio::fs::metadata(&path)
.await
.is_ok_and(|metadata| metadata.len() == 0);
return Ok(Some(RolloutMigrationOutcome {
thread_id,
rollout_path: path,
status: if empty {
RolloutMigrationStatus::SkippedEmpty
} else {
RolloutMigrationStatus::Failed
},
bytes_processed: 0,
message: (!empty).then(|| error.to_string()),
}));
}
};
let thread_id = metadata.meta.id;
if !matches_selection(&options.thread_ids, Some(thread_id)) {
return Ok(None);
}
let is_memory_consolidation = matches!(
metadata.meta.source,
SessionSource::Internal(InternalSessionSource::MemoryConsolidation)
| SessionSource::SubAgent(SubAgentSource::MemoryConsolidation)
) || matches!(
metadata.meta.thread_source,
Some(ThreadSource::MemoryConsolidation)
);
let kind = if !is_memory_consolidation
&& (matches!(metadata.meta.source, SessionSource::SubAgent(_))
|| matches!(metadata.meta.thread_source, Some(ThreadSource::Subagent)))
{
RolloutMigrationKind::Subagent
} else {
RolloutMigrationKind::Ordinary
};
let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
let pending_published_migration = metadata.meta.history_mode
== ThreadHistoryMode::Paginated
&& options.mode == RolloutMigrationMode::Apply
&& tokio::fs::try_exists(&journal_path)
.await
.map_err(migration_error)?;
if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
let bytes_before = limiter.bytes_processed;
let result = if pending_published_migration {
match self
.recover_published_migration(thread_id, &path, &journal_path, limiter)
.await
{
Ok(recovered_path) => {
path = recovered_path;
Ok(RolloutMigrationStatus::Migrated)
}
Err(ThreadStoreError::Conflict { message }) => {
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
return Ok(Some(skipped_busy_outcome(
thread_id,
path,
message,
bytes_processed,
)));
}
Err(error) => Err(error),
}
} else {
Ok(RolloutMigrationStatus::AlreadyPaginated)
};
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
return Ok(Some(migration_outcome(
thread_id,
path,
result,
bytes_processed,
)));
}
if options.mode == RolloutMigrationMode::DryRun {
return Ok(Some(migration_outcome(
thread_id,
path,
Ok(RolloutMigrationStatus::Eligible),
/*bytes_processed*/ 0,
)));
}
let _writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
Ok(guard) => guard,
Err(ThreadStoreError::Conflict { message }) => {
return Ok(Some(skipped_busy_outcome(
thread_id, path, message, /*bytes_processed*/ 0,
)));
}
Err(error) => {
return Ok(Some(migration_outcome(
thread_id,
path,
Err(error),
/*bytes_processed*/ 0,
)));
}
};
let bytes_before = limiter.bytes_processed;
let result = match self
.migrate_one_rollout(thread_id, &path, &journal_path, kind, limiter)
.await
{
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
Err(error) => {
if let Err(cleanup_error) = self
.cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
.await
{
Err(migration_error(format!(
"{error}; failed to clean up unpublished migration: {cleanup_error}"
)))
} else {
Err(error)
}
}
};
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
Ok(Some(match result {
Err(ThreadStoreError::Conflict { message }) => {
skipped_busy_outcome(thread_id, path, message, bytes_processed)
}
result => migration_outcome(thread_id, path, result, bytes_processed),
}))
}
async fn migrate_one_rollout(
&self,
thread_id: ThreadId,
rollout_path: &Path,
journal_path: &Path,
kind: RolloutMigrationKind,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<()> {
if let Some(state_db) = &self.state_db
@@ -403,8 +459,31 @@ impl LocalThreadStore {
source_permissions: &source_permissions,
canonical_session_meta: &canonical_session_meta,
};
let (expected_length, expected_ordinal) =
Self::write_canonical_rollout(&canonicalization_source, limiter).await?;
let bounded_subagent_context = if kind == RolloutMigrationKind::Subagent {
let RolloutItem::SessionMeta(session_meta) = &canonical_session_meta.item else {
return Err(migration_error("canonical session metadata is missing"));
};
let context =
subagent::select_bounded_context(source_path.to_path_buf(), session_meta.clone())
.await?;
limiter.account(source_metadata.len()).await;
context
} else {
None
};
let (_, expected_ordinal) = if let Some(items) = bounded_subagent_context {
Self::write_bounded_subagent_rollout(&canonicalization_source, items, limiter).await?
} else {
Self::write_rollout_with_rollback_plan(&canonicalization_source, limiter).await?
};
if kind == RolloutMigrationKind::Subagent {
rewrite_subagent_history_boundary(&staged_path, expected_ordinal).await?;
}
let expected_length = tokio::fs::metadata(&staged_path)
.await
.map_err(migration_error)?
.len();
// SQLite projection only starts after every staged-file mutation is durable.
let modified_at = source_modified;
@@ -473,10 +552,97 @@ impl LocalThreadStore {
.await
}
async fn write_canonical_rollout(
async fn build_rollback_plan(
source_path: &Path,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<RollbackPlan> {
let source_file = File::open(source_path).await.map_err(migration_error)?;
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
let mut bytes = Vec::new();
let mut planner = RollbackPlanner::new();
while let Some(record) = read_rollout_record(&mut source, &mut bytes).await? {
limiter.account(record.byte_count).await;
if let Some(line) = record.line {
planner.observe(&line)?;
}
}
Ok(planner.finish())
}
async fn write_rollout_with_rollback_plan(
input: &CanonicalizationSource<'_>,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<(u64, u64)> {
let attempt = Self::write_canonical_rollout(input, /*plan*/ None, limiter).await?;
match attempt {
CanonicalizationAttempt::Complete {
expected_length,
expected_ordinal,
} => Ok((expected_length, expected_ordinal)),
CanonicalizationAttempt::NeedsRollbackPlan => {
remove_file_if_present(input.staged_path).await?;
let plan = Self::build_rollback_plan(input.source_path, limiter).await?;
let CanonicalizationAttempt::Complete {
expected_length,
expected_ordinal,
} = Self::write_canonical_rollout(input, Some(&plan), limiter).await?
else {
return Err(migration_error(
"planned rollout still contains a rollback marker",
));
};
Ok((expected_length, expected_ordinal))
}
}
}
async fn write_bounded_subagent_rollout(
input: &CanonicalizationSource<'_>,
items: Vec<RolloutItem>,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<(u64, u64)> {
let staged_file = tokio::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(input.staged_path)
.await
.map_err(migration_error)?;
staged_file
.set_permissions(input.source_permissions.clone())
.await
.map_err(migration_error)?;
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
let written = canonicalizer
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
.await?;
limiter.account(written).await;
for item in items {
let line = RolloutLine {
timestamp: input.canonical_session_meta.timestamp.clone(),
ordinal: None,
item,
};
let written = canonicalizer.process_line(line, &mut staged).await?;
limiter.account(written).await;
}
let written = canonicalizer
.finish(&mut staged, &input.canonical_session_meta.timestamp)
.await?;
limiter.account(written).await;
staged.flush().await.map_err(migration_error)?;
Ok((
canonicalizer.output_byte_offset(),
canonicalizer.next_ordinal(),
))
}
async fn write_canonical_rollout(
input: &CanonicalizationSource<'_>,
plan: Option<&RollbackPlan>,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<CanonicalizationAttempt> {
let source_file = File::open(input.source_path)
.await
.map_err(migration_error)?;
@@ -494,6 +660,7 @@ impl LocalThreadStore {
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
let mut bytes = Vec::new();
let mut parsed_record_index = 0_usize;
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
let written = canonicalizer
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
@@ -506,24 +673,45 @@ impl LocalThreadStore {
let Some(line) = record.line else {
continue;
};
if matches!(
&line.item,
RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
) {
return Err(migration_error("legacy rollout contains a rollback marker"));
if plan.is_none()
&& matches!(
&line.item,
RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
)
{
return Ok(CanonicalizationAttempt::NeedsRollbackPlan);
}
let line = if let Some(plan) = plan {
let planned = plan.apply(parsed_record_index, line)?;
parsed_record_index = parsed_record_index
.checked_add(1)
.ok_or_else(|| migration_error("legacy rollout record index overflow"))?;
let Some(line) = planned else {
continue;
};
line
} else {
line
};
last_timestamp = line.timestamp.clone();
let written = canonicalizer.process_line(line, &mut staged).await?;
limiter.account(written).await;
}
if let Some(plan) = plan
&& parsed_record_index != plan.record_count()
{
return Err(migration_error(
"rollback plan source length changed during replay",
));
}
let written = canonicalizer.finish(&mut staged, &last_timestamp).await?;
limiter.account(written).await;
staged.flush().await.map_err(migration_error)?;
Ok((
canonicalizer.output_byte_offset(),
canonicalizer.next_ordinal(),
))
Ok(CanonicalizationAttempt::Complete {
expected_length: canonicalizer.output_byte_offset(),
expected_ordinal: canonicalizer.next_ordinal(),
})
}
async fn recover_published_migration(
@@ -532,7 +720,7 @@ impl LocalThreadStore {
rollout_path: &Path,
journal_path: &Path,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<()> {
) -> ThreadStoreResult<PathBuf> {
let _writer_guard = self.writer_lock_coordinator.acquire(thread_id)?;
let decompressed_path = rollout_path_is_compressed(rollout_path)
.then(|| decompressed_staged_rollout_path(rollout_path))
@@ -572,7 +760,8 @@ impl LocalThreadStore {
remove_file_if_present(decompressed_path).await?;
}
self.finish_published_migration(thread_id, journal_path)
.await
.await?;
Ok(rollout_path.to_path_buf())
}
async fn cleanup_failed_unpublished_migration(
@@ -589,7 +778,9 @@ impl LocalThreadStore {
}
thread_history::delete_thread(self, thread_id).await?;
remove_file_if_present(&staged_rollout_path(rollout_path)?).await?;
let staged_path = staged_rollout_path(rollout_path)?;
remove_file_if_present(&staged_path).await?;
remove_file_if_present(&rewritten_staged_rollout_path(&staged_path)?).await?;
remove_file_if_present(&compressed_staged_rollout_path(rollout_path)?).await?;
remove_file_if_present(&decompressed_staged_rollout_path(rollout_path)?).await?;
remove_file_if_present(journal_path).await?;
@@ -623,6 +814,11 @@ impl LocalThreadStore {
rollout_path: &Path,
limiter: &mut RolloutMigrationRateLimiter,
) -> ThreadStoreResult<()> {
let subagent_history_start_ordinal = codex_rollout::read_session_meta_line(rollout_path)
.await
.map_err(migration_error)?
.meta
.subagent_history_start_ordinal;
let file = File::open(rollout_path).await.map_err(migration_error)?;
let mut reader = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, file);
let mut line_bytes = Vec::new();
@@ -650,7 +846,11 @@ impl LocalThreadStore {
start_byte_offset: offset,
end_byte_offset: next_offset,
fallback_created_at_ms: Some(fallback_created_at_ms),
changes: project_rollout_line(&line),
changes: if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {
Default::default()
} else {
project_rollout_line(&line)
},
}));
offset = next_offset;
@@ -688,7 +888,7 @@ async fn read_rollout_record(
bytes: &mut Vec<u8>,
) -> ThreadStoreResult<Option<RolloutRecord>> {
bytes.clear();
let byte_count = reader
let mut byte_count = reader
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
.read_until(b'\n', bytes)
.await
@@ -697,11 +897,30 @@ async fn read_rollout_record(
return Ok(None);
}
if byte_count > MAX_ROLLOUT_LINE_BYTES {
return Err(migration_error(
"rollout contains an oversized JSONL record",
));
// Some historical tool outputs are too large to migrate safely in memory. Discard the
// whole record, including any unread suffix.
while bytes.last() != Some(&b'\n') {
bytes.clear();
let chunk_bytes = reader
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
.read_until(b'\n', bytes)
.await
.map_err(migration_error)?;
if chunk_bytes == 0 {
break;
}
byte_count = byte_count
.checked_add(chunk_bytes)
.ok_or_else(|| migration_error("rollout record byte count overflow"))?;
}
return Ok(Some(RolloutRecord {
line: None,
byte_count: byte_count as u64,
}));
}
let line = line_parser::parse_legacy_rollout_line(bytes).map_err(migration_error)?;
// Legacy records do not have ordinals, so malformed complete records cannot be repaired.
// Skip them and let the next newline-delimited record resynchronize the stream.
let line = line_parser::parse_legacy_rollout_line(bytes).unwrap_or(None);
Ok(Some(RolloutRecord {
line,
byte_count: byte_count as u64,
@@ -734,7 +953,15 @@ async fn find_rollout_paths(root: &Path) -> ThreadStoreResult<Vec<PathBuf>> {
if name.starts_with("rollout-")
&& (name.ends_with(".jsonl") || name.ends_with(".jsonl.zst"))
{
paths.push(entry.path());
let path = entry.path();
if name.ends_with(".jsonl.zst")
&& tokio::fs::try_exists(codex_rollout::plain_rollout_path(&path))
.await
.map_err(migration_error)?
{
continue;
}
paths.push(path);
}
}
}
@@ -747,6 +974,12 @@ fn matches_selection(selected: &[ThreadId], actual: Option<ThreadId>) -> bool {
selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))
}
fn rollout_path_is_compressed(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(".jsonl.zst"))
}
fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
let name = path.file_name()?.to_str()?;
let stem = name
@@ -756,12 +989,6 @@ fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
ThreadId::from_string(stem.get(start..)?).ok()
}
fn rollout_path_is_compressed(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(".jsonl.zst"))
}
fn migration_outcome(
thread_id: ThreadId,
rollout_path: PathBuf,
@@ -1,8 +1,9 @@
//! Replays normalized legacy rollout records into canonical paginated JSONL.
//!
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at
//! ordinal zero, and emits the surviving history selected by the caller.
//! events into modern turn items. Rollback planning happens before this writer sees a record, so
//! this module only assigns stable ordinals, keeps `SessionMeta` at ordinal zero, and emits the
//! already-selected surviving history.
//!
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
@@ -52,7 +53,6 @@ pub(super) struct LegacyRolloutCanonicalizer {
active_turn: Option<ActiveTurn>,
known_turn_ids: HashSet<String>,
reasoning: Option<ReasoningItem>,
saw_source_session_meta: bool,
}
impl LegacyRolloutCanonicalizer {
@@ -67,18 +67,17 @@ impl LegacyRolloutCanonicalizer {
active_turn: None,
known_turn_ids: HashSet::new(),
reasoning: None,
saw_source_session_meta: false,
}
}
pub(super) fn output_byte_offset(&self) -> u64 {
self.output_byte_offset
}
pub(super) fn next_ordinal(&self) -> u64 {
self.next_ordinal
}
pub(super) fn output_byte_offset(&self) -> u64 {
self.output_byte_offset
}
pub(super) async fn write_head_session_meta(
&mut self,
line: RolloutLine,
@@ -114,14 +113,7 @@ impl LegacyRolloutCanonicalizer {
let timestamp = line.timestamp;
let bytes_before = self.bytes_written;
match line.item {
RolloutItem::SessionMeta(metadata) => {
if !self.saw_source_session_meta {
self.saw_source_session_meta = true;
return Ok(0);
}
self.write_item(writer, &timestamp, RolloutItem::SessionMeta(metadata))
.await?;
}
RolloutItem::SessionMeta(_) => return Ok(0),
RolloutItem::ResponseItem(ResponseItem::Other) => {
return Err(migration_error(
"legacy rollout contains an unsupported response item",
@@ -26,7 +26,11 @@ pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLi
// Deserializing through Value is intentional. Some historical numeric
// payloads fail serde_json's streaming enum path but deserialize correctly
// once their tagged object shape has been materialized.
let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
let value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
parse_legacy_rollout_value(value)
}
pub(super) fn parse_legacy_rollout_value(mut value: Value) -> Result<Option<RolloutLine>, String> {
if should_skip_retired_record(&value) {
return Ok(None);
}
@@ -22,6 +22,13 @@ use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::fs::PermissionsExt;
use codex_protocol::ThreadId;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use tokio::fs::File;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::io::BufWriter;
use super::migration_error;
use crate::ThreadStoreResult;
@@ -139,6 +146,74 @@ pub(super) async fn compress_rollout_to_path(
.map_err(migration_error)
}
pub(super) fn rewritten_staged_rollout_path(staged_path: &Path) -> ThreadStoreResult<PathBuf> {
let filename = staged_path
.file_name()
.and_then(|filename| filename.to_str())
.ok_or_else(|| migration_error("staged rollout path has no valid filename"))?;
Ok(staged_path.with_file_name(format!("{filename}.head.tmp")))
}
pub(super) async fn rewrite_subagent_history_boundary(
staged_path: &Path,
boundary: u64,
) -> ThreadStoreResult<()> {
let rewritten_path = rewritten_staged_rollout_path(staged_path)?;
let permissions = tokio::fs::metadata(staged_path)
.await
.map_err(migration_error)?
.permissions();
let source = File::open(staged_path).await.map_err(migration_error)?;
let mut source = BufReader::new(source);
let mut head_bytes = Vec::new();
source
.read_until(b'\n', &mut head_bytes)
.await
.map_err(migration_error)?;
let mut head = serde_json::from_slice::<RolloutLine>(&head_bytes).map_err(migration_error)?;
let RolloutItem::SessionMeta(session_meta) = &mut head.item else {
return Err(migration_error(
"staged rollout head is not session metadata",
));
};
session_meta.meta.subagent_history_start_ordinal = Some(boundary);
let rewritten = tokio::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&rewritten_path)
.await
.map_err(migration_error)?;
rewritten
.set_permissions(permissions)
.await
.map_err(migration_error)?;
let mut rewritten = BufWriter::new(rewritten);
rewritten
.write_all(
serde_json::to_string(&head)
.map_err(migration_error)?
.as_bytes(),
)
.await
.map_err(migration_error)?;
rewritten.write_all(b"\n").await.map_err(migration_error)?;
tokio::io::copy(&mut source, &mut rewritten)
.await
.map_err(migration_error)?;
rewritten.flush().await.map_err(migration_error)?;
rewritten
.get_ref()
.sync_all()
.await
.map_err(migration_error)?;
drop(rewritten);
tokio::fs::rename(rewritten_path, staged_path)
.await
.map_err(migration_error)
}
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
tokio::fs::remove_file(path)
@@ -0,0 +1,140 @@
//! Defines which persisted history entries count as rollbackable user turns.
//!
//! Legacy `ThreadRolledBack { num_turns }` does not mean “drop the last N JSONL records.” A turn
//! can include contextual user/developer messages, paired `ResponseItem` + `UserMessage` records,
//! inter-agent communication, or compaction replacement history.
//!
//! This module mirrors the legacy cold-resume rollback boundary rules for persisted rollout shapes.
//! It stays local to migration because core's live predicate also knows about dynamically
//! registered contextual fragments that thread-store should not depend on.
use codex_protocol::items::parse_hook_prompt_fragment;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::InterAgentCommunication;
/// Match the rollback boundaries used by legacy history reconstruction without
/// treating every persisted lifecycle as a user turn. This intentionally stays
/// local to the frozen migration adapter: the full core predicate also knows
/// about dynamically registered contextual fragments that thread-store cannot
/// depend on.
pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool {
if matches!(response, ResponseItem::AgentMessage { .. }) {
return true;
}
let ResponseItem::Message { role, content, .. } = response else {
return false;
};
(role == "user" && !is_known_contextual_user_message_content(content))
|| (role == "assistant" && InterAgentCommunication::is_message_content(content))
}
pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool {
let ResponseItem::Message { role, content, .. } = response else {
return false;
};
(role == "user" && is_known_contextual_user_message_content(content))
|| (role == "developer" && is_known_contextual_developer_message_content(content))
}
/// Apply the same response-history cut used by legacy cold resume to a persisted
/// compaction checkpoint. The migration adapter uses a frozen contextual-fragment
/// matcher because thread-store cannot depend on core's runtime fragment registry.
pub(super) fn drop_last_n_user_turns(history: &mut Vec<ResponseItem>, num_turns: u32) {
if num_turns == 0 {
return;
}
let user_positions = history
.iter()
.enumerate()
.filter_map(|(index, item)| counts_as_boundary(item).then_some(index))
.collect::<Vec<_>>();
let Some(&first_turn_index) = user_positions.first() else {
return;
};
let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
let mut cut_index = if count >= user_positions.len() {
first_turn_index
} else {
user_positions[user_positions.len() - count]
};
while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1]) {
cut_index -= 1;
}
history.truncate(cut_index);
}
fn is_known_contextual_user_message_content(content: &[ContentItem]) -> bool {
content.iter().any(|item| {
let ContentItem::InputText { text } = item else {
return false;
};
is_known_contextual_user_text(text)
})
}
fn is_known_contextual_developer_message_content(content: &[ContentItem]) -> bool {
content.iter().any(|item| {
let ContentItem::InputText { text } = item else {
return false;
};
let text = text.trim_start();
[
"<permissions instructions>",
"<model_switch>",
"<apps_instructions>",
"<collaboration_mode>",
"<multi_agent_mode>",
"<environments_instructions>",
"<git_attribution>",
"<plugins_instructions>",
"<realtime_conversation>",
"<skills_instructions>",
"<tools>",
"<personality_spec>",
"<token_budget>",
"<context_window>",
"<context_window_guidance>",
"<rollout_budget>",
]
.iter()
.any(|prefix| {
text.get(..prefix.len())
.is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
})
})
}
// Keep this frozen alongside the legacy migration adapter. Core's live predicate also knows
// about dynamically registered fragments, but legacy rollouts only need these persisted shapes.
fn is_known_contextual_user_text(text: &str) -> bool {
let text = text.trim();
parse_hook_prompt_fragment(text).is_some()
|| [
("# AGENTS.md instructions", "</INSTRUCTIONS>"),
("<environment_context>", "</environment_context>"),
("<skill>", "</skill>"),
("<user_shell_command>", "</user_shell_command>"),
("<turn_aborted>", "</turn_aborted>"),
("<subagent_notification>", "</subagent_notification>"),
("<recommended_plugins>", "</recommended_plugins>"),
("<goal_context>", "</goal_context>"),
]
.iter()
.any(|(start, end)| text.starts_with(start) && text.ends_with(end))
|| (text.starts_with("<external_")
&& text
.split_once('>')
.and_then(|(start, _)| start.strip_prefix("<external_"))
.is_some_and(|key| text.ends_with(&format!("</external_{key}>"))))
|| (text.starts_with("<codex_internal_context")
&& text.ends_with("</codex_internal_context>"))
|| text.starts_with(
"Warning: The maximum number of unified exec processes you can keep open is",
)
|| (text.starts_with("Warning: apply_patch was requested via ")
&& text.ends_with("Use the apply_patch tool instead of exec_command."))
|| text.starts_with(
"Warning: Your account was flagged for potentially high-risk cyber activity",
)
}
@@ -0,0 +1,380 @@
//! Decides which legacy records remain visible after historical rollback.
//!
//! Legacy rollback removes logical instruction turns, not a physical suffix of the rollout file.
//! Most records happen to be ordered that way, but late completion events can target an older
//! surviving turn after a newer turn has started. This planner keeps compact per-record ownership
//! metadata for SQLite visibility, then combines it with `rollback_replay`'s cold-resume answer
//! before the writer makes its second streaming pass.
use std::collections::HashMap;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::CompactedItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::UserMessageEvent;
use super::migration_error;
use super::rollback;
use super::rollback_replay::ModelReplayPlanner;
use crate::ThreadStoreResult;
#[derive(Clone)]
struct CompactionFrame {
record_index: usize,
boundary_depth: usize,
owner: Option<usize>,
item: CompactedItem,
}
#[derive(Clone)]
struct PendingUserResponse {
boundary: usize,
content: Vec<ContentItem>,
}
/// Compact plan keyed by parsed source-record index.
pub(super) struct RollbackPlan {
record_boundaries: Vec<Option<usize>>,
boundary_alive: Vec<bool>,
compacted_items: HashMap<usize, CompactedItem>,
}
impl RollbackPlan {
pub(super) fn record_count(&self) -> usize {
self.record_boundaries.len()
}
pub(super) fn apply(
&self,
record_index: usize,
mut line: RolloutLine,
) -> ThreadStoreResult<Option<RolloutLine>> {
let boundary = self
.record_boundaries
.get(record_index)
.ok_or_else(|| migration_error("rollback plan is shorter than source replay"))?;
if matches!(
&line.item,
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))
) {
return Ok(None);
}
// A rolled-back turn can still own the empty checkpoint that keeps cold resume from
// replaying older history.
if let Some(compacted) = self.compacted_items.get(&record_index) {
line.item = RolloutItem::Compacted(compacted.clone());
return Ok(Some(line));
}
if boundary.is_some_and(|boundary| !self.boundary_alive[boundary]) {
return Ok(None);
}
Ok(Some(line))
}
}
/// Streaming builder for RollbackPlan.
pub(super) struct RollbackPlanner {
record_boundaries: Vec<Option<usize>>,
boundary_alive: Vec<bool>,
boundary_stack: Vec<usize>,
active_turn_id: Option<String>,
pending_turn_records: Vec<usize>,
pending_context_records: Vec<usize>,
pending_user_response: Option<PendingUserResponse>,
pending_delivery_boundary: Option<usize>,
turn_boundaries: HashMap<String, usize>,
compactions: Vec<CompactionFrame>,
model_replay: ModelReplayPlanner,
}
impl RollbackPlanner {
pub(super) fn new() -> Self {
Self {
record_boundaries: Vec::new(),
boundary_alive: Vec::new(),
boundary_stack: Vec::new(),
active_turn_id: None,
pending_turn_records: Vec::new(),
pending_context_records: Vec::new(),
pending_user_response: None,
pending_delivery_boundary: None,
turn_boundaries: HashMap::new(),
compactions: Vec::new(),
model_replay: ModelReplayPlanner::new(),
}
}
pub(super) fn observe(&mut self, line: &RolloutLine) -> ThreadStoreResult<()> {
let index = self.record_boundaries.len();
self.model_replay.observe(index, &line.item);
self.record_boundaries
.push(self.boundary_stack.last().copied());
let paired_user_boundary = match (&self.pending_user_response, &line.item) {
(Some(pending), RolloutItem::EventMsg(EventMsg::UserMessage(event)))
if user_response_matches_event(&pending.content, event) =>
{
Some(pending.boundary)
}
_ => None,
};
let paired_delivery_boundary = match (&self.pending_delivery_boundary, &line.item) {
(Some(boundary), RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })) => {
Some(*boundary)
}
_ => None,
};
self.pending_user_response = None;
self.pending_delivery_boundary = None;
match &line.item {
RolloutItem::SessionMeta(_) => self.record_boundaries[index] = None,
RolloutItem::ResponseItem(response) => {
if let Some(boundary) = paired_delivery_boundary {
self.record_boundaries[index] = Some(boundary);
} else if rollback::counts_as_boundary(response) {
let boundary = self.start_boundary(index);
if let ResponseItem::Message { role, content, .. } = response
&& role == "user"
{
self.pending_user_response = Some(PendingUserResponse {
boundary,
content: content.clone(),
});
}
} else if rollback::is_pre_turn_context_update(response) {
// Until another user boundary arrives, this is trailing context for the
// previous turn. Keep that fallback owner so rollback drops it when there is
// no later turn to attach it to.
self.pending_context_records.push(index);
}
}
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
self.apply_rollback(rollback.num_turns)?;
}
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
self.active_turn_id = Some(event.turn_id.clone());
self.pending_turn_records.clear();
self.pending_turn_records.push(index);
}
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
self.assign_targeted_record(index, Some(event.turn_id.as_str()));
if self.active_turn_id.as_deref() == Some(event.turn_id.as_str()) {
self.active_turn_id = None;
self.pending_turn_records.clear();
}
}
RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
self.assign_targeted_record(index, event.turn_id.as_deref());
if event
.turn_id
.as_deref()
.is_some_and(|turn_id| self.active_turn_id.as_deref() == Some(turn_id))
{
self.active_turn_id = None;
self.pending_turn_records.clear();
}
}
RolloutItem::EventMsg(EventMsg::UserMessage(_)) => {
let boundary = paired_user_boundary.unwrap_or_else(|| self.start_boundary(index));
self.record_boundaries[index] = Some(boundary);
}
RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => {
self.assign_targeted_record(index, Some(event.turn_id.as_str()));
}
RolloutItem::EventMsg(event) => {
self.assign_targeted_record(index, explicit_event_turn_id(event));
}
RolloutItem::InterAgentCommunication(_) => {
self.start_boundary(index);
}
RolloutItem::InterAgentCommunicationMetadata { .. } => {
let boundary = self.start_boundary(index);
self.pending_delivery_boundary = Some(boundary);
}
RolloutItem::Compacted(item) => {
let owner = self
.active_turn_id
.as_deref()
.and_then(|turn_id| self.turn_boundaries.get(turn_id).copied());
self.record_boundaries[index] = owner;
self.compactions.push(CompactionFrame {
record_index: index,
boundary_depth: self.boundary_stack.len(),
owner,
item: item.clone(),
});
}
RolloutItem::TurnContext(_) => {
if self.active_turn_id.is_some()
&& self
.active_turn_id
.as_deref()
.is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
{
self.pending_turn_records.push(index);
}
}
RolloutItem::WorldState(_) => {}
}
Ok(())
}
pub(super) fn finish(self) -> RollbackPlan {
let RollbackPlanner {
record_boundaries,
boundary_alive,
compactions,
model_replay,
..
} = self;
let replay_anchor = model_replay.finish().empty_replacement_history_compaction;
let compacted_items = compactions
.into_iter()
.filter_map(|mut frame| {
if Some(frame.record_index) == replay_anchor {
frame.item.replacement_history = Some(Vec::new());
return Some((frame.record_index, frame.item));
}
frame
.owner
.is_none_or(|boundary| boundary_alive[boundary])
.then_some((frame.record_index, frame.item))
})
.collect::<HashMap<_, _>>();
RollbackPlan {
record_boundaries,
boundary_alive,
compacted_items,
}
}
fn start_boundary(&mut self, index: usize) -> usize {
let boundary = self.boundary_alive.len();
self.boundary_alive.push(true);
let had_prior_boundary = !self.boundary_stack.is_empty();
if had_prior_boundary {
for pending_index in self.pending_context_records.drain(..) {
self.record_boundaries[pending_index] = Some(boundary);
}
} else {
self.pending_context_records.clear();
}
for pending_index in self.pending_turn_records.drain(..) {
self.record_boundaries[pending_index] = Some(boundary);
}
self.record_boundaries[index] = Some(boundary);
self.boundary_stack.push(boundary);
self.bind_active_turn(boundary);
boundary
}
fn bind_active_turn(&mut self, boundary: usize) {
if let Some(turn_id) = self.active_turn_id.as_ref() {
self.turn_boundaries.insert(turn_id.clone(), boundary);
}
}
fn assign_targeted_record(&mut self, index: usize, turn_id: Option<&str>) {
if let Some(boundary) = turn_id.and_then(|turn_id| self.turn_boundaries.get(turn_id)) {
self.record_boundaries[index] = Some(*boundary);
} else if self.active_turn_id.is_some()
&& self
.active_turn_id
.as_deref()
.is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
{
self.pending_turn_records.push(index);
}
}
fn apply_rollback(&mut self, num_turns: u32) -> ThreadStoreResult<()> {
let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
if count == 0 {
return Ok(());
}
let depth_before = self.boundary_stack.len();
for _ in 0..count {
let Some(boundary) = self.boundary_stack.pop() else {
break;
};
self.boundary_alive[boundary] = false;
}
let compaction_index = self.compactions.iter().rposition(|frame| {
frame
.owner
.is_none_or(|boundary| self.boundary_alive[boundary])
});
if let Some(compaction_index) = compaction_index {
let frame = &mut self.compactions[compaction_index];
let post_compaction_turns = depth_before.saturating_sub(frame.boundary_depth);
let remaining = count.saturating_sub(post_compaction_turns);
if remaining > 0 {
let replacement_history =
frame.item.replacement_history.as_mut().ok_or_else(|| {
migration_error(
"legacy rollback crosses a compaction without replacement history",
)
})?;
rollback::drop_last_n_user_turns(
replacement_history,
u32::try_from(remaining).unwrap_or(u32::MAX),
);
}
}
self.active_turn_id = None;
self.pending_turn_records.clear();
self.pending_context_records.clear();
self.pending_user_response = None;
self.pending_delivery_boundary = None;
Ok(())
}
}
fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {
match event {
EventMsg::ExecCommandEnd(event) => Some(event.turn_id.as_str()),
EventMsg::PatchApplyEnd(event) => Some(event.turn_id.as_str()),
EventMsg::DynamicToolCallResponse(event) => Some(event.turn_id.as_str()),
EventMsg::EnteredReviewMode(event) => event.turn_id.as_deref(),
EventMsg::ExitedReviewMode(event) => event.turn_id.as_deref(),
_ => None,
}
.filter(|turn_id| !turn_id.is_empty())
}
fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {
let mut text = String::new();
let mut images = Vec::new();
let mut audio = Vec::new();
for item in content {
match item {
ContentItem::InputText { text: item_text } => text.push_str(item_text),
ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
ContentItem::OutputText { .. } => return false,
}
}
text == event.message
&& images
== event
.images
.as_deref()
.unwrap_or_default()
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
&& audio
== event
.audio
.as_deref()
.unwrap_or_default()
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
&& event.local_images.is_empty()
&& event.local_audio.is_empty()
}
@@ -0,0 +1,191 @@
//! Mirrors the part of legacy cold resume that chooses the model-history checkpoint.
//!
//! Legacy rollback has two jobs. The forward history reducer decides which turns remain visible,
//! while cold resume scans backward to find the newest surviving compaction base. Migration needs
//! both answers because removing rollback markers must not change what the next resume sends to
//! the model.
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use super::rollback;
enum ReplayRecord {
Compacted {
record_index: usize,
has_replacement_history: bool,
},
Rollback(u32),
TurnStarted(String),
TurnComplete(String),
TurnAborted(Option<String>),
TurnContext(Option<String>),
UserBoundary,
}
#[derive(Default)]
struct ReverseSegment {
turn_id: Option<String>,
counts_as_user_turn: bool,
compaction_index: Option<usize>,
}
/// Rewrite needed when legacy reverse replay skips a compaction segment but still resumes from
/// the suffix after that checkpoint.
pub(super) struct ModelReplayPlan {
pub(super) empty_replacement_history_compaction: Option<usize>,
}
pub(super) struct ModelReplayPlanner {
records: Vec<ReplayRecord>,
}
impl ModelReplayPlanner {
pub(super) fn new() -> Self {
Self {
records: Vec::new(),
}
}
pub(super) fn observe(&mut self, record_index: usize, item: &RolloutItem) {
let record = match item {
RolloutItem::Compacted(compacted) => ReplayRecord::Compacted {
record_index,
has_replacement_history: compacted.replacement_history.is_some(),
},
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
ReplayRecord::Rollback(rollback.num_turns)
}
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
ReplayRecord::TurnStarted(event.turn_id.clone())
}
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
ReplayRecord::TurnComplete(event.turn_id.clone())
}
RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
ReplayRecord::TurnAborted(event.turn_id.clone())
}
RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()),
RolloutItem::EventMsg(EventMsg::UserMessage(_))
| RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary,
RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => {
ReplayRecord::UserBoundary
}
RolloutItem::SessionMeta(_)
| RolloutItem::ResponseItem(_)
| RolloutItem::EventMsg(_)
| RolloutItem::InterAgentCommunicationMetadata { .. }
| RolloutItem::WorldState(_) => return,
};
self.records.push(record);
}
pub(super) fn finish(self) -> ModelReplayPlan {
let mut selected_compaction = None;
let mut suffix_start = None;
let mut pending_rollback_turns = 0usize;
let mut active_segment: Option<ReverseSegment> = None;
for record in self.records.iter().rev() {
match record {
ReplayRecord::Compacted {
record_index,
has_replacement_history,
} => {
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
if segment.compaction_index.is_none() && *has_replacement_history {
segment.compaction_index = Some(*record_index);
suffix_start = record_index.checked_add(1);
}
}
ReplayRecord::Rollback(num_turns) => {
pending_rollback_turns = pending_rollback_turns
.saturating_add(usize::try_from(*num_turns).unwrap_or(usize::MAX));
}
ReplayRecord::TurnComplete(turn_id) => {
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
if segment.turn_id.is_none() {
segment.turn_id = Some(turn_id.clone());
}
}
ReplayRecord::TurnAborted(turn_id) => {
if let Some(segment) = active_segment.as_mut() {
if segment.turn_id.is_none() {
segment.turn_id = turn_id.clone();
}
} else if turn_id.is_some() {
active_segment = Some(ReverseSegment {
turn_id: turn_id.clone(),
..Default::default()
});
}
}
ReplayRecord::TurnContext(turn_id) => {
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
if segment.turn_id.is_none() {
segment.turn_id = turn_id.clone();
}
}
ReplayRecord::UserBoundary => {
active_segment
.get_or_insert_with(ReverseSegment::default)
.counts_as_user_turn = true;
}
ReplayRecord::TurnStarted(turn_id) => {
if active_segment.as_ref().is_some_and(|segment| {
turn_ids_are_compatible(segment.turn_id.as_deref(), Some(turn_id.as_str()))
}) && let Some(segment) = active_segment.take()
{
finalize_segment(
segment,
&mut selected_compaction,
&mut pending_rollback_turns,
);
}
}
}
}
if let Some(segment) = active_segment {
finalize_segment(
segment,
&mut selected_compaction,
&mut pending_rollback_turns,
);
}
// If reverse replay saw a replacement-history checkpoint but rolled back its segment,
// it resumes from that checkpoint's suffix without using the checkpoint history. Keep an
// empty checkpoint in the paginated file so older SQLite-visible records do not leak back
// into model context after the rollback markers are gone.
let empty_replacement_history_compaction = selected_compaction
.is_none()
.then_some(suffix_start)
.flatten()
.and_then(|start| start.checked_sub(1));
ModelReplayPlan {
empty_replacement_history_compaction,
}
}
}
fn finalize_segment(
segment: ReverseSegment,
selected_compaction: &mut Option<usize>,
pending_rollback_turns: &mut usize,
) {
if *pending_rollback_turns > 0 {
if segment.counts_as_user_turn {
*pending_rollback_turns -= 1;
}
return;
}
if selected_compaction.is_none() {
*selected_compaction = segment.compaction_index;
}
}
fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool {
active_turn_id
.is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id))
}
@@ -0,0 +1,59 @@
//! Selects a smaller resume context for legacy subagent rollouts.
//!
//! Legacy subagents copied the parent's full rollout into every child rollout. Migrating those
//! files verbatim can preserve gigabytes of duplicated history even though a resumed subagent only
//! needs its latest bounded model context.
//!
//! This module reverse-scans the legacy rollout for a safe model-context checkpoint and returns the
//! smallest replay needed to resume from there. Malformed reverse-scanned records are skipped. If
//! we cannot prove that a bounded replay is safe, we return `None` and let the caller fall back to
//! full tolerant replay.
use std::fs::File;
use std::path::PathBuf;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::SessionMetaLine;
use codex_rollout::ModelContextScan;
use codex_rollout::ModelContextScanProgress;
use codex_rollout::ReverseJsonlScanner;
use codex_rollout::ScanOutcome;
use serde_json::Value;
use super::line_parser;
use super::migration_error;
use crate::ThreadStoreResult;
/// Returns a bounded replay when the legacy child has enough durable context to reconstruct from
/// a suffix. Without that proof, the caller streams the complete legacy replay instead.
pub(super) async fn select_bounded_context(
rollout_path: PathBuf,
session_meta: SessionMetaLine,
) -> ThreadStoreResult<Option<Vec<RolloutItem>>> {
tokio::task::spawn_blocking(move || {
let file = File::open(rollout_path).map_err(migration_error)?;
let mut scanner = ReverseJsonlScanner::new(file)
.map_err(migration_error)?
.with_max_record_bytes(super::MAX_ROLLOUT_LINE_BYTES);
let mut scan = ModelContextScan::default();
while let Some(outcome) = scanner.scan_next::<Value>().map_err(migration_error)? {
let value = match outcome {
ScanOutcome::Parsed(value) => value,
ScanOutcome::Rejected(_) => continue,
};
let Ok(Some(line)) = line_parser::parse_legacy_rollout_value(value) else {
continue;
};
if scan.push(line.item) == ModelContextScanProgress::Complete {
let mut items = scan.finish(session_meta);
items.retain(|item| !matches!(item, RolloutItem::SessionMeta(_)));
return Ok(Some(items));
}
}
Ok(None)
})
.await
.map_err(migration_error)?
}
File diff suppressed because it is too large. Load diff