mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Preserve legacy semantics during rollout migration (#37191)
## Why Legacy rollouts can contain historical rollbacks, compaction checkpoints, and subagent copies of parent history. Migrating those records verbatim can change the visible conversation or the model context used when a thread resumes, and can retain large amounts of duplicated subagent history. ## What changed - Plan and replay historical rollbacks by logical turn, preserving surviving lifecycle events and the correct compaction anchor for future resumes. - Migrate subagent rollouts from a safe bounded replay when possible and record the history boundary so copied parent context is not projected as child turns. - Include archived and compressed rollouts while keeping their storage form and location intact. - Tolerate malformed, partial, oversized, and empty JSONL records during the migration scan. ## Testing Add coverage for rollback and compaction combinations, subagent replay bounds, archived and compressed rollouts, recovery, concurrent maintenance, and damaged JSONL input. GitOrigin-RevId: 3948e16d43f75df8571b0a92af12a181d1d08fdf
This commit is contained in:
1 parent
f141dc77f0
commit
aac9f84247
11 files changed
+2752
-251
No files matched your search
@@ -23,6 +23,8 @@ pub struct ReverseJsonlScanner<R> {
|
|||||||
chunk_position: usize,
|
chunk_position: usize,
|
||||||
chunk: Vec<u8>,
|
chunk: Vec<u8>,
|
||||||
record_reversed: Vec<u8>,
|
record_reversed: Vec<u8>,
|
||||||
|
max_record_bytes: Option<usize>,
|
||||||
|
discarding_oversized_record: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<R> ReverseJsonlScanner<R>
|
impl<R> ReverseJsonlScanner<R>
|
||||||
@@ -52,9 +54,17 @@ where
|
|||||||
chunk_position: 0,
|
chunk_position: 0,
|
||||||
chunk: vec![0; READ_CHUNK_SIZE],
|
chunk: vec![0; READ_CHUNK_SIZE],
|
||||||
record_reversed: Vec::new(),
|
record_reversed: Vec::new(),
|
||||||
|
max_record_bytes: None,
|
||||||
|
discarding_oversized_record: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Skips records larger than the configured limit without buffering or parsing them.
|
||||||
|
pub fn with_max_record_bytes(mut self, max_record_bytes: usize) -> Self {
|
||||||
|
self.max_record_bytes = Some(max_record_bytes);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Scans the next nonblank record.
|
/// Scans the next nonblank record.
|
||||||
///
|
///
|
||||||
/// I/O failures are returned as [`Err`]. Invalid JSON records are returned as
|
/// I/O failures are returned as [`Err`]. Invalid JSON records are returned as
|
||||||
@@ -66,6 +76,10 @@ where
|
|||||||
loop {
|
loop {
|
||||||
if self.chunk_position == 0 {
|
if self.chunk_position == 0 {
|
||||||
if self.next_chunk_end == 0 {
|
if self.next_chunk_end == 0 {
|
||||||
|
if self.discarding_oversized_record {
|
||||||
|
self.discarding_oversized_record = false;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
return Ok(self.finish_record());
|
return Ok(self.finish_record());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,14 +93,36 @@ where
|
|||||||
|
|
||||||
let chunk = &self.chunk[..self.chunk_position];
|
let chunk = &self.chunk[..self.chunk_position];
|
||||||
if let Some(newline_position) = chunk.iter().rposition(|byte| *byte == b'\n') {
|
if let Some(newline_position) = chunk.iter().rposition(|byte| *byte == b'\n') {
|
||||||
self.record_reversed
|
let fragment = &chunk[newline_position + 1..];
|
||||||
.extend(chunk[newline_position + 1..].iter().rev().copied());
|
if !self.discarding_oversized_record {
|
||||||
|
if self.max_record_bytes.is_some_and(|max_record_bytes| {
|
||||||
|
self.record_reversed.len().saturating_add(fragment.len()) > max_record_bytes
|
||||||
|
}) {
|
||||||
|
self.record_reversed.clear();
|
||||||
|
self.discarding_oversized_record = true;
|
||||||
|
} else {
|
||||||
|
self.record_reversed.extend(fragment.iter().rev().copied());
|
||||||
|
}
|
||||||
|
}
|
||||||
self.chunk_position = newline_position;
|
self.chunk_position = newline_position;
|
||||||
|
if self.discarding_oversized_record {
|
||||||
|
self.discarding_oversized_record = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if let Some(outcome) = self.finish_record() {
|
if let Some(outcome) = self.finish_record() {
|
||||||
return Ok(Some(outcome));
|
return Ok(Some(outcome));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
self.record_reversed.extend(chunk.iter().rev().copied());
|
if !self.discarding_oversized_record {
|
||||||
|
if self.max_record_bytes.is_some_and(|max_record_bytes| {
|
||||||
|
self.record_reversed.len().saturating_add(chunk.len()) > max_record_bytes
|
||||||
|
}) {
|
||||||
|
self.record_reversed.clear();
|
||||||
|
self.discarding_oversized_record = true;
|
||||||
|
} else {
|
||||||
|
self.record_reversed.extend(chunk.iter().rev().copied());
|
||||||
|
}
|
||||||
|
}
|
||||||
self.chunk_position = 0;
|
self.chunk_position = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,6 +68,21 @@ not-json
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn skips_records_over_the_configured_limit() -> std::io::Result<()> {
|
||||||
|
let oversized = record(&"x".repeat(128));
|
||||||
|
let input = format!(
|
||||||
|
"{}\n{}\n{}\n",
|
||||||
|
serde_json::to_string(&record("first"))?,
|
||||||
|
serde_json::to_string(&oversized)?,
|
||||||
|
serde_json::to_string(&record("third"))?
|
||||||
|
);
|
||||||
|
let mut scanner = ReverseJsonlScanner::new(Cursor::new(input.into_bytes()))?
|
||||||
|
.with_max_record_bytes(/*max_record_bytes*/ 32);
|
||||||
|
|
||||||
|
assert_records(&mut scanner, &["third", "first"])
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn accepts_valid_json_at_eof() -> std::io::Result<()> {
|
fn accepts_valid_json_at_eof() -> std::io::Result<()> {
|
||||||
let input = b"{\"value\":\"first\"}\n{\"value\":\"second\"}";
|
let input = b"{\"value\":\"first\"}\n{\"value\":\"second\"}";
|
||||||
|
|||||||
@@ -15,8 +15,11 @@ use std::time::Duration;
|
|||||||
use chrono::DateTime;
|
use chrono::DateTime;
|
||||||
use codex_app_server_protocol::project_rollout_line;
|
use codex_app_server_protocol::project_rollout_line;
|
||||||
use codex_protocol::ThreadId;
|
use codex_protocol::ThreadId;
|
||||||
|
use codex_protocol::protocol::InternalSessionSource;
|
||||||
use codex_protocol::protocol::RolloutItem;
|
use codex_protocol::protocol::RolloutItem;
|
||||||
use codex_protocol::protocol::RolloutLine;
|
use codex_protocol::protocol::RolloutLine;
|
||||||
|
use codex_protocol::protocol::SessionSource;
|
||||||
|
use codex_protocol::protocol::SubAgentSource;
|
||||||
use codex_protocol::protocol::ThreadHistoryMode;
|
use codex_protocol::protocol::ThreadHistoryMode;
|
||||||
use codex_protocol::protocol::ThreadSource;
|
use codex_protocol::protocol::ThreadSource;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
@@ -39,6 +42,10 @@ mod canonicalizer;
|
|||||||
mod legacy_event;
|
mod legacy_event;
|
||||||
mod line_parser;
|
mod line_parser;
|
||||||
mod publish;
|
mod publish;
|
||||||
|
mod rollback;
|
||||||
|
mod rollback_plan;
|
||||||
|
mod rollback_replay;
|
||||||
|
mod subagent;
|
||||||
|
|
||||||
use canonicalizer::LegacyRolloutCanonicalizer;
|
use canonicalizer::LegacyRolloutCanonicalizer;
|
||||||
use publish::compress_rollout_to_path;
|
use publish::compress_rollout_to_path;
|
||||||
@@ -48,13 +55,25 @@ use publish::decompressed_staged_rollout_path;
|
|||||||
use publish::migration_journal_path;
|
use publish::migration_journal_path;
|
||||||
use publish::pending_migration_thread_ids;
|
use publish::pending_migration_thread_ids;
|
||||||
use publish::remove_file_if_present;
|
use publish::remove_file_if_present;
|
||||||
|
use publish::rewrite_subagent_history_boundary;
|
||||||
|
use publish::rewritten_staged_rollout_path;
|
||||||
use publish::staged_rollout_path;
|
use publish::staged_rollout_path;
|
||||||
use publish::sync_parent_directory;
|
use publish::sync_parent_directory;
|
||||||
use publish::write_migration_journal;
|
use publish::write_migration_journal;
|
||||||
|
use rollback_plan::RollbackPlan;
|
||||||
|
use rollback_plan::RollbackPlanner;
|
||||||
|
|
||||||
const PROJECTION_BATCH_BYTES: u64 = 256 * 1024;
|
const PROJECTION_BATCH_BYTES: u64 = 256 * 1024;
|
||||||
const MAX_ROLLOUT_LINE_BYTES: usize = 16 * 1024 * 1024;
|
const MAX_ROLLOUT_LINE_BYTES: usize = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
enum CanonicalizationAttempt {
|
||||||
|
Complete {
|
||||||
|
expected_length: u64,
|
||||||
|
expected_ordinal: u64,
|
||||||
|
},
|
||||||
|
NeedsRollbackPlan,
|
||||||
|
}
|
||||||
|
|
||||||
struct CanonicalizationSource<'a> {
|
struct CanonicalizationSource<'a> {
|
||||||
thread_id: ThreadId,
|
thread_id: ThreadId,
|
||||||
source_path: &'a Path,
|
source_path: &'a Path,
|
||||||
@@ -99,7 +118,7 @@ pub enum RolloutMigrationStatus {
|
|||||||
Eligible,
|
Eligible,
|
||||||
Migrated,
|
Migrated,
|
||||||
AlreadyPaginated,
|
AlreadyPaginated,
|
||||||
SkippedSubagent,
|
SkippedEmpty,
|
||||||
SkippedBusy,
|
SkippedBusy,
|
||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
@@ -132,6 +151,12 @@ struct RolloutRecord {
|
|||||||
byte_count: u64,
|
byte_count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
enum RolloutMigrationKind {
|
||||||
|
Ordinary,
|
||||||
|
Subagent,
|
||||||
|
}
|
||||||
|
|
||||||
impl RolloutMigrationRateLimiter {
|
impl RolloutMigrationRateLimiter {
|
||||||
fn new(max_mib_per_second: u64) -> ThreadStoreResult<Self> {
|
fn new(max_mib_per_second: u64) -> ThreadStoreResult<Self> {
|
||||||
let bytes_per_second = max_mib_per_second
|
let bytes_per_second = max_mib_per_second
|
||||||
@@ -166,7 +191,7 @@ impl RolloutMigrationRateLimiter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl LocalThreadStore {
|
impl LocalThreadStore {
|
||||||
/// Inspect or migrate eligible legacy rollout files beneath active sessions.
|
/// Inspect or migrate eligible legacy rollout files beneath active and archived sessions.
|
||||||
pub async fn migrate_rollouts(
|
pub async fn migrate_rollouts(
|
||||||
&self,
|
&self,
|
||||||
options: RolloutMigrationOptions,
|
options: RolloutMigrationOptions,
|
||||||
@@ -186,6 +211,15 @@ impl LocalThreadStore {
|
|||||||
let mut paths =
|
let mut paths =
|
||||||
find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR))
|
find_rollout_paths(&self.config.codex_home.join(codex_rollout::SESSIONS_SUBDIR))
|
||||||
.await?;
|
.await?;
|
||||||
|
paths.extend(
|
||||||
|
find_rollout_paths(
|
||||||
|
&self
|
||||||
|
.config
|
||||||
|
.codex_home
|
||||||
|
.join(codex_rollout::ARCHIVED_SESSIONS_SUBDIR),
|
||||||
|
)
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
if options.mode == RolloutMigrationMode::Apply {
|
if options.mode == RolloutMigrationMode::Apply {
|
||||||
let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;
|
let pending_thread_ids = pending_migration_thread_ids(&self.config.codex_home).await?;
|
||||||
paths.sort_by_key(|path| {
|
paths.sort_by_key(|path| {
|
||||||
@@ -196,145 +230,167 @@ impl LocalThreadStore {
|
|||||||
let mut report = RolloutMigrationReport::default();
|
let mut report = RolloutMigrationReport::default();
|
||||||
|
|
||||||
for path in paths {
|
for path in paths {
|
||||||
let metadata = match codex_rollout::read_session_meta_line(&path).await {
|
if let Some(outcome) = self
|
||||||
Ok(metadata) => metadata,
|
.migrate_rollout_path(path, &options, &mut limiter)
|
||||||
Err(error) => {
|
.await?
|
||||||
let thread_id = thread_id_from_rollout_filename(&path);
|
|
||||||
if matches_selection(&options.thread_ids, thread_id) {
|
|
||||||
report.outcomes.push(RolloutMigrationOutcome {
|
|
||||||
thread_id,
|
|
||||||
rollout_path: path,
|
|
||||||
status: RolloutMigrationStatus::Failed,
|
|
||||||
bytes_processed: 0,
|
|
||||||
message: Some(error.to_string()),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let thread_id = metadata.meta.id;
|
|
||||||
if !matches_selection(&options.thread_ids, Some(thread_id)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if metadata.meta.source.is_non_root_agent()
|
|
||||||
|| matches!(
|
|
||||||
metadata.meta.thread_source,
|
|
||||||
Some(ThreadSource::Subagent | ThreadSource::MemoryConsolidation)
|
|
||||||
)
|
|
||||||
{
|
{
|
||||||
report.outcomes.push(RolloutMigrationOutcome {
|
report.outcomes.push(outcome);
|
||||||
thread_id: Some(thread_id),
|
|
||||||
rollout_path: path,
|
|
||||||
status: RolloutMigrationStatus::SkippedSubagent,
|
|
||||||
bytes_processed: 0,
|
|
||||||
message: None,
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
|
|
||||||
if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
|
|
||||||
let bytes_before = limiter.bytes_processed;
|
|
||||||
let result = if options.mode == RolloutMigrationMode::Apply
|
|
||||||
&& tokio::fs::try_exists(&journal_path)
|
|
||||||
.await
|
|
||||||
.map_err(migration_error)?
|
|
||||||
{
|
|
||||||
match self
|
|
||||||
.recover_published_migration(thread_id, &path, &journal_path, &mut limiter)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
|
|
||||||
Err(ThreadStoreError::Conflict { message }) => {
|
|
||||||
let bytes_processed =
|
|
||||||
limiter.bytes_processed.saturating_sub(bytes_before);
|
|
||||||
report.outcomes.push(skipped_busy_outcome(
|
|
||||||
thread_id,
|
|
||||||
path,
|
|
||||||
message,
|
|
||||||
bytes_processed,
|
|
||||||
));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Err(error) => Err(error),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Ok(RolloutMigrationStatus::AlreadyPaginated)
|
|
||||||
};
|
|
||||||
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
|
|
||||||
report
|
|
||||||
.outcomes
|
|
||||||
.push(migration_outcome(thread_id, path, result, bytes_processed));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if options.mode == RolloutMigrationMode::DryRun {
|
|
||||||
report.outcomes.push(RolloutMigrationOutcome {
|
|
||||||
thread_id: Some(thread_id),
|
|
||||||
rollout_path: path,
|
|
||||||
status: RolloutMigrationStatus::Eligible,
|
|
||||||
bytes_processed: 0,
|
|
||||||
message: None,
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
|
|
||||||
Ok(guard) => guard,
|
|
||||||
Err(ThreadStoreError::Conflict { message }) => {
|
|
||||||
report.outcomes.push(skipped_busy_outcome(
|
|
||||||
thread_id, path, message, /*bytes_processed*/ 0,
|
|
||||||
));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
report.outcomes.push(migration_outcome(
|
|
||||||
thread_id,
|
|
||||||
path,
|
|
||||||
Err(error),
|
|
||||||
/*bytes_processed*/ 0,
|
|
||||||
));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let bytes_before = limiter.bytes_processed;
|
|
||||||
let result = match self
|
|
||||||
.migrate_one_rollout(thread_id, &path, &journal_path, &mut limiter)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
|
|
||||||
Err(error) => {
|
|
||||||
if let Err(cleanup_error) = self
|
|
||||||
.cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Err(migration_error(format!(
|
|
||||||
"{error}; failed to clean up unpublished migration: {cleanup_error}"
|
|
||||||
)))
|
|
||||||
} else {
|
|
||||||
Err(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
drop(writer_guard);
|
|
||||||
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
|
|
||||||
let outcome = match result {
|
|
||||||
Err(ThreadStoreError::Conflict { message }) => {
|
|
||||||
skipped_busy_outcome(thread_id, path, message, bytes_processed)
|
|
||||||
}
|
|
||||||
result => migration_outcome(thread_id, path, result, bytes_processed),
|
|
||||||
};
|
|
||||||
report.outcomes.push(outcome);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(report)
|
Ok(report)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn migrate_rollout_path(
|
||||||
|
&self,
|
||||||
|
mut path: PathBuf,
|
||||||
|
options: &RolloutMigrationOptions,
|
||||||
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
|
) -> ThreadStoreResult<Option<RolloutMigrationOutcome>> {
|
||||||
|
let metadata = match codex_rollout::read_session_meta_line(&path).await {
|
||||||
|
Ok(metadata) => metadata,
|
||||||
|
Err(error) => {
|
||||||
|
let thread_id = thread_id_from_rollout_filename(&path);
|
||||||
|
if !matches_selection(&options.thread_ids, thread_id) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let empty = tokio::fs::metadata(&path)
|
||||||
|
.await
|
||||||
|
.is_ok_and(|metadata| metadata.len() == 0);
|
||||||
|
return Ok(Some(RolloutMigrationOutcome {
|
||||||
|
thread_id,
|
||||||
|
rollout_path: path,
|
||||||
|
status: if empty {
|
||||||
|
RolloutMigrationStatus::SkippedEmpty
|
||||||
|
} else {
|
||||||
|
RolloutMigrationStatus::Failed
|
||||||
|
},
|
||||||
|
bytes_processed: 0,
|
||||||
|
message: (!empty).then(|| error.to_string()),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let thread_id = metadata.meta.id;
|
||||||
|
if !matches_selection(&options.thread_ids, Some(thread_id)) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let is_memory_consolidation = matches!(
|
||||||
|
metadata.meta.source,
|
||||||
|
SessionSource::Internal(InternalSessionSource::MemoryConsolidation)
|
||||||
|
| SessionSource::SubAgent(SubAgentSource::MemoryConsolidation)
|
||||||
|
) || matches!(
|
||||||
|
metadata.meta.thread_source,
|
||||||
|
Some(ThreadSource::MemoryConsolidation)
|
||||||
|
);
|
||||||
|
let kind = if !is_memory_consolidation
|
||||||
|
&& (matches!(metadata.meta.source, SessionSource::SubAgent(_))
|
||||||
|
|| matches!(metadata.meta.thread_source, Some(ThreadSource::Subagent)))
|
||||||
|
{
|
||||||
|
RolloutMigrationKind::Subagent
|
||||||
|
} else {
|
||||||
|
RolloutMigrationKind::Ordinary
|
||||||
|
};
|
||||||
|
|
||||||
|
let journal_path = migration_journal_path(&self.config.codex_home, thread_id);
|
||||||
|
let pending_published_migration = metadata.meta.history_mode
|
||||||
|
== ThreadHistoryMode::Paginated
|
||||||
|
&& options.mode == RolloutMigrationMode::Apply
|
||||||
|
&& tokio::fs::try_exists(&journal_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
if metadata.meta.history_mode == ThreadHistoryMode::Paginated {
|
||||||
|
let bytes_before = limiter.bytes_processed;
|
||||||
|
let result = if pending_published_migration {
|
||||||
|
match self
|
||||||
|
.recover_published_migration(thread_id, &path, &journal_path, limiter)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(recovered_path) => {
|
||||||
|
path = recovered_path;
|
||||||
|
Ok(RolloutMigrationStatus::Migrated)
|
||||||
|
}
|
||||||
|
Err(ThreadStoreError::Conflict { message }) => {
|
||||||
|
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
|
||||||
|
return Ok(Some(skipped_busy_outcome(
|
||||||
|
thread_id,
|
||||||
|
path,
|
||||||
|
message,
|
||||||
|
bytes_processed,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Ok(RolloutMigrationStatus::AlreadyPaginated)
|
||||||
|
};
|
||||||
|
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
|
||||||
|
return Ok(Some(migration_outcome(
|
||||||
|
thread_id,
|
||||||
|
path,
|
||||||
|
result,
|
||||||
|
bytes_processed,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.mode == RolloutMigrationMode::DryRun {
|
||||||
|
return Ok(Some(migration_outcome(
|
||||||
|
thread_id,
|
||||||
|
path,
|
||||||
|
Ok(RolloutMigrationStatus::Eligible),
|
||||||
|
/*bytes_processed*/ 0,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let _writer_guard = match self.writer_lock_coordinator.acquire(thread_id) {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(ThreadStoreError::Conflict { message }) => {
|
||||||
|
return Ok(Some(skipped_busy_outcome(
|
||||||
|
thread_id, path, message, /*bytes_processed*/ 0,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
return Ok(Some(migration_outcome(
|
||||||
|
thread_id,
|
||||||
|
path,
|
||||||
|
Err(error),
|
||||||
|
/*bytes_processed*/ 0,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let bytes_before = limiter.bytes_processed;
|
||||||
|
let result = match self
|
||||||
|
.migrate_one_rollout(thread_id, &path, &journal_path, kind, limiter)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(()) => Ok(RolloutMigrationStatus::Migrated),
|
||||||
|
Err(error) => {
|
||||||
|
if let Err(cleanup_error) = self
|
||||||
|
.cleanup_failed_unpublished_migration(thread_id, &path, &journal_path)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Err(migration_error(format!(
|
||||||
|
"{error}; failed to clean up unpublished migration: {cleanup_error}"
|
||||||
|
)))
|
||||||
|
} else {
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let bytes_processed = limiter.bytes_processed.saturating_sub(bytes_before);
|
||||||
|
Ok(Some(match result {
|
||||||
|
Err(ThreadStoreError::Conflict { message }) => {
|
||||||
|
skipped_busy_outcome(thread_id, path, message, bytes_processed)
|
||||||
|
}
|
||||||
|
result => migration_outcome(thread_id, path, result, bytes_processed),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn migrate_one_rollout(
|
async fn migrate_one_rollout(
|
||||||
&self,
|
&self,
|
||||||
thread_id: ThreadId,
|
thread_id: ThreadId,
|
||||||
rollout_path: &Path,
|
rollout_path: &Path,
|
||||||
journal_path: &Path,
|
journal_path: &Path,
|
||||||
|
kind: RolloutMigrationKind,
|
||||||
limiter: &mut RolloutMigrationRateLimiter,
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
) -> ThreadStoreResult<()> {
|
) -> ThreadStoreResult<()> {
|
||||||
if let Some(state_db) = &self.state_db
|
if let Some(state_db) = &self.state_db
|
||||||
@@ -403,8 +459,31 @@ impl LocalThreadStore {
|
|||||||
source_permissions: &source_permissions,
|
source_permissions: &source_permissions,
|
||||||
canonical_session_meta: &canonical_session_meta,
|
canonical_session_meta: &canonical_session_meta,
|
||||||
};
|
};
|
||||||
let (expected_length, expected_ordinal) =
|
let bounded_subagent_context = if kind == RolloutMigrationKind::Subagent {
|
||||||
Self::write_canonical_rollout(&canonicalization_source, limiter).await?;
|
let RolloutItem::SessionMeta(session_meta) = &canonical_session_meta.item else {
|
||||||
|
return Err(migration_error("canonical session metadata is missing"));
|
||||||
|
};
|
||||||
|
let context =
|
||||||
|
subagent::select_bounded_context(source_path.to_path_buf(), session_meta.clone())
|
||||||
|
.await?;
|
||||||
|
limiter.account(source_metadata.len()).await;
|
||||||
|
context
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let (_, expected_ordinal) = if let Some(items) = bounded_subagent_context {
|
||||||
|
Self::write_bounded_subagent_rollout(&canonicalization_source, items, limiter).await?
|
||||||
|
} else {
|
||||||
|
Self::write_rollout_with_rollback_plan(&canonicalization_source, limiter).await?
|
||||||
|
};
|
||||||
|
|
||||||
|
if kind == RolloutMigrationKind::Subagent {
|
||||||
|
rewrite_subagent_history_boundary(&staged_path, expected_ordinal).await?;
|
||||||
|
}
|
||||||
|
let expected_length = tokio::fs::metadata(&staged_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?
|
||||||
|
.len();
|
||||||
|
|
||||||
// SQLite projection only starts after every staged-file mutation is durable.
|
// SQLite projection only starts after every staged-file mutation is durable.
|
||||||
let modified_at = source_modified;
|
let modified_at = source_modified;
|
||||||
@@ -473,10 +552,97 @@ impl LocalThreadStore {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write_canonical_rollout(
|
async fn build_rollback_plan(
|
||||||
|
source_path: &Path,
|
||||||
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
|
) -> ThreadStoreResult<RollbackPlan> {
|
||||||
|
let source_file = File::open(source_path).await.map_err(migration_error)?;
|
||||||
|
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let mut planner = RollbackPlanner::new();
|
||||||
|
while let Some(record) = read_rollout_record(&mut source, &mut bytes).await? {
|
||||||
|
limiter.account(record.byte_count).await;
|
||||||
|
if let Some(line) = record.line {
|
||||||
|
planner.observe(&line)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(planner.finish())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_rollout_with_rollback_plan(
|
||||||
input: &CanonicalizationSource<'_>,
|
input: &CanonicalizationSource<'_>,
|
||||||
limiter: &mut RolloutMigrationRateLimiter,
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
) -> ThreadStoreResult<(u64, u64)> {
|
) -> ThreadStoreResult<(u64, u64)> {
|
||||||
|
let attempt = Self::write_canonical_rollout(input, /*plan*/ None, limiter).await?;
|
||||||
|
match attempt {
|
||||||
|
CanonicalizationAttempt::Complete {
|
||||||
|
expected_length,
|
||||||
|
expected_ordinal,
|
||||||
|
} => Ok((expected_length, expected_ordinal)),
|
||||||
|
CanonicalizationAttempt::NeedsRollbackPlan => {
|
||||||
|
remove_file_if_present(input.staged_path).await?;
|
||||||
|
let plan = Self::build_rollback_plan(input.source_path, limiter).await?;
|
||||||
|
let CanonicalizationAttempt::Complete {
|
||||||
|
expected_length,
|
||||||
|
expected_ordinal,
|
||||||
|
} = Self::write_canonical_rollout(input, Some(&plan), limiter).await?
|
||||||
|
else {
|
||||||
|
return Err(migration_error(
|
||||||
|
"planned rollout still contains a rollback marker",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
Ok((expected_length, expected_ordinal))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_bounded_subagent_rollout(
|
||||||
|
input: &CanonicalizationSource<'_>,
|
||||||
|
items: Vec<RolloutItem>,
|
||||||
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
|
) -> ThreadStoreResult<(u64, u64)> {
|
||||||
|
let staged_file = tokio::fs::OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.truncate(true)
|
||||||
|
.write(true)
|
||||||
|
.open(input.staged_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
staged_file
|
||||||
|
.set_permissions(input.source_permissions.clone())
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
|
||||||
|
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
|
||||||
|
let written = canonicalizer
|
||||||
|
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
|
||||||
|
.await?;
|
||||||
|
limiter.account(written).await;
|
||||||
|
for item in items {
|
||||||
|
let line = RolloutLine {
|
||||||
|
timestamp: input.canonical_session_meta.timestamp.clone(),
|
||||||
|
ordinal: None,
|
||||||
|
item,
|
||||||
|
};
|
||||||
|
let written = canonicalizer.process_line(line, &mut staged).await?;
|
||||||
|
limiter.account(written).await;
|
||||||
|
}
|
||||||
|
let written = canonicalizer
|
||||||
|
.finish(&mut staged, &input.canonical_session_meta.timestamp)
|
||||||
|
.await?;
|
||||||
|
limiter.account(written).await;
|
||||||
|
staged.flush().await.map_err(migration_error)?;
|
||||||
|
Ok((
|
||||||
|
canonicalizer.output_byte_offset(),
|
||||||
|
canonicalizer.next_ordinal(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_canonical_rollout(
|
||||||
|
input: &CanonicalizationSource<'_>,
|
||||||
|
plan: Option<&RollbackPlan>,
|
||||||
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
|
) -> ThreadStoreResult<CanonicalizationAttempt> {
|
||||||
let source_file = File::open(input.source_path)
|
let source_file = File::open(input.source_path)
|
||||||
.await
|
.await
|
||||||
.map_err(migration_error)?;
|
.map_err(migration_error)?;
|
||||||
@@ -494,6 +660,7 @@ impl LocalThreadStore {
|
|||||||
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
|
let mut source = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, source_file);
|
||||||
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
|
let mut staged = BufWriter::with_capacity(PROJECTION_BATCH_BYTES as usize, staged_file);
|
||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
|
let mut parsed_record_index = 0_usize;
|
||||||
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
|
let mut canonicalizer = LegacyRolloutCanonicalizer::new(input.thread_id);
|
||||||
let written = canonicalizer
|
let written = canonicalizer
|
||||||
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
|
.write_head_session_meta(input.canonical_session_meta.clone(), &mut staged)
|
||||||
@@ -506,24 +673,45 @@ impl LocalThreadStore {
|
|||||||
let Some(line) = record.line else {
|
let Some(line) = record.line else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
if matches!(
|
if plan.is_none()
|
||||||
&line.item,
|
&& matches!(
|
||||||
RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
|
&line.item,
|
||||||
) {
|
RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ThreadRolledBack(_))
|
||||||
return Err(migration_error("legacy rollout contains a rollback marker"));
|
)
|
||||||
|
{
|
||||||
|
return Ok(CanonicalizationAttempt::NeedsRollbackPlan);
|
||||||
}
|
}
|
||||||
|
let line = if let Some(plan) = plan {
|
||||||
|
let planned = plan.apply(parsed_record_index, line)?;
|
||||||
|
parsed_record_index = parsed_record_index
|
||||||
|
.checked_add(1)
|
||||||
|
.ok_or_else(|| migration_error("legacy rollout record index overflow"))?;
|
||||||
|
let Some(line) = planned else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
line
|
||||||
|
} else {
|
||||||
|
line
|
||||||
|
};
|
||||||
last_timestamp = line.timestamp.clone();
|
last_timestamp = line.timestamp.clone();
|
||||||
let written = canonicalizer.process_line(line, &mut staged).await?;
|
let written = canonicalizer.process_line(line, &mut staged).await?;
|
||||||
limiter.account(written).await;
|
limiter.account(written).await;
|
||||||
}
|
}
|
||||||
|
if let Some(plan) = plan
|
||||||
|
&& parsed_record_index != plan.record_count()
|
||||||
|
{
|
||||||
|
return Err(migration_error(
|
||||||
|
"rollback plan source length changed during replay",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let written = canonicalizer.finish(&mut staged, &last_timestamp).await?;
|
let written = canonicalizer.finish(&mut staged, &last_timestamp).await?;
|
||||||
limiter.account(written).await;
|
limiter.account(written).await;
|
||||||
staged.flush().await.map_err(migration_error)?;
|
staged.flush().await.map_err(migration_error)?;
|
||||||
Ok((
|
Ok(CanonicalizationAttempt::Complete {
|
||||||
canonicalizer.output_byte_offset(),
|
expected_length: canonicalizer.output_byte_offset(),
|
||||||
canonicalizer.next_ordinal(),
|
expected_ordinal: canonicalizer.next_ordinal(),
|
||||||
))
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn recover_published_migration(
|
async fn recover_published_migration(
|
||||||
@@ -532,7 +720,7 @@ impl LocalThreadStore {
|
|||||||
rollout_path: &Path,
|
rollout_path: &Path,
|
||||||
journal_path: &Path,
|
journal_path: &Path,
|
||||||
limiter: &mut RolloutMigrationRateLimiter,
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
) -> ThreadStoreResult<()> {
|
) -> ThreadStoreResult<PathBuf> {
|
||||||
let _writer_guard = self.writer_lock_coordinator.acquire(thread_id)?;
|
let _writer_guard = self.writer_lock_coordinator.acquire(thread_id)?;
|
||||||
let decompressed_path = rollout_path_is_compressed(rollout_path)
|
let decompressed_path = rollout_path_is_compressed(rollout_path)
|
||||||
.then(|| decompressed_staged_rollout_path(rollout_path))
|
.then(|| decompressed_staged_rollout_path(rollout_path))
|
||||||
@@ -572,7 +760,8 @@ impl LocalThreadStore {
|
|||||||
remove_file_if_present(decompressed_path).await?;
|
remove_file_if_present(decompressed_path).await?;
|
||||||
}
|
}
|
||||||
self.finish_published_migration(thread_id, journal_path)
|
self.finish_published_migration(thread_id, journal_path)
|
||||||
.await
|
.await?;
|
||||||
|
Ok(rollout_path.to_path_buf())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cleanup_failed_unpublished_migration(
|
async fn cleanup_failed_unpublished_migration(
|
||||||
@@ -589,7 +778,9 @@ impl LocalThreadStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
thread_history::delete_thread(self, thread_id).await?;
|
thread_history::delete_thread(self, thread_id).await?;
|
||||||
remove_file_if_present(&staged_rollout_path(rollout_path)?).await?;
|
let staged_path = staged_rollout_path(rollout_path)?;
|
||||||
|
remove_file_if_present(&staged_path).await?;
|
||||||
|
remove_file_if_present(&rewritten_staged_rollout_path(&staged_path)?).await?;
|
||||||
remove_file_if_present(&compressed_staged_rollout_path(rollout_path)?).await?;
|
remove_file_if_present(&compressed_staged_rollout_path(rollout_path)?).await?;
|
||||||
remove_file_if_present(&decompressed_staged_rollout_path(rollout_path)?).await?;
|
remove_file_if_present(&decompressed_staged_rollout_path(rollout_path)?).await?;
|
||||||
remove_file_if_present(journal_path).await?;
|
remove_file_if_present(journal_path).await?;
|
||||||
@@ -623,6 +814,11 @@ impl LocalThreadStore {
|
|||||||
rollout_path: &Path,
|
rollout_path: &Path,
|
||||||
limiter: &mut RolloutMigrationRateLimiter,
|
limiter: &mut RolloutMigrationRateLimiter,
|
||||||
) -> ThreadStoreResult<()> {
|
) -> ThreadStoreResult<()> {
|
||||||
|
let subagent_history_start_ordinal = codex_rollout::read_session_meta_line(rollout_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?
|
||||||
|
.meta
|
||||||
|
.subagent_history_start_ordinal;
|
||||||
let file = File::open(rollout_path).await.map_err(migration_error)?;
|
let file = File::open(rollout_path).await.map_err(migration_error)?;
|
||||||
let mut reader = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, file);
|
let mut reader = BufReader::with_capacity(PROJECTION_BATCH_BYTES as usize, file);
|
||||||
let mut line_bytes = Vec::new();
|
let mut line_bytes = Vec::new();
|
||||||
@@ -650,7 +846,11 @@ impl LocalThreadStore {
|
|||||||
start_byte_offset: offset,
|
start_byte_offset: offset,
|
||||||
end_byte_offset: next_offset,
|
end_byte_offset: next_offset,
|
||||||
fallback_created_at_ms: Some(fallback_created_at_ms),
|
fallback_created_at_ms: Some(fallback_created_at_ms),
|
||||||
changes: project_rollout_line(&line),
|
changes: if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) {
|
||||||
|
Default::default()
|
||||||
|
} else {
|
||||||
|
project_rollout_line(&line)
|
||||||
|
},
|
||||||
}));
|
}));
|
||||||
offset = next_offset;
|
offset = next_offset;
|
||||||
|
|
||||||
@@ -688,7 +888,7 @@ async fn read_rollout_record(
|
|||||||
bytes: &mut Vec<u8>,
|
bytes: &mut Vec<u8>,
|
||||||
) -> ThreadStoreResult<Option<RolloutRecord>> {
|
) -> ThreadStoreResult<Option<RolloutRecord>> {
|
||||||
bytes.clear();
|
bytes.clear();
|
||||||
let byte_count = reader
|
let mut byte_count = reader
|
||||||
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
|
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
|
||||||
.read_until(b'\n', bytes)
|
.read_until(b'\n', bytes)
|
||||||
.await
|
.await
|
||||||
@@ -697,11 +897,30 @@ async fn read_rollout_record(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
if byte_count > MAX_ROLLOUT_LINE_BYTES {
|
if byte_count > MAX_ROLLOUT_LINE_BYTES {
|
||||||
return Err(migration_error(
|
// Some historical tool outputs are too large to migrate safely in memory. Discard the
|
||||||
"rollout contains an oversized JSONL record",
|
// whole record, including any unread suffix.
|
||||||
));
|
while bytes.last() != Some(&b'\n') {
|
||||||
|
bytes.clear();
|
||||||
|
let chunk_bytes = reader
|
||||||
|
.take((MAX_ROLLOUT_LINE_BYTES + 1) as u64)
|
||||||
|
.read_until(b'\n', bytes)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
if chunk_bytes == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
byte_count = byte_count
|
||||||
|
.checked_add(chunk_bytes)
|
||||||
|
.ok_or_else(|| migration_error("rollout record byte count overflow"))?;
|
||||||
|
}
|
||||||
|
return Ok(Some(RolloutRecord {
|
||||||
|
line: None,
|
||||||
|
byte_count: byte_count as u64,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
let line = line_parser::parse_legacy_rollout_line(bytes).map_err(migration_error)?;
|
// Legacy records do not have ordinals, so malformed complete records cannot be repaired.
|
||||||
|
// Skip them and let the next newline-delimited record resynchronize the stream.
|
||||||
|
let line = line_parser::parse_legacy_rollout_line(bytes).unwrap_or(None);
|
||||||
Ok(Some(RolloutRecord {
|
Ok(Some(RolloutRecord {
|
||||||
line,
|
line,
|
||||||
byte_count: byte_count as u64,
|
byte_count: byte_count as u64,
|
||||||
@@ -734,7 +953,15 @@ async fn find_rollout_paths(root: &Path) -> ThreadStoreResult<Vec<PathBuf>> {
|
|||||||
if name.starts_with("rollout-")
|
if name.starts_with("rollout-")
|
||||||
&& (name.ends_with(".jsonl") || name.ends_with(".jsonl.zst"))
|
&& (name.ends_with(".jsonl") || name.ends_with(".jsonl.zst"))
|
||||||
{
|
{
|
||||||
paths.push(entry.path());
|
let path = entry.path();
|
||||||
|
if name.ends_with(".jsonl.zst")
|
||||||
|
&& tokio::fs::try_exists(codex_rollout::plain_rollout_path(&path))
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
paths.push(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -747,6 +974,12 @@ fn matches_selection(selected: &[ThreadId], actual: Option<ThreadId>) -> bool {
|
|||||||
selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))
|
selected.is_empty() || actual.is_some_and(|thread_id| selected.contains(&thread_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rollout_path_is_compressed(path: &Path) -> bool {
|
||||||
|
path.file_name()
|
||||||
|
.and_then(|name| name.to_str())
|
||||||
|
.is_some_and(|name| name.ends_with(".jsonl.zst"))
|
||||||
|
}
|
||||||
|
|
||||||
fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
|
fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
|
||||||
let name = path.file_name()?.to_str()?;
|
let name = path.file_name()?.to_str()?;
|
||||||
let stem = name
|
let stem = name
|
||||||
@@ -756,12 +989,6 @@ fn thread_id_from_rollout_filename(path: &Path) -> Option<ThreadId> {
|
|||||||
ThreadId::from_string(stem.get(start..)?).ok()
|
ThreadId::from_string(stem.get(start..)?).ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rollout_path_is_compressed(path: &Path) -> bool {
|
|
||||||
path.file_name()
|
|
||||||
.and_then(|name| name.to_str())
|
|
||||||
.is_some_and(|name| name.ends_with(".jsonl.zst"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn migration_outcome(
|
fn migration_outcome(
|
||||||
thread_id: ThreadId,
|
thread_id: ThreadId,
|
||||||
rollout_path: PathBuf,
|
rollout_path: PathBuf,
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
//! Replays normalized legacy rollout records into canonical paginated JSONL.
|
//! Replays normalized legacy rollout records into canonical paginated JSONL.
|
||||||
//!
|
//!
|
||||||
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
|
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
|
||||||
//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at
|
//! events into modern turn items. Rollback planning happens before this writer sees a record, so
|
||||||
//! ordinal zero, and emits the surviving history selected by the caller.
|
//! this module only assigns stable ordinals, keeps `SessionMeta` at ordinal zero, and emits the
|
||||||
|
//! already-selected surviving history.
|
||||||
//!
|
//!
|
||||||
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
|
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
|
||||||
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
|
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
|
||||||
@@ -52,7 +53,6 @@ pub(super) struct LegacyRolloutCanonicalizer {
|
|||||||
active_turn: Option<ActiveTurn>,
|
active_turn: Option<ActiveTurn>,
|
||||||
known_turn_ids: HashSet<String>,
|
known_turn_ids: HashSet<String>,
|
||||||
reasoning: Option<ReasoningItem>,
|
reasoning: Option<ReasoningItem>,
|
||||||
saw_source_session_meta: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LegacyRolloutCanonicalizer {
|
impl LegacyRolloutCanonicalizer {
|
||||||
@@ -67,18 +67,17 @@ impl LegacyRolloutCanonicalizer {
|
|||||||
active_turn: None,
|
active_turn: None,
|
||||||
known_turn_ids: HashSet::new(),
|
known_turn_ids: HashSet::new(),
|
||||||
reasoning: None,
|
reasoning: None,
|
||||||
saw_source_session_meta: false,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn output_byte_offset(&self) -> u64 {
|
|
||||||
self.output_byte_offset
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn next_ordinal(&self) -> u64 {
|
pub(super) fn next_ordinal(&self) -> u64 {
|
||||||
self.next_ordinal
|
self.next_ordinal
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn output_byte_offset(&self) -> u64 {
|
||||||
|
self.output_byte_offset
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn write_head_session_meta(
|
pub(super) async fn write_head_session_meta(
|
||||||
&mut self,
|
&mut self,
|
||||||
line: RolloutLine,
|
line: RolloutLine,
|
||||||
@@ -114,14 +113,7 @@ impl LegacyRolloutCanonicalizer {
|
|||||||
let timestamp = line.timestamp;
|
let timestamp = line.timestamp;
|
||||||
let bytes_before = self.bytes_written;
|
let bytes_before = self.bytes_written;
|
||||||
match line.item {
|
match line.item {
|
||||||
RolloutItem::SessionMeta(metadata) => {
|
RolloutItem::SessionMeta(_) => return Ok(0),
|
||||||
if !self.saw_source_session_meta {
|
|
||||||
self.saw_source_session_meta = true;
|
|
||||||
return Ok(0);
|
|
||||||
}
|
|
||||||
self.write_item(writer, ×tamp, RolloutItem::SessionMeta(metadata))
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
RolloutItem::ResponseItem(ResponseItem::Other) => {
|
RolloutItem::ResponseItem(ResponseItem::Other) => {
|
||||||
return Err(migration_error(
|
return Err(migration_error(
|
||||||
"legacy rollout contains an unsupported response item",
|
"legacy rollout contains an unsupported response item",
|
||||||
|
|||||||
@@ -26,7 +26,11 @@ pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLi
|
|||||||
// Deserializing through Value is intentional. Some historical numeric
|
// Deserializing through Value is intentional. Some historical numeric
|
||||||
// payloads fail serde_json's streaming enum path but deserialize correctly
|
// payloads fail serde_json's streaming enum path but deserialize correctly
|
||||||
// once their tagged object shape has been materialized.
|
// once their tagged object shape has been materialized.
|
||||||
let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
|
let value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
|
||||||
|
parse_legacy_rollout_value(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn parse_legacy_rollout_value(mut value: Value) -> Result<Option<RolloutLine>, String> {
|
||||||
if should_skip_retired_record(&value) {
|
if should_skip_retired_record(&value) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,13 @@ use std::os::unix::fs::OpenOptionsExt;
|
|||||||
use std::os::unix::fs::PermissionsExt;
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
|
||||||
use codex_protocol::ThreadId;
|
use codex_protocol::ThreadId;
|
||||||
|
use codex_protocol::protocol::RolloutItem;
|
||||||
|
use codex_protocol::protocol::RolloutLine;
|
||||||
|
use tokio::fs::File;
|
||||||
|
use tokio::io::AsyncBufReadExt;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use tokio::io::BufReader;
|
||||||
|
use tokio::io::BufWriter;
|
||||||
|
|
||||||
use super::migration_error;
|
use super::migration_error;
|
||||||
use crate::ThreadStoreResult;
|
use crate::ThreadStoreResult;
|
||||||
@@ -139,6 +146,74 @@ pub(super) async fn compress_rollout_to_path(
|
|||||||
.map_err(migration_error)
|
.map_err(migration_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn rewritten_staged_rollout_path(staged_path: &Path) -> ThreadStoreResult<PathBuf> {
|
||||||
|
let filename = staged_path
|
||||||
|
.file_name()
|
||||||
|
.and_then(|filename| filename.to_str())
|
||||||
|
.ok_or_else(|| migration_error("staged rollout path has no valid filename"))?;
|
||||||
|
Ok(staged_path.with_file_name(format!("{filename}.head.tmp")))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn rewrite_subagent_history_boundary(
|
||||||
|
staged_path: &Path,
|
||||||
|
boundary: u64,
|
||||||
|
) -> ThreadStoreResult<()> {
|
||||||
|
let rewritten_path = rewritten_staged_rollout_path(staged_path)?;
|
||||||
|
let permissions = tokio::fs::metadata(staged_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?
|
||||||
|
.permissions();
|
||||||
|
let source = File::open(staged_path).await.map_err(migration_error)?;
|
||||||
|
let mut source = BufReader::new(source);
|
||||||
|
let mut head_bytes = Vec::new();
|
||||||
|
source
|
||||||
|
.read_until(b'\n', &mut head_bytes)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
let mut head = serde_json::from_slice::<RolloutLine>(&head_bytes).map_err(migration_error)?;
|
||||||
|
let RolloutItem::SessionMeta(session_meta) = &mut head.item else {
|
||||||
|
return Err(migration_error(
|
||||||
|
"staged rollout head is not session metadata",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
session_meta.meta.subagent_history_start_ordinal = Some(boundary);
|
||||||
|
|
||||||
|
let rewritten = tokio::fs::OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.truncate(true)
|
||||||
|
.write(true)
|
||||||
|
.open(&rewritten_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
rewritten
|
||||||
|
.set_permissions(permissions)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
let mut rewritten = BufWriter::new(rewritten);
|
||||||
|
rewritten
|
||||||
|
.write_all(
|
||||||
|
serde_json::to_string(&head)
|
||||||
|
.map_err(migration_error)?
|
||||||
|
.as_bytes(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
rewritten.write_all(b"\n").await.map_err(migration_error)?;
|
||||||
|
tokio::io::copy(&mut source, &mut rewritten)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
rewritten.flush().await.map_err(migration_error)?;
|
||||||
|
rewritten
|
||||||
|
.get_ref()
|
||||||
|
.sync_all()
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?;
|
||||||
|
drop(rewritten);
|
||||||
|
tokio::fs::rename(rewritten_path, staged_path)
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
|
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
|
||||||
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
|
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
|
||||||
tokio::fs::remove_file(path)
|
tokio::fs::remove_file(path)
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
//! Defines which persisted history entries count as rollbackable user turns.
|
||||||
|
//!
|
||||||
|
//! Legacy `ThreadRolledBack { num_turns }` does not mean “drop the last N JSONL records.” A turn
|
||||||
|
//! can include contextual user/developer messages, paired `ResponseItem` + `UserMessage` records,
|
||||||
|
//! inter-agent communication, or compaction replacement history.
|
||||||
|
//!
|
||||||
|
//! This module mirrors the legacy cold-resume rollback boundary rules for persisted rollout shapes.
|
||||||
|
//! It stays local to migration because core's live predicate also knows about dynamically
|
||||||
|
//! registered contextual fragments that thread-store should not depend on.
|
||||||
|
|
||||||
|
use codex_protocol::items::parse_hook_prompt_fragment;
|
||||||
|
use codex_protocol::models::ContentItem;
|
||||||
|
use codex_protocol::models::ResponseItem;
|
||||||
|
use codex_protocol::protocol::InterAgentCommunication;
|
||||||
|
|
||||||
|
/// Match the rollback boundaries used by legacy history reconstruction without
|
||||||
|
/// treating every persisted lifecycle as a user turn. This intentionally stays
|
||||||
|
/// local to the frozen migration adapter: the full core predicate also knows
|
||||||
|
/// about dynamically registered contextual fragments that thread-store cannot
|
||||||
|
/// depend on.
|
||||||
|
pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool {
|
||||||
|
if matches!(response, ResponseItem::AgentMessage { .. }) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let ResponseItem::Message { role, content, .. } = response else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
(role == "user" && !is_known_contextual_user_message_content(content))
|
||||||
|
|| (role == "assistant" && InterAgentCommunication::is_message_content(content))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool {
|
||||||
|
let ResponseItem::Message { role, content, .. } = response else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
(role == "user" && is_known_contextual_user_message_content(content))
|
||||||
|
|| (role == "developer" && is_known_contextual_developer_message_content(content))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply the same response-history cut used by legacy cold resume to a persisted
|
||||||
|
/// compaction checkpoint. The migration adapter uses a frozen contextual-fragment
|
||||||
|
/// matcher because thread-store cannot depend on core's runtime fragment registry.
|
||||||
|
pub(super) fn drop_last_n_user_turns(history: &mut Vec<ResponseItem>, num_turns: u32) {
|
||||||
|
if num_turns == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let user_positions = history
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| counts_as_boundary(item).then_some(index))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let Some(&first_turn_index) = user_positions.first() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
|
||||||
|
let mut cut_index = if count >= user_positions.len() {
|
||||||
|
first_turn_index
|
||||||
|
} else {
|
||||||
|
user_positions[user_positions.len() - count]
|
||||||
|
};
|
||||||
|
while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1]) {
|
||||||
|
cut_index -= 1;
|
||||||
|
}
|
||||||
|
history.truncate(cut_index);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_known_contextual_user_message_content(content: &[ContentItem]) -> bool {
|
||||||
|
content.iter().any(|item| {
|
||||||
|
let ContentItem::InputText { text } = item else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
is_known_contextual_user_text(text)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_known_contextual_developer_message_content(content: &[ContentItem]) -> bool {
|
||||||
|
content.iter().any(|item| {
|
||||||
|
let ContentItem::InputText { text } = item else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let text = text.trim_start();
|
||||||
|
[
|
||||||
|
"<permissions instructions>",
|
||||||
|
"<model_switch>",
|
||||||
|
"<apps_instructions>",
|
||||||
|
"<collaboration_mode>",
|
||||||
|
"<multi_agent_mode>",
|
||||||
|
"<environments_instructions>",
|
||||||
|
"<git_attribution>",
|
||||||
|
"<plugins_instructions>",
|
||||||
|
"<realtime_conversation>",
|
||||||
|
"<skills_instructions>",
|
||||||
|
"<tools>",
|
||||||
|
"<personality_spec>",
|
||||||
|
"<token_budget>",
|
||||||
|
"<context_window>",
|
||||||
|
"<context_window_guidance>",
|
||||||
|
"<rollout_budget>",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| {
|
||||||
|
text.get(..prefix.len())
|
||||||
|
.is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep this frozen alongside the legacy migration adapter. Core's live predicate also knows
|
||||||
|
// about dynamically registered fragments, but legacy rollouts only need these persisted shapes.
|
||||||
|
fn is_known_contextual_user_text(text: &str) -> bool {
|
||||||
|
let text = text.trim();
|
||||||
|
parse_hook_prompt_fragment(text).is_some()
|
||||||
|
|| [
|
||||||
|
("# AGENTS.md instructions", "</INSTRUCTIONS>"),
|
||||||
|
("<environment_context>", "</environment_context>"),
|
||||||
|
("<skill>", "</skill>"),
|
||||||
|
("<user_shell_command>", "</user_shell_command>"),
|
||||||
|
("<turn_aborted>", "</turn_aborted>"),
|
||||||
|
("<subagent_notification>", "</subagent_notification>"),
|
||||||
|
("<recommended_plugins>", "</recommended_plugins>"),
|
||||||
|
("<goal_context>", "</goal_context>"),
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|(start, end)| text.starts_with(start) && text.ends_with(end))
|
||||||
|
|| (text.starts_with("<external_")
|
||||||
|
&& text
|
||||||
|
.split_once('>')
|
||||||
|
.and_then(|(start, _)| start.strip_prefix("<external_"))
|
||||||
|
.is_some_and(|key| text.ends_with(&format!("</external_{key}>"))))
|
||||||
|
|| (text.starts_with("<codex_internal_context")
|
||||||
|
&& text.ends_with("</codex_internal_context>"))
|
||||||
|
|| text.starts_with(
|
||||||
|
"Warning: The maximum number of unified exec processes you can keep open is",
|
||||||
|
)
|
||||||
|
|| (text.starts_with("Warning: apply_patch was requested via ")
|
||||||
|
&& text.ends_with("Use the apply_patch tool instead of exec_command."))
|
||||||
|
|| text.starts_with(
|
||||||
|
"Warning: Your account was flagged for potentially high-risk cyber activity",
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
//! Decides which legacy records remain visible after historical rollback.
|
||||||
|
//!
|
||||||
|
//! Legacy rollback removes logical instruction turns, not a physical suffix of the rollout file.
|
||||||
|
//! Most records happen to be ordered that way, but late completion events can target an older
|
||||||
|
//! surviving turn after a newer turn has started. This planner keeps compact per-record ownership
|
||||||
|
//! metadata for SQLite visibility, then combines it with `rollback_replay`'s cold-resume answer
|
||||||
|
//! before the writer makes its second streaming pass.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use codex_protocol::models::ContentItem;
|
||||||
|
use codex_protocol::models::ResponseItem;
|
||||||
|
use codex_protocol::protocol::CompactedItem;
|
||||||
|
use codex_protocol::protocol::EventMsg;
|
||||||
|
use codex_protocol::protocol::RolloutItem;
|
||||||
|
use codex_protocol::protocol::RolloutLine;
|
||||||
|
use codex_protocol::protocol::UserMessageEvent;
|
||||||
|
|
||||||
|
use super::migration_error;
|
||||||
|
use super::rollback;
|
||||||
|
use super::rollback_replay::ModelReplayPlanner;
|
||||||
|
use crate::ThreadStoreResult;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct CompactionFrame {
|
||||||
|
record_index: usize,
|
||||||
|
boundary_depth: usize,
|
||||||
|
owner: Option<usize>,
|
||||||
|
item: CompactedItem,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct PendingUserResponse {
|
||||||
|
boundary: usize,
|
||||||
|
content: Vec<ContentItem>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compact plan keyed by parsed source-record index.
|
||||||
|
pub(super) struct RollbackPlan {
|
||||||
|
record_boundaries: Vec<Option<usize>>,
|
||||||
|
boundary_alive: Vec<bool>,
|
||||||
|
compacted_items: HashMap<usize, CompactedItem>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RollbackPlan {
|
||||||
|
pub(super) fn record_count(&self) -> usize {
|
||||||
|
self.record_boundaries.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn apply(
|
||||||
|
&self,
|
||||||
|
record_index: usize,
|
||||||
|
mut line: RolloutLine,
|
||||||
|
) -> ThreadStoreResult<Option<RolloutLine>> {
|
||||||
|
let boundary = self
|
||||||
|
.record_boundaries
|
||||||
|
.get(record_index)
|
||||||
|
.ok_or_else(|| migration_error("rollback plan is shorter than source replay"))?;
|
||||||
|
if matches!(
|
||||||
|
&line.item,
|
||||||
|
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_))
|
||||||
|
) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
// A rolled-back turn can still own the empty checkpoint that keeps cold resume from
|
||||||
|
// replaying older history.
|
||||||
|
if let Some(compacted) = self.compacted_items.get(&record_index) {
|
||||||
|
line.item = RolloutItem::Compacted(compacted.clone());
|
||||||
|
return Ok(Some(line));
|
||||||
|
}
|
||||||
|
if boundary.is_some_and(|boundary| !self.boundary_alive[boundary]) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Ok(Some(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streaming builder for RollbackPlan.
|
||||||
|
pub(super) struct RollbackPlanner {
|
||||||
|
record_boundaries: Vec<Option<usize>>,
|
||||||
|
boundary_alive: Vec<bool>,
|
||||||
|
boundary_stack: Vec<usize>,
|
||||||
|
active_turn_id: Option<String>,
|
||||||
|
pending_turn_records: Vec<usize>,
|
||||||
|
pending_context_records: Vec<usize>,
|
||||||
|
pending_user_response: Option<PendingUserResponse>,
|
||||||
|
pending_delivery_boundary: Option<usize>,
|
||||||
|
turn_boundaries: HashMap<String, usize>,
|
||||||
|
compactions: Vec<CompactionFrame>,
|
||||||
|
model_replay: ModelReplayPlanner,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RollbackPlanner {
|
||||||
|
pub(super) fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
record_boundaries: Vec::new(),
|
||||||
|
boundary_alive: Vec::new(),
|
||||||
|
boundary_stack: Vec::new(),
|
||||||
|
active_turn_id: None,
|
||||||
|
pending_turn_records: Vec::new(),
|
||||||
|
pending_context_records: Vec::new(),
|
||||||
|
pending_user_response: None,
|
||||||
|
pending_delivery_boundary: None,
|
||||||
|
turn_boundaries: HashMap::new(),
|
||||||
|
compactions: Vec::new(),
|
||||||
|
model_replay: ModelReplayPlanner::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn observe(&mut self, line: &RolloutLine) -> ThreadStoreResult<()> {
|
||||||
|
let index = self.record_boundaries.len();
|
||||||
|
self.model_replay.observe(index, &line.item);
|
||||||
|
self.record_boundaries
|
||||||
|
.push(self.boundary_stack.last().copied());
|
||||||
|
let paired_user_boundary = match (&self.pending_user_response, &line.item) {
|
||||||
|
(Some(pending), RolloutItem::EventMsg(EventMsg::UserMessage(event)))
|
||||||
|
if user_response_matches_event(&pending.content, event) =>
|
||||||
|
{
|
||||||
|
Some(pending.boundary)
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let paired_delivery_boundary = match (&self.pending_delivery_boundary, &line.item) {
|
||||||
|
(Some(boundary), RolloutItem::ResponseItem(ResponseItem::AgentMessage { .. })) => {
|
||||||
|
Some(*boundary)
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
self.pending_user_response = None;
|
||||||
|
self.pending_delivery_boundary = None;
|
||||||
|
|
||||||
|
match &line.item {
|
||||||
|
RolloutItem::SessionMeta(_) => self.record_boundaries[index] = None,
|
||||||
|
RolloutItem::ResponseItem(response) => {
|
||||||
|
if let Some(boundary) = paired_delivery_boundary {
|
||||||
|
self.record_boundaries[index] = Some(boundary);
|
||||||
|
} else if rollback::counts_as_boundary(response) {
|
||||||
|
let boundary = self.start_boundary(index);
|
||||||
|
if let ResponseItem::Message { role, content, .. } = response
|
||||||
|
&& role == "user"
|
||||||
|
{
|
||||||
|
self.pending_user_response = Some(PendingUserResponse {
|
||||||
|
boundary,
|
||||||
|
content: content.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if rollback::is_pre_turn_context_update(response) {
|
||||||
|
// Until another user boundary arrives, this is trailing context for the
|
||||||
|
// previous turn. Keep that fallback owner so rollback drops it when there is
|
||||||
|
// no later turn to attach it to.
|
||||||
|
self.pending_context_records.push(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
|
||||||
|
self.apply_rollback(rollback.num_turns)?;
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
|
||||||
|
self.active_turn_id = Some(event.turn_id.clone());
|
||||||
|
self.pending_turn_records.clear();
|
||||||
|
self.pending_turn_records.push(index);
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
|
||||||
|
self.assign_targeted_record(index, Some(event.turn_id.as_str()));
|
||||||
|
if self.active_turn_id.as_deref() == Some(event.turn_id.as_str()) {
|
||||||
|
self.active_turn_id = None;
|
||||||
|
self.pending_turn_records.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
|
||||||
|
self.assign_targeted_record(index, event.turn_id.as_deref());
|
||||||
|
if event
|
||||||
|
.turn_id
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|turn_id| self.active_turn_id.as_deref() == Some(turn_id))
|
||||||
|
{
|
||||||
|
self.active_turn_id = None;
|
||||||
|
self.pending_turn_records.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::UserMessage(_)) => {
|
||||||
|
let boundary = paired_user_boundary.unwrap_or_else(|| self.start_boundary(index));
|
||||||
|
self.record_boundaries[index] = Some(boundary);
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => {
|
||||||
|
self.assign_targeted_record(index, Some(event.turn_id.as_str()));
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(event) => {
|
||||||
|
self.assign_targeted_record(index, explicit_event_turn_id(event));
|
||||||
|
}
|
||||||
|
RolloutItem::InterAgentCommunication(_) => {
|
||||||
|
self.start_boundary(index);
|
||||||
|
}
|
||||||
|
RolloutItem::InterAgentCommunicationMetadata { .. } => {
|
||||||
|
let boundary = self.start_boundary(index);
|
||||||
|
self.pending_delivery_boundary = Some(boundary);
|
||||||
|
}
|
||||||
|
RolloutItem::Compacted(item) => {
|
||||||
|
let owner = self
|
||||||
|
.active_turn_id
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|turn_id| self.turn_boundaries.get(turn_id).copied());
|
||||||
|
self.record_boundaries[index] = owner;
|
||||||
|
self.compactions.push(CompactionFrame {
|
||||||
|
record_index: index,
|
||||||
|
boundary_depth: self.boundary_stack.len(),
|
||||||
|
owner,
|
||||||
|
item: item.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
RolloutItem::TurnContext(_) => {
|
||||||
|
if self.active_turn_id.is_some()
|
||||||
|
&& self
|
||||||
|
.active_turn_id
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
|
||||||
|
{
|
||||||
|
self.pending_turn_records.push(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RolloutItem::WorldState(_) => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn finish(self) -> RollbackPlan {
|
||||||
|
let RollbackPlanner {
|
||||||
|
record_boundaries,
|
||||||
|
boundary_alive,
|
||||||
|
compactions,
|
||||||
|
model_replay,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
let replay_anchor = model_replay.finish().empty_replacement_history_compaction;
|
||||||
|
let compacted_items = compactions
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|mut frame| {
|
||||||
|
if Some(frame.record_index) == replay_anchor {
|
||||||
|
frame.item.replacement_history = Some(Vec::new());
|
||||||
|
return Some((frame.record_index, frame.item));
|
||||||
|
}
|
||||||
|
frame
|
||||||
|
.owner
|
||||||
|
.is_none_or(|boundary| boundary_alive[boundary])
|
||||||
|
.then_some((frame.record_index, frame.item))
|
||||||
|
})
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
RollbackPlan {
|
||||||
|
record_boundaries,
|
||||||
|
boundary_alive,
|
||||||
|
compacted_items,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_boundary(&mut self, index: usize) -> usize {
|
||||||
|
let boundary = self.boundary_alive.len();
|
||||||
|
self.boundary_alive.push(true);
|
||||||
|
let had_prior_boundary = !self.boundary_stack.is_empty();
|
||||||
|
if had_prior_boundary {
|
||||||
|
for pending_index in self.pending_context_records.drain(..) {
|
||||||
|
self.record_boundaries[pending_index] = Some(boundary);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.pending_context_records.clear();
|
||||||
|
}
|
||||||
|
for pending_index in self.pending_turn_records.drain(..) {
|
||||||
|
self.record_boundaries[pending_index] = Some(boundary);
|
||||||
|
}
|
||||||
|
self.record_boundaries[index] = Some(boundary);
|
||||||
|
self.boundary_stack.push(boundary);
|
||||||
|
self.bind_active_turn(boundary);
|
||||||
|
boundary
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bind_active_turn(&mut self, boundary: usize) {
|
||||||
|
if let Some(turn_id) = self.active_turn_id.as_ref() {
|
||||||
|
self.turn_boundaries.insert(turn_id.clone(), boundary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assign_targeted_record(&mut self, index: usize, turn_id: Option<&str>) {
|
||||||
|
if let Some(boundary) = turn_id.and_then(|turn_id| self.turn_boundaries.get(turn_id)) {
|
||||||
|
self.record_boundaries[index] = Some(*boundary);
|
||||||
|
} else if self.active_turn_id.is_some()
|
||||||
|
&& self
|
||||||
|
.active_turn_id
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|turn_id| !self.turn_boundaries.contains_key(turn_id))
|
||||||
|
{
|
||||||
|
self.pending_turn_records.push(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_rollback(&mut self, num_turns: u32) -> ThreadStoreResult<()> {
|
||||||
|
let count = usize::try_from(num_turns).unwrap_or(usize::MAX);
|
||||||
|
if count == 0 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let depth_before = self.boundary_stack.len();
|
||||||
|
for _ in 0..count {
|
||||||
|
let Some(boundary) = self.boundary_stack.pop() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
self.boundary_alive[boundary] = false;
|
||||||
|
}
|
||||||
|
let compaction_index = self.compactions.iter().rposition(|frame| {
|
||||||
|
frame
|
||||||
|
.owner
|
||||||
|
.is_none_or(|boundary| self.boundary_alive[boundary])
|
||||||
|
});
|
||||||
|
if let Some(compaction_index) = compaction_index {
|
||||||
|
let frame = &mut self.compactions[compaction_index];
|
||||||
|
let post_compaction_turns = depth_before.saturating_sub(frame.boundary_depth);
|
||||||
|
let remaining = count.saturating_sub(post_compaction_turns);
|
||||||
|
if remaining > 0 {
|
||||||
|
let replacement_history =
|
||||||
|
frame.item.replacement_history.as_mut().ok_or_else(|| {
|
||||||
|
migration_error(
|
||||||
|
"legacy rollback crosses a compaction without replacement history",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
rollback::drop_last_n_user_turns(
|
||||||
|
replacement_history,
|
||||||
|
u32::try_from(remaining).unwrap_or(u32::MAX),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.active_turn_id = None;
|
||||||
|
self.pending_turn_records.clear();
|
||||||
|
self.pending_context_records.clear();
|
||||||
|
self.pending_user_response = None;
|
||||||
|
self.pending_delivery_boundary = None;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {
|
||||||
|
match event {
|
||||||
|
EventMsg::ExecCommandEnd(event) => Some(event.turn_id.as_str()),
|
||||||
|
EventMsg::PatchApplyEnd(event) => Some(event.turn_id.as_str()),
|
||||||
|
EventMsg::DynamicToolCallResponse(event) => Some(event.turn_id.as_str()),
|
||||||
|
EventMsg::EnteredReviewMode(event) => event.turn_id.as_deref(),
|
||||||
|
EventMsg::ExitedReviewMode(event) => event.turn_id.as_deref(),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
.filter(|turn_id| !turn_id.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {
|
||||||
|
let mut text = String::new();
|
||||||
|
let mut images = Vec::new();
|
||||||
|
let mut audio = Vec::new();
|
||||||
|
for item in content {
|
||||||
|
match item {
|
||||||
|
ContentItem::InputText { text: item_text } => text.push_str(item_text),
|
||||||
|
ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),
|
||||||
|
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
|
||||||
|
ContentItem::OutputText { .. } => return false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
text == event.message
|
||||||
|
&& images
|
||||||
|
== event
|
||||||
|
.images
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.iter()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
&& audio
|
||||||
|
== event
|
||||||
|
.audio
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.iter()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
&& event.local_images.is_empty()
|
||||||
|
&& event.local_audio.is_empty()
|
||||||
|
}
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
//! Mirrors the part of legacy cold resume that chooses the model-history checkpoint.
|
||||||
|
//!
|
||||||
|
//! Legacy rollback has two jobs. The forward history reducer decides which turns remain visible,
|
||||||
|
//! while cold resume scans backward to find the newest surviving compaction base. Migration needs
|
||||||
|
//! both answers because removing rollback markers must not change what the next resume sends to
|
||||||
|
//! the model.
|
||||||
|
|
||||||
|
use codex_protocol::protocol::EventMsg;
|
||||||
|
use codex_protocol::protocol::RolloutItem;
|
||||||
|
|
||||||
|
use super::rollback;
|
||||||
|
|
||||||
|
enum ReplayRecord {
|
||||||
|
Compacted {
|
||||||
|
record_index: usize,
|
||||||
|
has_replacement_history: bool,
|
||||||
|
},
|
||||||
|
Rollback(u32),
|
||||||
|
TurnStarted(String),
|
||||||
|
TurnComplete(String),
|
||||||
|
TurnAborted(Option<String>),
|
||||||
|
TurnContext(Option<String>),
|
||||||
|
UserBoundary,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ReverseSegment {
|
||||||
|
turn_id: Option<String>,
|
||||||
|
counts_as_user_turn: bool,
|
||||||
|
compaction_index: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rewrite needed when legacy reverse replay skips a compaction segment but still resumes from
|
||||||
|
/// the suffix after that checkpoint.
|
||||||
|
pub(super) struct ModelReplayPlan {
|
||||||
|
pub(super) empty_replacement_history_compaction: Option<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct ModelReplayPlanner {
|
||||||
|
records: Vec<ReplayRecord>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ModelReplayPlanner {
|
||||||
|
pub(super) fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
records: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn observe(&mut self, record_index: usize, item: &RolloutItem) {
|
||||||
|
let record = match item {
|
||||||
|
RolloutItem::Compacted(compacted) => ReplayRecord::Compacted {
|
||||||
|
record_index,
|
||||||
|
has_replacement_history: compacted.replacement_history.is_some(),
|
||||||
|
},
|
||||||
|
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => {
|
||||||
|
ReplayRecord::Rollback(rollback.num_turns)
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
|
||||||
|
ReplayRecord::TurnStarted(event.turn_id.clone())
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
|
||||||
|
ReplayRecord::TurnComplete(event.turn_id.clone())
|
||||||
|
}
|
||||||
|
RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => {
|
||||||
|
ReplayRecord::TurnAborted(event.turn_id.clone())
|
||||||
|
}
|
||||||
|
RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()),
|
||||||
|
RolloutItem::EventMsg(EventMsg::UserMessage(_))
|
||||||
|
| RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary,
|
||||||
|
RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => {
|
||||||
|
ReplayRecord::UserBoundary
|
||||||
|
}
|
||||||
|
RolloutItem::SessionMeta(_)
|
||||||
|
| RolloutItem::ResponseItem(_)
|
||||||
|
| RolloutItem::EventMsg(_)
|
||||||
|
| RolloutItem::InterAgentCommunicationMetadata { .. }
|
||||||
|
| RolloutItem::WorldState(_) => return,
|
||||||
|
};
|
||||||
|
self.records.push(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn finish(self) -> ModelReplayPlan {
|
||||||
|
let mut selected_compaction = None;
|
||||||
|
let mut suffix_start = None;
|
||||||
|
let mut pending_rollback_turns = 0usize;
|
||||||
|
let mut active_segment: Option<ReverseSegment> = None;
|
||||||
|
|
||||||
|
for record in self.records.iter().rev() {
|
||||||
|
match record {
|
||||||
|
ReplayRecord::Compacted {
|
||||||
|
record_index,
|
||||||
|
has_replacement_history,
|
||||||
|
} => {
|
||||||
|
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
|
||||||
|
if segment.compaction_index.is_none() && *has_replacement_history {
|
||||||
|
segment.compaction_index = Some(*record_index);
|
||||||
|
suffix_start = record_index.checked_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReplayRecord::Rollback(num_turns) => {
|
||||||
|
pending_rollback_turns = pending_rollback_turns
|
||||||
|
.saturating_add(usize::try_from(*num_turns).unwrap_or(usize::MAX));
|
||||||
|
}
|
||||||
|
ReplayRecord::TurnComplete(turn_id) => {
|
||||||
|
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
|
||||||
|
if segment.turn_id.is_none() {
|
||||||
|
segment.turn_id = Some(turn_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReplayRecord::TurnAborted(turn_id) => {
|
||||||
|
if let Some(segment) = active_segment.as_mut() {
|
||||||
|
if segment.turn_id.is_none() {
|
||||||
|
segment.turn_id = turn_id.clone();
|
||||||
|
}
|
||||||
|
} else if turn_id.is_some() {
|
||||||
|
active_segment = Some(ReverseSegment {
|
||||||
|
turn_id: turn_id.clone(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReplayRecord::TurnContext(turn_id) => {
|
||||||
|
let segment = active_segment.get_or_insert_with(ReverseSegment::default);
|
||||||
|
if segment.turn_id.is_none() {
|
||||||
|
segment.turn_id = turn_id.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ReplayRecord::UserBoundary => {
|
||||||
|
active_segment
|
||||||
|
.get_or_insert_with(ReverseSegment::default)
|
||||||
|
.counts_as_user_turn = true;
|
||||||
|
}
|
||||||
|
ReplayRecord::TurnStarted(turn_id) => {
|
||||||
|
if active_segment.as_ref().is_some_and(|segment| {
|
||||||
|
turn_ids_are_compatible(segment.turn_id.as_deref(), Some(turn_id.as_str()))
|
||||||
|
}) && let Some(segment) = active_segment.take()
|
||||||
|
{
|
||||||
|
finalize_segment(
|
||||||
|
segment,
|
||||||
|
&mut selected_compaction,
|
||||||
|
&mut pending_rollback_turns,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(segment) = active_segment {
|
||||||
|
finalize_segment(
|
||||||
|
segment,
|
||||||
|
&mut selected_compaction,
|
||||||
|
&mut pending_rollback_turns,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If reverse replay saw a replacement-history checkpoint but rolled back its segment,
|
||||||
|
// it resumes from that checkpoint's suffix without using the checkpoint history. Keep an
|
||||||
|
// empty checkpoint in the paginated file so older SQLite-visible records do not leak back
|
||||||
|
// into model context after the rollback markers are gone.
|
||||||
|
let empty_replacement_history_compaction = selected_compaction
|
||||||
|
.is_none()
|
||||||
|
.then_some(suffix_start)
|
||||||
|
.flatten()
|
||||||
|
.and_then(|start| start.checked_sub(1));
|
||||||
|
ModelReplayPlan {
|
||||||
|
empty_replacement_history_compaction,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finalize_segment(
|
||||||
|
segment: ReverseSegment,
|
||||||
|
selected_compaction: &mut Option<usize>,
|
||||||
|
pending_rollback_turns: &mut usize,
|
||||||
|
) {
|
||||||
|
if *pending_rollback_turns > 0 {
|
||||||
|
if segment.counts_as_user_turn {
|
||||||
|
*pending_rollback_turns -= 1;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if selected_compaction.is_none() {
|
||||||
|
*selected_compaction = segment.compaction_index;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool {
|
||||||
|
active_turn_id
|
||||||
|
.is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id))
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
//! Selects a smaller resume context for legacy subagent rollouts.
|
||||||
|
//!
|
||||||
|
//! Legacy subagents copied the parent's full rollout into every child rollout. Migrating those
|
||||||
|
//! files verbatim can preserve gigabytes of duplicated history even though a resumed subagent only
|
||||||
|
//! needs its latest bounded model context.
|
||||||
|
//!
|
||||||
|
//! This module reverse-scans the legacy rollout for a safe model-context checkpoint and returns the
|
||||||
|
//! smallest replay needed to resume from there. Malformed reverse-scanned records are skipped. If
|
||||||
|
//! we cannot prove that a bounded replay is safe, we return `None` and let the caller fall back to
|
||||||
|
//! full tolerant replay.
|
||||||
|
|
||||||
|
use std::fs::File;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use codex_protocol::protocol::RolloutItem;
|
||||||
|
use codex_protocol::protocol::SessionMetaLine;
|
||||||
|
use codex_rollout::ModelContextScan;
|
||||||
|
use codex_rollout::ModelContextScanProgress;
|
||||||
|
use codex_rollout::ReverseJsonlScanner;
|
||||||
|
use codex_rollout::ScanOutcome;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use super::line_parser;
|
||||||
|
use super::migration_error;
|
||||||
|
use crate::ThreadStoreResult;
|
||||||
|
|
||||||
|
/// Returns a bounded replay when the legacy child has enough durable context to reconstruct from
|
||||||
|
/// a suffix. Without that proof, the caller streams the complete legacy replay instead.
|
||||||
|
pub(super) async fn select_bounded_context(
|
||||||
|
rollout_path: PathBuf,
|
||||||
|
session_meta: SessionMetaLine,
|
||||||
|
) -> ThreadStoreResult<Option<Vec<RolloutItem>>> {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let file = File::open(rollout_path).map_err(migration_error)?;
|
||||||
|
let mut scanner = ReverseJsonlScanner::new(file)
|
||||||
|
.map_err(migration_error)?
|
||||||
|
.with_max_record_bytes(super::MAX_ROLLOUT_LINE_BYTES);
|
||||||
|
let mut scan = ModelContextScan::default();
|
||||||
|
|
||||||
|
while let Some(outcome) = scanner.scan_next::<Value>().map_err(migration_error)? {
|
||||||
|
let value = match outcome {
|
||||||
|
ScanOutcome::Parsed(value) => value,
|
||||||
|
ScanOutcome::Rejected(_) => continue,
|
||||||
|
};
|
||||||
|
let Ok(Some(line)) = line_parser::parse_legacy_rollout_value(value) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if scan.push(line.item) == ModelContextScanProgress::Complete {
|
||||||
|
let mut items = scan.finish(session_meta);
|
||||||
|
items.retain(|item| !matches!(item, RolloutItem::SessionMeta(_)));
|
||||||
|
return Ok(Some(items));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(None)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(migration_error)?
|
||||||
|
}
|
||||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user