Add legacy rollout migration to paginated history (#37175)

## What changed

- Add `LocalThreadStore::migrate_rollouts` with dry-run and apply modes,
  optional thread selection, throughput limiting, and per-rollout outcomes.
- Canonicalize legacy JSONL records into paginated history while preserving
  model-visible conversation context, including compressed rollouts and copied
  fork history.
- Publish replacements atomically and journal migrations so interrupted SQLite
  projections can be recovered. Coordinate migration with rollout compression
  and skip rollouts with active writers.
- Make paginated history a one-way SQLite promotion so stale metadata cannot
  downgrade a migrated thread.

## Testing

- Cover legacy record normalization, dry runs, idempotency, malformed input,
  compressed rollouts, active writers, interrupted migration recovery, and a
  cold app-server resume after migration.

GitOrigin-RevId: b9991b659f28ebb52da39ce62e8b4e0bae2ba2bb
This commit is contained in:
Owen Lin authored and copyberry committed 2026-08-05 22:57:27 +00:00
1 parent e3465b48ad
commit 6bb6e9045f
18 files changed
+3115 -5

No files matched your search

@@ -0,0 +1,505 @@
//! Replays normalized legacy rollout records into canonical paginated JSONL.
//!
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at
//! ordinal zero, and emits the surviving history selected by the caller.
//!
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
//! module.
use chrono::DateTime;
use codex_protocol::ThreadId;
use codex_protocol::items::ReasoningItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::parse_hook_prompt_message;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use std::collections::HashSet;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use tokio::io::BufWriter;
use super::legacy_event;
use super::migration_error;
use crate::ThreadStoreResult;
#[derive(Clone)]
struct ActiveTurn {
id: String,
explicit: bool,
saw_user: bool,
}
enum ReasoningTextKind {
Summary,
Raw,
}
pub(super) struct LegacyRolloutCanonicalizer {
thread_id: ThreadId,
next_ordinal: u64,
next_item_index: u64,
output_byte_offset: u64,
bytes_written: u64,
source_line_index: u64,
active_turn: Option<ActiveTurn>,
known_turn_ids: HashSet<String>,
reasoning: Option<ReasoningItem>,
saw_source_session_meta: bool,
}
impl LegacyRolloutCanonicalizer {
pub(super) fn new(thread_id: ThreadId) -> Self {
Self {
thread_id,
next_ordinal: 0,
next_item_index: 1,
output_byte_offset: 0,
bytes_written: 0,
source_line_index: 0,
active_turn: None,
known_turn_ids: HashSet::new(),
reasoning: None,
saw_source_session_meta: false,
}
}
pub(super) fn output_byte_offset(&self) -> u64 {
self.output_byte_offset
}
pub(super) fn next_ordinal(&self) -> u64 {
self.next_ordinal
}
pub(super) async fn write_head_session_meta(
&mut self,
line: RolloutLine,
writer: &mut BufWriter<File>,
) -> ThreadStoreResult<u64> {
let timestamp = line.timestamp;
let RolloutItem::SessionMeta(mut metadata) = line.item else {
return Err(migration_error("canonical session metadata is missing"));
};
if metadata.meta.id != self.thread_id {
return Err(migration_error("rollout metadata thread id changed"));
}
metadata.meta.history_mode = ThreadHistoryMode::Paginated;
metadata.meta.history_base = None;
metadata.meta.subagent_history_start_ordinal = None;
let bytes_before = self.bytes_written;
self.write_item(writer, &timestamp, RolloutItem::SessionMeta(metadata))
.await?;
Ok(self.bytes_written - bytes_before)
}
pub(super) async fn process_line(
&mut self,
line: RolloutLine,
writer: &mut BufWriter<File>,
) -> ThreadStoreResult<u64> {
let source_index = self.source_line_index;
self.source_line_index = self
.source_line_index
.checked_add(1)
.ok_or_else(|| migration_error("legacy rollout line index overflow"))?;
let timestamp = line.timestamp;
let bytes_before = self.bytes_written;
match line.item {
RolloutItem::SessionMeta(metadata) => {
if !self.saw_source_session_meta {
self.saw_source_session_meta = true;
return Ok(0);
}
self.write_item(writer, &timestamp, RolloutItem::SessionMeta(metadata))
.await?;
}
RolloutItem::ResponseItem(ResponseItem::Other) => {
return Err(migration_error(
"legacy rollout contains an unsupported response item",
));
}
RolloutItem::ResponseItem(response) => {
let hook = match &response {
ResponseItem::Message {
role, content, id, ..
} if role == "user" => parse_hook_prompt_message(id.as_deref(), content),
_ => None,
};
self.write_item(writer, &timestamp, RolloutItem::ResponseItem(response))
.await?;
if let Some(hook) = hook {
self.ensure_turn(writer, &timestamp, source_index).await?;
self.reasoning = None;
self.write_completed_item(writer, &timestamp, TurnItem::HookPrompt(hook))
.await?;
}
}
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_)) => {
return Err(migration_error(
"rollback marker reached canonical writer without a rollback plan",
));
}
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
self.finish_implicit_turn(writer, &timestamp).await?;
self.known_turn_ids.insert(event.turn_id.clone());
self.active_turn = Some(ActiveTurn {
id: event.turn_id.clone(),
explicit: true,
saw_user: false,
});
self.reasoning = None;
self.write_item(
writer,
&timestamp,
RolloutItem::EventMsg(EventMsg::TurnStarted(event)),
)
.await?;
}
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
self.reasoning = None;
if self
.active_turn
.as_ref()
.is_some_and(|turn| turn.id == event.turn_id)
{
self.active_turn = None;
}
self.write_item(
writer,
&timestamp,
RolloutItem::EventMsg(EventMsg::TurnComplete(event)),
)
.await?;
}
RolloutItem::EventMsg(EventMsg::TurnAborted(mut event)) => {
if event.turn_id.is_none() {
event.turn_id = self.active_turn.as_ref().map(|turn| turn.id.clone());
}
if self
.active_turn
.as_ref()
.is_some_and(|turn| event.turn_id.as_deref() == Some(turn.id.as_str()))
{
self.active_turn = None;
}
self.reasoning = None;
self.write_item(
writer,
&timestamp,
RolloutItem::EventMsg(EventMsg::TurnAborted(event)),
)
.await?;
}
RolloutItem::EventMsg(EventMsg::UserMessage(event)) => {
if self
.active_turn
.as_ref()
.is_some_and(|turn| !turn.explicit && turn.saw_user)
{
self.finish_implicit_turn(writer, &timestamp).await?;
}
self.ensure_turn(writer, &timestamp, source_index).await?;
let item = legacy_event::user_message_item(event, &mut || self.next_item_id())?;
if let Some(turn) = self.active_turn.as_mut() {
turn.saw_user = true;
}
self.reasoning = None;
self.write_completed_item(writer, &timestamp, item).await?;
}
RolloutItem::EventMsg(EventMsg::AgentReasoning(event)) => {
self.write_reasoning(
writer,
&timestamp,
source_index,
event.text,
ReasoningTextKind::Summary,
)
.await?;
}
RolloutItem::EventMsg(EventMsg::AgentReasoningRawContent(event)) => {
self.write_reasoning(
writer,
&timestamp,
source_index,
event.text,
ReasoningTextKind::Raw,
)
.await?;
}
RolloutItem::EventMsg(EventMsg::ItemCompleted(mut event)) => {
event.thread_id = self.thread_id;
self.reasoning = None;
self.write_item(
writer,
&timestamp,
RolloutItem::EventMsg(EventMsg::ItemCompleted(event)),
)
.await?;
}
RolloutItem::EventMsg(event) => {
if let Some((item, turn_id)) =
legacy_event::completed_item(&event, &mut || self.next_item_id())?
{
match turn_id {
Some(turn_id)
if self
.active_turn
.as_ref()
.is_some_and(|turn| turn.id.as_str() != turn_id.as_str()) =>
{
self.write_completed_item_to_turn(writer, &timestamp, turn_id, item)
.await?;
}
Some(turn_id) => {
if self.active_turn.is_none()
&& self.known_turn_ids.contains(turn_id.as_str())
{
self.reasoning = None;
self.write_completed_item_to_turn(
writer, &timestamp, turn_id, item,
)
.await?;
} else if self.active_turn.is_none() {
self.start_implicit_turn(writer, &timestamp, turn_id)
.await?;
self.reasoning = None;
self.write_completed_item(writer, &timestamp, item).await?;
} else {
self.reasoning = None;
self.write_completed_item(writer, &timestamp, item).await?;
}
}
None => {
self.ensure_turn(writer, &timestamp, source_index).await?;
self.reasoning = None;
self.write_completed_item(writer, &timestamp, item).await?;
}
}
} else {
let item = RolloutItem::EventMsg(event);
if codex_rollout::is_persisted_rollout_item(&item, ThreadHistoryMode::Paginated)
{
self.write_item(writer, &timestamp, item).await?;
}
}
}
item @ RolloutItem::InterAgentCommunication(_) => {
self.write_item(writer, &timestamp, item).await?;
}
item @ RolloutItem::Compacted(_) => {
self.write_item(writer, &timestamp, item).await?;
}
item @ (RolloutItem::InterAgentCommunicationMetadata { .. }
| RolloutItem::TurnContext(_)
| RolloutItem::WorldState(_)) => {
self.write_item(writer, &timestamp, item).await?;
}
}
Ok(self.bytes_written - bytes_before)
}
pub(super) async fn finish(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
) -> ThreadStoreResult<u64> {
let bytes_before = self.bytes_written;
self.finish_implicit_turn(writer, timestamp).await?;
Ok(self.bytes_written - bytes_before)
}
async fn ensure_turn(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
source_index: u64,
) -> ThreadStoreResult<()> {
if self.active_turn.is_some() {
return Ok(());
}
let turn_id = format!("rollout-{source_index}");
self.start_implicit_turn(writer, timestamp, turn_id).await
}
async fn start_implicit_turn(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
turn_id: String,
) -> ThreadStoreResult<()> {
self.active_turn = Some(ActiveTurn {
id: turn_id.clone(),
explicit: false,
saw_user: false,
});
self.known_turn_ids.insert(turn_id.clone());
self.reasoning = None;
let started_at = DateTime::parse_from_rfc3339(timestamp)
.map_err(migration_error)?
.timestamp();
self.write_item(
writer,
timestamp,
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
turn_id,
trace_id: None,
started_at: Some(started_at),
model_context_window: None,
collaboration_mode_kind: Default::default(),
})),
)
.await
}
async fn finish_implicit_turn(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
) -> ThreadStoreResult<()> {
let Some(turn) = self.active_turn.as_ref() else {
return Ok(());
};
if turn.explicit {
return Ok(());
}
let turn_id = turn.id.clone();
self.active_turn = None;
self.reasoning = None;
self.write_item(
writer,
timestamp,
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
turn_id,
last_agent_message: None,
error: None,
started_at: None,
completed_at: Some(
DateTime::parse_from_rfc3339(timestamp)
.map_err(migration_error)?
.timestamp(),
),
duration_ms: None,
time_to_first_token_ms: None,
})),
)
.await
}
async fn write_completed_item(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
item: TurnItem,
) -> ThreadStoreResult<()> {
let turn_id = self
.active_turn
.as_ref()
.map(|turn| turn.id.clone())
.ok_or_else(|| migration_error("completed rollout item has no active turn"))?;
self.write_completed_item_to_turn(writer, timestamp, turn_id, item)
.await
}
async fn write_completed_item_to_turn(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
turn_id: String,
item: TurnItem,
) -> ThreadStoreResult<()> {
let completed_at_ms = DateTime::parse_from_rfc3339(timestamp)
.map_err(migration_error)?
.timestamp_millis();
self.write_item(
writer,
timestamp,
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
thread_id: self.thread_id,
turn_id,
item,
started_at_ms: None,
completed_at_ms,
})),
)
.await
}
async fn write_reasoning(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
source_index: u64,
text: String,
kind: ReasoningTextKind,
) -> ThreadStoreResult<()> {
if text.is_empty() {
return Ok(());
}
self.ensure_turn(writer, timestamp, source_index).await?;
let mut item = match self.reasoning.take() {
Some(item) => item,
None => ReasoningItem {
id: self.next_item_id()?,
summary_text: Vec::new(),
raw_content: Vec::new(),
},
};
match kind {
ReasoningTextKind::Summary => item.summary_text.push(text),
ReasoningTextKind::Raw => item.raw_content.push(text),
}
self.reasoning = Some(item.clone());
self.write_completed_item(writer, timestamp, TurnItem::Reasoning(item))
.await
}
async fn write_item(
&mut self,
writer: &mut BufWriter<File>,
timestamp: &str,
item: RolloutItem,
) -> ThreadStoreResult<()> {
let mut bytes = serde_json::to_vec(&RolloutLine {
timestamp: timestamp.to_string(),
ordinal: Some(self.next_ordinal),
item,
})
.map_err(migration_error)?;
bytes.push(b'\n');
writer.write_all(&bytes).await.map_err(migration_error)?;
let byte_count = u64::try_from(bytes.len())
.map_err(|_| migration_error("rollout record exceeds addressable size"))?;
self.output_byte_offset = self
.output_byte_offset
.checked_add(byte_count)
.ok_or_else(|| migration_error("paginated rollout byte offset overflow"))?;
self.bytes_written = self
.bytes_written
.checked_add(byte_count)
.ok_or_else(|| migration_error("paginated rollout byte count overflow"))?;
self.next_ordinal = self
.next_ordinal
.checked_add(1)
.ok_or_else(|| migration_error("paginated rollout ordinal overflow"))?;
Ok(())
}
fn next_item_id(&mut self) -> ThreadStoreResult<String> {
let item_id = format!("item-{}", self.next_item_index);
self.next_item_index = self
.next_item_index
.checked_add(1)
.ok_or_else(|| migration_error("legacy rollout item id overflow"))?;
Ok(item_id)
}
}
@@ -0,0 +1,251 @@
//! Converts legacy persisted events into modern paginated turn items.
//!
//! Legacy rollouts persisted events like `ExecCommandEnd`, `PatchApplyEnd`, and
//! `McpToolCallEnd`. Paginated history instead persists `ItemCompleted(item)` records containing
//! canonical `TurnItem`s.
//!
//! This module owns that old-event -> new-item mapping, including stable synthesized item IDs.
//! It intentionally stays separate from live thread-history reduction because migration needs a
//! frozen adapter for historical rollout payloads.
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::AgentMessageItem;
use codex_protocol::items::CommandExecutionItem;
use codex_protocol::items::ContextCompactionItem;
use codex_protocol::items::DynamicToolCallItem;
use codex_protocol::items::DynamicToolCallStatus;
use codex_protocol::items::EnteredReviewModeItem;
use codex_protocol::items::ExitedReviewModeItem;
use codex_protocol::items::FileChangeItem;
use codex_protocol::items::ImageGenerationItem;
use codex_protocol::items::McpToolCallError;
use codex_protocol::items::McpToolCallItem;
use codex_protocol::items::McpToolCallStatus;
use codex_protocol::items::SubAgentActivityItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
use codex_protocol::user_input::UserInput;
use crate::ThreadStoreResult;
pub(super) fn user_message_item(
event: UserMessageEvent,
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
) -> ThreadStoreResult<TurnItem> {
let mut content = Vec::new();
if !event.message.trim().is_empty() {
content.push(UserInput::Text {
text: event.message,
text_elements: event.text_elements,
});
}
if let Some(images) = event.images {
for (index, image_url) in images.into_iter().enumerate() {
content.push(UserInput::Image {
image_url,
detail: event.image_details.get(index).copied().flatten(),
});
}
}
for (index, path) in event.local_images.into_iter().enumerate() {
content.push(UserInput::LocalImage {
path,
detail: event.local_image_details.get(index).copied().flatten(),
});
}
if let Some(audio) = event.audio {
content.extend(
audio
.into_iter()
.map(|audio_url| UserInput::Audio { audio_url }),
);
}
content.extend(
event
.local_audio
.into_iter()
.map(|path| UserInput::LocalAudio { path }),
);
Ok(TurnItem::UserMessage(UserMessageItem {
id: next_item_id()?,
client_id: event.client_id,
content,
}))
}
/// Convert historical completion events into the canonical completed-item payloads
/// that paginated rollouts persist. This deliberately remains a frozen migration
/// adapter rather than sharing the live thread-history reducer: migration must
/// preserve historical payloads while synthesizing stable item and turn IDs.
pub(super) fn completed_item(
event: &EventMsg,
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
) -> ThreadStoreResult<Option<(TurnItem, Option<String>)>> {
let result = match event {
EventMsg::AgentMessage(event) if !event.message.is_empty() => Some((
TurnItem::AgentMessage(AgentMessageItem {
id: next_item_id()?,
content: vec![AgentMessageContent::Text {
text: event.message.clone(),
}],
phase: event.phase.clone(),
memory_citation: event.memory_citation.clone(),
}),
None,
)),
EventMsg::PatchApplyEnd(event) => Some((
TurnItem::FileChange(FileChangeItem {
id: event.call_id.clone(),
changes: event.changes.clone(),
status: Some(event.status.clone()),
auto_approved: None,
stdout: (!event.stdout.is_empty()).then(|| event.stdout.clone()),
stderr: (!event.stderr.is_empty()).then(|| event.stderr.clone()),
}),
(!event.turn_id.is_empty()).then(|| event.turn_id.clone()),
)),
EventMsg::McpToolCallEnd(event) => {
let (result, error) = match &event.result {
Ok(result) => (Some(result.clone()), None),
Err(message) => (
None,
Some(McpToolCallError {
message: message.clone(),
}),
),
};
Some((
TurnItem::McpToolCall(McpToolCallItem {
id: event.call_id.clone(),
server: event.invocation.server.clone(),
tool: event.invocation.tool.clone(),
arguments: event
.invocation
.arguments
.clone()
.unwrap_or(serde_json::Value::Null),
connector_id: event.connector_id.clone(),
mcp_app_resource_uri: event.mcp_app_resource_uri.clone(),
link_id: event.link_id.clone(),
app_name: event.app_name.clone(),
action_name: event.action_name.clone(),
plugin_id: event.plugin_id.clone(),
read_only_hint: event.read_only_hint,
status: if event.is_success() {
McpToolCallStatus::Completed
} else {
McpToolCallStatus::Failed
},
result,
error,
duration: Some(event.duration),
}),
None,
))
}
EventMsg::WebSearchEnd(event) => Some((
TurnItem::WebSearch(WebSearchItem {
id: event.call_id.clone(),
query: event.query.clone(),
action: event.action.clone(),
results: event.results.clone(),
}),
None,
)),
EventMsg::ImageGenerationEnd(event) => Some((
TurnItem::ImageGeneration(ImageGenerationItem {
id: event.call_id.clone(),
status: event.status.clone(),
revised_prompt: event.revised_prompt.clone(),
result: event.result.clone(),
saved_path: event.saved_path.clone(),
}),
None,
)),
EventMsg::ContextCompacted(_) => Some((
TurnItem::ContextCompaction(ContextCompactionItem {
id: next_item_id()?,
}),
None,
)),
EventMsg::EnteredReviewMode(event) => Some((
TurnItem::EnteredReviewMode(EnteredReviewModeItem {
id: match event.item_id.clone() {
Some(id) => id,
None => next_item_id()?,
},
target: event.target.clone(),
user_facing_hint: event
.user_facing_hint
.clone()
.unwrap_or_else(|| "Review requested.".to_string()),
}),
event.turn_id.clone(),
)),
EventMsg::ExitedReviewMode(event) => Some((
TurnItem::ExitedReviewMode(ExitedReviewModeItem {
id: match event.item_id.clone() {
Some(id) => id,
None => next_item_id()?,
},
review_output: event.review_output.clone(),
}),
event.turn_id.clone(),
)),
EventMsg::SubAgentActivity(event) => Some((
TurnItem::SubAgentActivity(SubAgentActivityItem {
id: event.event_id.clone(),
kind: event.kind,
agent_thread_id: event.agent_thread_id,
agent_path: event.agent_path.clone(),
}),
None,
)),
EventMsg::ExecCommandEnd(event) => Some((
TurnItem::CommandExecution(CommandExecutionItem {
id: event.call_id.clone(),
plugin_id: event.plugin_id.clone(),
script_path: event.script_path.clone(),
process_id: event.process_id.clone(),
command: event.command.clone(),
cwd: event.cwd.clone(),
parsed_cmd: event.parsed_cmd.clone(),
source: event.source,
interaction_input: event.interaction_input.clone(),
status: event.status.clone().into(),
stdout: (!event.stdout.is_empty()).then(|| event.stdout.clone()),
stderr: (!event.stderr.is_empty()).then(|| event.stderr.clone()),
aggregated_output: (!event.aggregated_output.is_empty())
.then(|| event.aggregated_output.clone()),
exit_code: Some(event.exit_code),
duration: Some(event.duration),
formatted_output: (!event.formatted_output.is_empty())
.then(|| event.formatted_output.clone()),
}),
Some(event.turn_id.clone()),
)),
EventMsg::DynamicToolCallResponse(event) => Some((
TurnItem::DynamicToolCall(DynamicToolCallItem {
id: event.call_id.clone(),
namespace: event.namespace.clone(),
tool: event.tool.clone(),
arguments: event.arguments.clone(),
status: if event.success {
DynamicToolCallStatus::Completed
} else {
DynamicToolCallStatus::Failed
},
content_items: Some(event.content_items.clone()),
success: Some(event.success),
error: event.error.clone(),
duration: Some(event.duration),
}),
Some(event.turn_id.clone()),
)),
_ => None,
};
Ok(result)
}
@@ -0,0 +1,197 @@
//! Makes historical rollout JSON parse like today's protocol types.
//!
//! Legacy rollouts span a bunch of old wire shapes: renamed fields, old sandbox policy layouts,
//! string timestamps, legacy command cwd paths, retired events, and payloads that only deserialize
//! correctly after going through `serde_json::Value`.
//!
//! This module only applies narrowly scoped compatibility rewrites for shapes we know existed in
//! persisted rollouts. It does not decide turn boundaries, rollback behavior, or what gets written
//! into paginated history.
use chrono::DateTime;
use codex_protocol::protocol::RolloutLine;
use codex_utils_path_uri::LegacyAppPathString;
use codex_utils_path_uri::PathUri;
use serde_json::Map;
use serde_json::Value;
/// Parse one legacy rollout line after applying narrowly scoped compatibility
/// rewrites for historical wire shapes that current protocol types no longer
/// accept.
pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLine>, String> {
if bytes.iter().all(u8::is_ascii_whitespace) {
return Ok(None);
}
// Deserializing through Value is intentional. Some historical numeric
// payloads fail serde_json's streaming enum path but deserialize correctly
// once their tagged object shape has been materialized.
let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
if should_skip_retired_record(&value) {
return Ok(None);
}
normalize_legacy_turn_context(&mut value);
normalize_legacy_sandbox_policy(&mut value);
normalize_legacy_rate_limit_resets(&mut value);
normalize_legacy_review_entry(&mut value);
normalize_legacy_command_cwd(&mut value)?;
serde_json::from_value(value)
.map(Some)
.map_err(|error| error.to_string())
}
fn should_skip_retired_record(value: &Value) -> bool {
matches!(
event_type(value),
Some("guardian_assessment" | "thread_name_updated" | "undo_completed")
) || (rollout_type(value) == Some("response_item")
&& value
.get("payload")
.and_then(Value::as_object)
.and_then(|payload| payload.get("type"))
.and_then(Value::as_str)
== Some("ghost_snapshot"))
}
fn normalize_legacy_turn_context(value: &mut Value) {
if rollout_type(value) != Some("turn_context") {
return;
}
let Some(payload) = payload_object_mut(value) else {
return;
};
let Some(collaboration_mode) = payload
.get_mut("collaboration_mode")
.and_then(Value::as_object_mut)
else {
return;
};
if collaboration_mode.contains_key("settings") {
return;
}
let mut settings = Map::new();
for key in ["model", "reasoning_effort", "developer_instructions"] {
if let Some(value) = collaboration_mode.get(key) {
settings.insert(key.to_string(), value.clone());
}
}
if !settings.is_empty() {
collaboration_mode.insert("settings".to_string(), Value::Object(settings));
}
}
fn normalize_legacy_review_entry(value: &mut Value) {
if event_type(value) != Some("entered_review_mode") {
return;
}
let Some(payload) = payload_object_mut(value) else {
return;
};
if payload.contains_key("target") {
return;
}
let Some(prompt) = payload.get("prompt").and_then(Value::as_str) else {
return;
};
payload.insert(
"target".to_string(),
serde_json::json!({
"type": "custom",
"instructions": prompt,
}),
);
}
fn normalize_legacy_rate_limit_resets(value: &mut Value) {
if event_type(value) != Some("token_count") {
return;
}
let Some(rate_limits) = payload_object_mut(value)
.and_then(|payload| payload.get_mut("rate_limits"))
.and_then(Value::as_object_mut)
else {
return;
};
for window_name in ["primary", "secondary"] {
let Some(window) = rate_limits
.get_mut(window_name)
.and_then(Value::as_object_mut)
else {
continue;
};
let Some(resets_at) = window.get("resets_at").and_then(Value::as_str) else {
continue;
};
let Ok(resets_at) = DateTime::parse_from_rfc3339(resets_at) else {
continue;
};
window.insert("resets_at".to_string(), Value::from(resets_at.timestamp()));
}
}
fn normalize_legacy_sandbox_policy(value: &mut Value) {
if rollout_type(value) != Some("turn_context") {
return;
}
let Some(sandbox_policy) = payload_object_mut(value)
.and_then(|payload| payload.get_mut("sandbox_policy"))
.and_then(Value::as_object_mut)
else {
return;
};
if sandbox_policy.contains_key("type") {
return;
}
if let Some(mode) = sandbox_policy.get("mode").cloned() {
sandbox_policy.insert("type".to_string(), mode);
}
}
fn normalize_legacy_command_cwd(value: &mut Value) -> Result<(), String> {
if !matches!(
event_type(value),
Some("exec_command_begin" | "exec_command_end")
) {
return Ok(());
}
let Some(payload) = payload_object_mut(value) else {
return Ok(());
};
let Some(cwd) = payload.get("cwd").and_then(Value::as_str) else {
return Ok(());
};
if cwd.starts_with("file:") {
return Ok(());
}
let cwd = PathUri::try_from(LegacyAppPathString::from_string(cwd))
.map_err(|error| format!("invalid legacy command cwd: {error}"))?;
payload.insert(
"cwd".to_string(),
serde_json::to_value(cwd).map_err(|error| error.to_string())?,
);
Ok(())
}
fn rollout_type(value: &Value) -> Option<&str> {
value.get("type").and_then(Value::as_str)
}
fn event_type(value: &Value) -> Option<&str> {
if rollout_type(value) != Some("event_msg") {
return None;
}
value
.get("payload")
.and_then(Value::as_object)
.and_then(|payload| payload.get("type"))
.and_then(Value::as_str)
}
fn payload_object_mut(value: &mut Value) -> Option<&mut Map<String, Value>> {
value.get_mut("payload").and_then(Value::as_object_mut)
}
#[cfg(test)]
#[path = "line_parser_tests.rs"]
mod tests;
@@ -0,0 +1,228 @@
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use pretty_assertions::assert_eq;
use serde_json::json;
use super::parse_legacy_rollout_line;
fn line(payload_type: &str, payload: serde_json::Value) -> Vec<u8> {
serde_json::to_vec(&json!({
"timestamp": "2025-01-03T12:00:00Z",
"type": payload_type,
"payload": payload,
}))
.expect("serialize fixture")
}
#[test]
fn parses_legacy_numeric_event_payloads_through_value() {
let bytes = line(
"event_msg",
json!({
"type": "token_count",
"info": null,
"rate_limits": {
"primary": {
"used_percent": 2,
"window_minutes": 300,
"resets_at": 1_770_414_841,
},
"secondary": {
"used_percent": 12,
"window_minutes": 10_080,
"resets_at": 1_770_698_702,
},
"credits": {
"has_credits": false,
"unlimited": false,
"balance": null,
},
"plan_type": null,
},
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy token count")
.expect("keep legacy token count");
assert!(matches!(
parsed.item,
RolloutItem::EventMsg(EventMsg::TokenCount(_))
));
}
#[test]
fn normalizes_legacy_rate_limit_reset_timestamps() {
let bytes = line(
"event_msg",
json!({
"type": "token_count",
"info": null,
"rate_limits": {
"primary": {
"used_percent": 2,
"window_minutes": 300,
"resets_at": "2025-10-19T08:51:37.876641+00:00",
},
"secondary": null,
"credits": null,
"plan_type": null,
},
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy reset timestamp")
.expect("keep legacy token count");
assert!(matches!(
parsed.item,
RolloutItem::EventMsg(EventMsg::TokenCount(_))
));
}
#[test]
fn normalizes_legacy_turn_context_collaboration_mode() {
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
let bytes = line(
"turn_context",
json!({
"cwd": cwd,
"approval_policy": "never",
"sandbox_policy": {"type": "danger-full-access"},
"model": "gpt-test",
"personality": null,
"collaboration_mode": {
"mode": "plan",
"model": "gpt-test",
"reasoning_effort": null,
"developer_instructions": null,
},
"effort": null,
"summary": "auto",
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy turn context")
.expect("keep legacy turn context");
let RolloutItem::TurnContext(context) = parsed.item else {
panic!("expected turn context");
};
assert_eq!(
context
.collaboration_mode
.expect("collaboration mode")
.model(),
"gpt-test"
);
}
#[test]
fn normalizes_legacy_turn_context_sandbox_policy() {
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
let bytes = line(
"turn_context",
json!({
"cwd": cwd,
"approval_policy": "never",
"sandbox_policy": {"mode": "danger-full-access"},
"model": "gpt-test",
"personality": null,
"effort": null,
"summary": "auto",
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy sandbox policy")
.expect("keep legacy turn context");
assert!(matches!(parsed.item, RolloutItem::TurnContext(_)));
}
#[test]
fn normalizes_legacy_review_entry_prompt() {
let bytes = line(
"event_msg",
json!({
"type": "entered_review_mode",
"prompt": "review these changes",
"user_facing_hint": "Review requested.",
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy review entry")
.expect("keep legacy review entry");
assert!(matches!(
parsed.item,
RolloutItem::EventMsg(EventMsg::EnteredReviewMode(_))
));
}
#[test]
fn normalizes_legacy_plain_command_cwd() {
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
let bytes = line(
"event_msg",
json!({
"type": "exec_command_end",
"call_id": "call-1",
"turn_id": "turn-1",
"command": ["echo", "ok"],
"cwd": cwd,
"parsed_cmd": [],
"source": "agent",
"stdout": "",
"stderr": "",
"aggregated_output": "",
"exit_code": 0,
"duration": {"secs": 0, "nanos": 0},
"formatted_output": "",
"status": "completed",
}),
);
let parsed = parse_legacy_rollout_line(&bytes)
.expect("parse legacy command")
.expect("keep legacy command");
assert!(matches!(
parsed.item,
RolloutItem::EventMsg(EventMsg::ExecCommandEnd(_))
));
}
#[test]
fn skips_only_known_retired_events() {
for event_type in [
"guardian_assessment",
"thread_name_updated",
"undo_completed",
] {
let bytes = line("event_msg", json!({"type": event_type}));
assert!(
parse_legacy_rollout_line(&bytes)
.expect("inspect retired event")
.is_none()
);
}
let unknown = line("event_msg", json!({"type": "unknown_legacy_event"}));
assert!(parse_legacy_rollout_line(&unknown).is_err());
}
#[test]
fn skips_legacy_ghost_snapshots() {
let ghost_snapshot = line(
"response_item",
json!({
"type": "ghost_snapshot",
"ghost_commit": {"id": "legacy"},
}),
);
assert!(
parse_legacy_rollout_line(&ghost_snapshot)
.expect("inspect ghost snapshot")
.is_none()
);
}
@@ -0,0 +1,193 @@
//! Filesystem helpers for safely publishing migrated rollouts.
//!
//! This module owns temporary rollout paths, durable `.pending` journals, compressed rollout
//! staging, cleanup, and parent-directory syncs. The migration orchestrator decides when to call
//! these helpers; this module keeps the low-level file operations in one place.
//!
//! A migration can crash between publishing JSONL and finishing SQLite metadata. The journal is
//! the durable handoff between those steps, so cleanup must only remove it once the paginated
//! rollout and SQLite state are both complete.
use std::collections::HashSet;
use std::fs::Permissions;
use std::io;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::time::SystemTime;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use codex_protocol::ThreadId;
use super::migration_error;
use crate::ThreadStoreResult;
const MIGRATION_JOURNAL_DIRECTORY: &str = "rollout-migrations";
pub(super) fn migration_journal_path(codex_home: &Path, thread_id: ThreadId) -> PathBuf {
codex_home
.join(MIGRATION_JOURNAL_DIRECTORY)
.join(format!("{thread_id}.pending"))
}
pub(super) async fn pending_migration_thread_ids(
codex_home: &Path,
) -> ThreadStoreResult<HashSet<ThreadId>> {
let mut entries = match tokio::fs::read_dir(codex_home.join(MIGRATION_JOURNAL_DIRECTORY)).await
{
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(HashSet::new()),
Err(error) => return Err(migration_error(error)),
};
let mut thread_ids = HashSet::new();
while let Some(entry) = entries.next_entry().await.map_err(migration_error)? {
if !entry.file_type().await.map_err(migration_error)?.is_file() {
continue;
}
let filename = entry.file_name();
let Some(thread_id) = filename
.to_str()
.and_then(|filename| filename.strip_suffix(".pending"))
.and_then(|thread_id| ThreadId::from_string(thread_id).ok())
else {
continue;
};
thread_ids.insert(thread_id);
}
Ok(thread_ids)
}
pub(super) fn staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
staged_path(rollout_path, "paginated")
}
pub(super) fn decompressed_staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
staged_path(rollout_path, "decompressed")
}
pub(super) fn compressed_staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
staged_path(rollout_path, "paginated.zst")
}
fn staged_path(rollout_path: &Path, suffix: &str) -> ThreadStoreResult<PathBuf> {
let filename = rollout_path
.file_name()
.and_then(|filename| filename.to_str())
.ok_or_else(|| migration_error("rollout path has no valid filename"))?;
Ok(rollout_path.with_file_name(format!(".{filename}.{suffix}.tmp")))
}
pub(super) async fn decompress_rollout_to_path(
compressed_path: &Path,
plain_path: &Path,
) -> ThreadStoreResult<()> {
let compressed_path = compressed_path.to_path_buf();
let plain_path = plain_path.to_path_buf();
tokio::task::spawn_blocking(move || -> io::Result<()> {
let input = std::fs::File::open(compressed_path)?;
let mut decoder = zstd::stream::read::Decoder::new(input)?;
let mut options = std::fs::OpenOptions::new();
options.create(true).truncate(true).write(true);
#[cfg(unix)]
options.mode(0o600);
let mut output = options.open(plain_path)?;
#[cfg(unix)]
output.set_permissions(Permissions::from_mode(0o600))?;
io::copy(&mut decoder, &mut output)?;
output.flush()
})
.await
.map_err(migration_error)?
.map_err(migration_error)
}
pub(super) async fn compress_rollout_to_path(
plain_path: &Path,
compressed_path: &Path,
permissions: Permissions,
modified_at: Option<SystemTime>,
) -> ThreadStoreResult<()> {
let plain_path = plain_path.to_path_buf();
let compressed_path = compressed_path.to_path_buf();
tokio::task::spawn_blocking(move || -> io::Result<()> {
let mut input = std::fs::File::open(plain_path)?;
let output = std::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&compressed_path)?;
let mut encoder = zstd::stream::write::Encoder::new(output, 3)?;
io::copy(&mut input, &mut encoder)?;
let output = encoder.finish()?;
output.set_permissions(permissions)?;
if let Some(modified_at) = modified_at {
output.set_times(std::fs::FileTimes::new().set_modified(modified_at))?;
}
output.sync_all()?;
let input = std::fs::File::open(compressed_path)?;
let mut decoder = zstd::stream::read::Decoder::new(input)?;
io::copy(&mut decoder, &mut io::sink())?;
Ok(())
})
.await
.map_err(migration_error)?
.map_err(migration_error)
}
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
tokio::fs::remove_file(path)
.await
.map_err(migration_error)?;
}
Ok(())
}
pub(super) async fn write_migration_journal(path: &Path) -> ThreadStoreResult<()> {
let parent = path
.parent()
.ok_or_else(|| migration_error("rollout migration journal has no parent directory"))?;
let parent_already_exists = tokio::fs::try_exists(parent)
.await
.map_err(migration_error)?;
tokio::fs::create_dir_all(parent)
.await
.map_err(migration_error)?;
if !parent_already_exists {
sync_parent_directory(parent).await?;
}
let file = tokio::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(path)
.await
.map_err(migration_error)?;
file.sync_all().await.map_err(migration_error)?;
drop(file);
sync_parent_directory(path).await
}
pub(super) async fn sync_parent_directory(path: &Path) -> ThreadStoreResult<()> {
#[cfg(unix)]
{
let parent = path
.parent()
.ok_or_else(|| migration_error("rollout path has no parent directory"))?
.to_path_buf();
tokio::task::spawn_blocking(move || std::fs::File::open(parent)?.sync_all())
.await
.map_err(migration_error)?
.map_err(migration_error)?;
}
#[cfg(not(unix))]
{
let _ = path;
}
Ok(())
}