mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Add legacy rollout migration to paginated history (#37175)
## What changed - Add `LocalThreadStore::migrate_rollouts` with dry-run and apply modes, optional thread selection, throughput limiting, and per-rollout outcomes. - Canonicalize legacy JSONL records into paginated history while preserving model-visible conversation context, including compressed rollouts and copied fork history. - Publish replacements atomically and journal migrations so interrupted SQLite projections can be recovered. Coordinate migration with rollout compression and skip rollouts with active writers. - Make paginated history a one-way SQLite promotion so stale metadata cannot downgrade a migrated thread. ## Testing - Cover legacy record normalization, dry runs, idempotency, malformed input, compressed rollouts, active writers, interrupted migration recovery, and a cold app-server resume after migration. GitOrigin-RevId: b9991b659f28ebb52da39ce62e8b4e0bae2ba2bb
This commit is contained in:
1 parent
e3465b48ad
commit
6bb6e9045f
18 files changed
+3115
-5
No files matched your search
@@ -0,0 +1,505 @@
|
||||
//! Replays normalized legacy rollout records into canonical paginated JSONL.
|
||||
//!
|
||||
//! `line_parser` makes old JSON shapes parseable, `legacy_event` converts obsolete completion
|
||||
//! events into modern turn items. This module assigns stable ordinals, keeps `SessionMeta` at
|
||||
//! ordinal zero, and emits the surviving history selected by the caller.
|
||||
//!
|
||||
//! The goal is to preserve the model-visible conversation, not to preserve every legacy record
|
||||
//! byte-for-byte. Filesystem publishing and SQLite projection intentionally live outside this
|
||||
//! module.
|
||||
|
||||
use chrono::DateTime;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::ReasoningItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::parse_hook_prompt_message;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use std::collections::HashSet;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::io::BufWriter;
|
||||
|
||||
use super::legacy_event;
|
||||
use super::migration_error;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveTurn {
|
||||
id: String,
|
||||
explicit: bool,
|
||||
saw_user: bool,
|
||||
}
|
||||
|
||||
enum ReasoningTextKind {
|
||||
Summary,
|
||||
Raw,
|
||||
}
|
||||
|
||||
pub(super) struct LegacyRolloutCanonicalizer {
|
||||
thread_id: ThreadId,
|
||||
next_ordinal: u64,
|
||||
next_item_index: u64,
|
||||
output_byte_offset: u64,
|
||||
bytes_written: u64,
|
||||
source_line_index: u64,
|
||||
active_turn: Option<ActiveTurn>,
|
||||
known_turn_ids: HashSet<String>,
|
||||
reasoning: Option<ReasoningItem>,
|
||||
saw_source_session_meta: bool,
|
||||
}
|
||||
|
||||
impl LegacyRolloutCanonicalizer {
|
||||
pub(super) fn new(thread_id: ThreadId) -> Self {
|
||||
Self {
|
||||
thread_id,
|
||||
next_ordinal: 0,
|
||||
next_item_index: 1,
|
||||
output_byte_offset: 0,
|
||||
bytes_written: 0,
|
||||
source_line_index: 0,
|
||||
active_turn: None,
|
||||
known_turn_ids: HashSet::new(),
|
||||
reasoning: None,
|
||||
saw_source_session_meta: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn output_byte_offset(&self) -> u64 {
|
||||
self.output_byte_offset
|
||||
}
|
||||
|
||||
pub(super) fn next_ordinal(&self) -> u64 {
|
||||
self.next_ordinal
|
||||
}
|
||||
|
||||
pub(super) async fn write_head_session_meta(
|
||||
&mut self,
|
||||
line: RolloutLine,
|
||||
writer: &mut BufWriter<File>,
|
||||
) -> ThreadStoreResult<u64> {
|
||||
let timestamp = line.timestamp;
|
||||
let RolloutItem::SessionMeta(mut metadata) = line.item else {
|
||||
return Err(migration_error("canonical session metadata is missing"));
|
||||
};
|
||||
if metadata.meta.id != self.thread_id {
|
||||
return Err(migration_error("rollout metadata thread id changed"));
|
||||
}
|
||||
metadata.meta.history_mode = ThreadHistoryMode::Paginated;
|
||||
metadata.meta.history_base = None;
|
||||
metadata.meta.subagent_history_start_ordinal = None;
|
||||
|
||||
let bytes_before = self.bytes_written;
|
||||
self.write_item(writer, ×tamp, RolloutItem::SessionMeta(metadata))
|
||||
.await?;
|
||||
Ok(self.bytes_written - bytes_before)
|
||||
}
|
||||
|
||||
pub(super) async fn process_line(
|
||||
&mut self,
|
||||
line: RolloutLine,
|
||||
writer: &mut BufWriter<File>,
|
||||
) -> ThreadStoreResult<u64> {
|
||||
let source_index = self.source_line_index;
|
||||
self.source_line_index = self
|
||||
.source_line_index
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| migration_error("legacy rollout line index overflow"))?;
|
||||
let timestamp = line.timestamp;
|
||||
let bytes_before = self.bytes_written;
|
||||
match line.item {
|
||||
RolloutItem::SessionMeta(metadata) => {
|
||||
if !self.saw_source_session_meta {
|
||||
self.saw_source_session_meta = true;
|
||||
return Ok(0);
|
||||
}
|
||||
self.write_item(writer, ×tamp, RolloutItem::SessionMeta(metadata))
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::ResponseItem(ResponseItem::Other) => {
|
||||
return Err(migration_error(
|
||||
"legacy rollout contains an unsupported response item",
|
||||
));
|
||||
}
|
||||
RolloutItem::ResponseItem(response) => {
|
||||
let hook = match &response {
|
||||
ResponseItem::Message {
|
||||
role, content, id, ..
|
||||
} if role == "user" => parse_hook_prompt_message(id.as_deref(), content),
|
||||
_ => None,
|
||||
};
|
||||
self.write_item(writer, ×tamp, RolloutItem::ResponseItem(response))
|
||||
.await?;
|
||||
if let Some(hook) = hook {
|
||||
self.ensure_turn(writer, ×tamp, source_index).await?;
|
||||
self.reasoning = None;
|
||||
self.write_completed_item(writer, ×tamp, TurnItem::HookPrompt(hook))
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(_)) => {
|
||||
return Err(migration_error(
|
||||
"rollback marker reached canonical writer without a rollback plan",
|
||||
));
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => {
|
||||
self.finish_implicit_turn(writer, ×tamp).await?;
|
||||
self.known_turn_ids.insert(event.turn_id.clone());
|
||||
self.active_turn = Some(ActiveTurn {
|
||||
id: event.turn_id.clone(),
|
||||
explicit: true,
|
||||
saw_user: false,
|
||||
});
|
||||
self.reasoning = None;
|
||||
self.write_item(
|
||||
writer,
|
||||
×tamp,
|
||||
RolloutItem::EventMsg(EventMsg::TurnStarted(event)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => {
|
||||
self.reasoning = None;
|
||||
if self
|
||||
.active_turn
|
||||
.as_ref()
|
||||
.is_some_and(|turn| turn.id == event.turn_id)
|
||||
{
|
||||
self.active_turn = None;
|
||||
}
|
||||
self.write_item(
|
||||
writer,
|
||||
×tamp,
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(event)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::TurnAborted(mut event)) => {
|
||||
if event.turn_id.is_none() {
|
||||
event.turn_id = self.active_turn.as_ref().map(|turn| turn.id.clone());
|
||||
}
|
||||
if self
|
||||
.active_turn
|
||||
.as_ref()
|
||||
.is_some_and(|turn| event.turn_id.as_deref() == Some(turn.id.as_str()))
|
||||
{
|
||||
self.active_turn = None;
|
||||
}
|
||||
self.reasoning = None;
|
||||
self.write_item(
|
||||
writer,
|
||||
×tamp,
|
||||
RolloutItem::EventMsg(EventMsg::TurnAborted(event)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::UserMessage(event)) => {
|
||||
if self
|
||||
.active_turn
|
||||
.as_ref()
|
||||
.is_some_and(|turn| !turn.explicit && turn.saw_user)
|
||||
{
|
||||
self.finish_implicit_turn(writer, ×tamp).await?;
|
||||
}
|
||||
self.ensure_turn(writer, ×tamp, source_index).await?;
|
||||
let item = legacy_event::user_message_item(event, &mut || self.next_item_id())?;
|
||||
if let Some(turn) = self.active_turn.as_mut() {
|
||||
turn.saw_user = true;
|
||||
}
|
||||
self.reasoning = None;
|
||||
self.write_completed_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::AgentReasoning(event)) => {
|
||||
self.write_reasoning(
|
||||
writer,
|
||||
×tamp,
|
||||
source_index,
|
||||
event.text,
|
||||
ReasoningTextKind::Summary,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::AgentReasoningRawContent(event)) => {
|
||||
self.write_reasoning(
|
||||
writer,
|
||||
×tamp,
|
||||
source_index,
|
||||
event.text,
|
||||
ReasoningTextKind::Raw,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(mut event)) => {
|
||||
event.thread_id = self.thread_id;
|
||||
self.reasoning = None;
|
||||
self.write_item(
|
||||
writer,
|
||||
×tamp,
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(event)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RolloutItem::EventMsg(event) => {
|
||||
if let Some((item, turn_id)) =
|
||||
legacy_event::completed_item(&event, &mut || self.next_item_id())?
|
||||
{
|
||||
match turn_id {
|
||||
Some(turn_id)
|
||||
if self
|
||||
.active_turn
|
||||
.as_ref()
|
||||
.is_some_and(|turn| turn.id.as_str() != turn_id.as_str()) =>
|
||||
{
|
||||
self.write_completed_item_to_turn(writer, ×tamp, turn_id, item)
|
||||
.await?;
|
||||
}
|
||||
Some(turn_id) => {
|
||||
if self.active_turn.is_none()
|
||||
&& self.known_turn_ids.contains(turn_id.as_str())
|
||||
{
|
||||
self.reasoning = None;
|
||||
self.write_completed_item_to_turn(
|
||||
writer, ×tamp, turn_id, item,
|
||||
)
|
||||
.await?;
|
||||
} else if self.active_turn.is_none() {
|
||||
self.start_implicit_turn(writer, ×tamp, turn_id)
|
||||
.await?;
|
||||
self.reasoning = None;
|
||||
self.write_completed_item(writer, ×tamp, item).await?;
|
||||
} else {
|
||||
self.reasoning = None;
|
||||
self.write_completed_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
self.ensure_turn(writer, ×tamp, source_index).await?;
|
||||
self.reasoning = None;
|
||||
self.write_completed_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let item = RolloutItem::EventMsg(event);
|
||||
if codex_rollout::is_persisted_rollout_item(&item, ThreadHistoryMode::Paginated)
|
||||
{
|
||||
self.write_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
item @ RolloutItem::InterAgentCommunication(_) => {
|
||||
self.write_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
item @ RolloutItem::Compacted(_) => {
|
||||
self.write_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
item @ (RolloutItem::InterAgentCommunicationMetadata { .. }
|
||||
| RolloutItem::TurnContext(_)
|
||||
| RolloutItem::WorldState(_)) => {
|
||||
self.write_item(writer, ×tamp, item).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(self.bytes_written - bytes_before)
|
||||
}
|
||||
|
||||
pub(super) async fn finish(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
) -> ThreadStoreResult<u64> {
|
||||
let bytes_before = self.bytes_written;
|
||||
self.finish_implicit_turn(writer, timestamp).await?;
|
||||
Ok(self.bytes_written - bytes_before)
|
||||
}
|
||||
|
||||
async fn ensure_turn(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
source_index: u64,
|
||||
) -> ThreadStoreResult<()> {
|
||||
if self.active_turn.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let turn_id = format!("rollout-{source_index}");
|
||||
self.start_implicit_turn(writer, timestamp, turn_id).await
|
||||
}
|
||||
|
||||
async fn start_implicit_turn(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
turn_id: String,
|
||||
) -> ThreadStoreResult<()> {
|
||||
self.active_turn = Some(ActiveTurn {
|
||||
id: turn_id.clone(),
|
||||
explicit: false,
|
||||
saw_user: false,
|
||||
});
|
||||
self.known_turn_ids.insert(turn_id.clone());
|
||||
self.reasoning = None;
|
||||
let started_at = DateTime::parse_from_rfc3339(timestamp)
|
||||
.map_err(migration_error)?
|
||||
.timestamp();
|
||||
self.write_item(
|
||||
writer,
|
||||
timestamp,
|
||||
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_id,
|
||||
trace_id: None,
|
||||
started_at: Some(started_at),
|
||||
model_context_window: None,
|
||||
collaboration_mode_kind: Default::default(),
|
||||
})),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn finish_implicit_turn(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let Some(turn) = self.active_turn.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
if turn.explicit {
|
||||
return Ok(());
|
||||
}
|
||||
let turn_id = turn.id.clone();
|
||||
self.active_turn = None;
|
||||
self.reasoning = None;
|
||||
self.write_item(
|
||||
writer,
|
||||
timestamp,
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id,
|
||||
last_agent_message: None,
|
||||
error: None,
|
||||
started_at: None,
|
||||
completed_at: Some(
|
||||
DateTime::parse_from_rfc3339(timestamp)
|
||||
.map_err(migration_error)?
|
||||
.timestamp(),
|
||||
),
|
||||
duration_ms: None,
|
||||
time_to_first_token_ms: None,
|
||||
})),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_completed_item(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
item: TurnItem,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let turn_id = self
|
||||
.active_turn
|
||||
.as_ref()
|
||||
.map(|turn| turn.id.clone())
|
||||
.ok_or_else(|| migration_error("completed rollout item has no active turn"))?;
|
||||
self.write_completed_item_to_turn(writer, timestamp, turn_id, item)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_completed_item_to_turn(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
turn_id: String,
|
||||
item: TurnItem,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let completed_at_ms = DateTime::parse_from_rfc3339(timestamp)
|
||||
.map_err(migration_error)?
|
||||
.timestamp_millis();
|
||||
self.write_item(
|
||||
writer,
|
||||
timestamp,
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
thread_id: self.thread_id,
|
||||
turn_id,
|
||||
item,
|
||||
started_at_ms: None,
|
||||
completed_at_ms,
|
||||
})),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_reasoning(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
source_index: u64,
|
||||
text: String,
|
||||
kind: ReasoningTextKind,
|
||||
) -> ThreadStoreResult<()> {
|
||||
if text.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.ensure_turn(writer, timestamp, source_index).await?;
|
||||
let mut item = match self.reasoning.take() {
|
||||
Some(item) => item,
|
||||
None => ReasoningItem {
|
||||
id: self.next_item_id()?,
|
||||
summary_text: Vec::new(),
|
||||
raw_content: Vec::new(),
|
||||
},
|
||||
};
|
||||
match kind {
|
||||
ReasoningTextKind::Summary => item.summary_text.push(text),
|
||||
ReasoningTextKind::Raw => item.raw_content.push(text),
|
||||
}
|
||||
self.reasoning = Some(item.clone());
|
||||
self.write_completed_item(writer, timestamp, TurnItem::Reasoning(item))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_item(
|
||||
&mut self,
|
||||
writer: &mut BufWriter<File>,
|
||||
timestamp: &str,
|
||||
item: RolloutItem,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let mut bytes = serde_json::to_vec(&RolloutLine {
|
||||
timestamp: timestamp.to_string(),
|
||||
ordinal: Some(self.next_ordinal),
|
||||
item,
|
||||
})
|
||||
.map_err(migration_error)?;
|
||||
bytes.push(b'\n');
|
||||
writer.write_all(&bytes).await.map_err(migration_error)?;
|
||||
let byte_count = u64::try_from(bytes.len())
|
||||
.map_err(|_| migration_error("rollout record exceeds addressable size"))?;
|
||||
self.output_byte_offset = self
|
||||
.output_byte_offset
|
||||
.checked_add(byte_count)
|
||||
.ok_or_else(|| migration_error("paginated rollout byte offset overflow"))?;
|
||||
self.bytes_written = self
|
||||
.bytes_written
|
||||
.checked_add(byte_count)
|
||||
.ok_or_else(|| migration_error("paginated rollout byte count overflow"))?;
|
||||
self.next_ordinal = self
|
||||
.next_ordinal
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| migration_error("paginated rollout ordinal overflow"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_item_id(&mut self) -> ThreadStoreResult<String> {
|
||||
let item_id = format!("item-{}", self.next_item_index);
|
||||
self.next_item_index = self
|
||||
.next_item_index
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| migration_error("legacy rollout item id overflow"))?;
|
||||
Ok(item_id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
//! Converts legacy persisted events into modern paginated turn items.
|
||||
//!
|
||||
//! Legacy rollouts persisted events like `ExecCommandEnd`, `PatchApplyEnd`, and
|
||||
//! `McpToolCallEnd`. Paginated history instead persists `ItemCompleted(item)` records containing
|
||||
//! canonical `TurnItem`s.
|
||||
//!
|
||||
//! This module owns that old-event -> new-item mapping, including stable synthesized item IDs.
|
||||
//! It intentionally stays separate from live thread-history reduction because migration needs a
|
||||
//! frozen adapter for historical rollout payloads.
|
||||
|
||||
use codex_protocol::items::AgentMessageContent;
|
||||
use codex_protocol::items::AgentMessageItem;
|
||||
use codex_protocol::items::CommandExecutionItem;
|
||||
use codex_protocol::items::ContextCompactionItem;
|
||||
use codex_protocol::items::DynamicToolCallItem;
|
||||
use codex_protocol::items::DynamicToolCallStatus;
|
||||
use codex_protocol::items::EnteredReviewModeItem;
|
||||
use codex_protocol::items::ExitedReviewModeItem;
|
||||
use codex_protocol::items::FileChangeItem;
|
||||
use codex_protocol::items::ImageGenerationItem;
|
||||
use codex_protocol::items::McpToolCallError;
|
||||
use codex_protocol::items::McpToolCallItem;
|
||||
use codex_protocol::items::McpToolCallStatus;
|
||||
use codex_protocol::items::SubAgentActivityItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::items::WebSearchItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
pub(super) fn user_message_item(
|
||||
event: UserMessageEvent,
|
||||
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
|
||||
) -> ThreadStoreResult<TurnItem> {
|
||||
let mut content = Vec::new();
|
||||
if !event.message.trim().is_empty() {
|
||||
content.push(UserInput::Text {
|
||||
text: event.message,
|
||||
text_elements: event.text_elements,
|
||||
});
|
||||
}
|
||||
if let Some(images) = event.images {
|
||||
for (index, image_url) in images.into_iter().enumerate() {
|
||||
content.push(UserInput::Image {
|
||||
image_url,
|
||||
detail: event.image_details.get(index).copied().flatten(),
|
||||
});
|
||||
}
|
||||
}
|
||||
for (index, path) in event.local_images.into_iter().enumerate() {
|
||||
content.push(UserInput::LocalImage {
|
||||
path,
|
||||
detail: event.local_image_details.get(index).copied().flatten(),
|
||||
});
|
||||
}
|
||||
if let Some(audio) = event.audio {
|
||||
content.extend(
|
||||
audio
|
||||
.into_iter()
|
||||
.map(|audio_url| UserInput::Audio { audio_url }),
|
||||
);
|
||||
}
|
||||
content.extend(
|
||||
event
|
||||
.local_audio
|
||||
.into_iter()
|
||||
.map(|path| UserInput::LocalAudio { path }),
|
||||
);
|
||||
Ok(TurnItem::UserMessage(UserMessageItem {
|
||||
id: next_item_id()?,
|
||||
client_id: event.client_id,
|
||||
content,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Convert historical completion events into the canonical completed-item payloads
|
||||
/// that paginated rollouts persist. This deliberately remains a frozen migration
|
||||
/// adapter rather than sharing the live thread-history reducer: migration must
|
||||
/// preserve historical payloads while synthesizing stable item and turn IDs.
|
||||
pub(super) fn completed_item(
|
||||
event: &EventMsg,
|
||||
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
|
||||
) -> ThreadStoreResult<Option<(TurnItem, Option<String>)>> {
|
||||
let result = match event {
|
||||
EventMsg::AgentMessage(event) if !event.message.is_empty() => Some((
|
||||
TurnItem::AgentMessage(AgentMessageItem {
|
||||
id: next_item_id()?,
|
||||
content: vec![AgentMessageContent::Text {
|
||||
text: event.message.clone(),
|
||||
}],
|
||||
phase: event.phase.clone(),
|
||||
memory_citation: event.memory_citation.clone(),
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
EventMsg::PatchApplyEnd(event) => Some((
|
||||
TurnItem::FileChange(FileChangeItem {
|
||||
id: event.call_id.clone(),
|
||||
changes: event.changes.clone(),
|
||||
status: Some(event.status.clone()),
|
||||
auto_approved: None,
|
||||
stdout: (!event.stdout.is_empty()).then(|| event.stdout.clone()),
|
||||
stderr: (!event.stderr.is_empty()).then(|| event.stderr.clone()),
|
||||
}),
|
||||
(!event.turn_id.is_empty()).then(|| event.turn_id.clone()),
|
||||
)),
|
||||
EventMsg::McpToolCallEnd(event) => {
|
||||
let (result, error) = match &event.result {
|
||||
Ok(result) => (Some(result.clone()), None),
|
||||
Err(message) => (
|
||||
None,
|
||||
Some(McpToolCallError {
|
||||
message: message.clone(),
|
||||
}),
|
||||
),
|
||||
};
|
||||
Some((
|
||||
TurnItem::McpToolCall(McpToolCallItem {
|
||||
id: event.call_id.clone(),
|
||||
server: event.invocation.server.clone(),
|
||||
tool: event.invocation.tool.clone(),
|
||||
arguments: event
|
||||
.invocation
|
||||
.arguments
|
||||
.clone()
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
connector_id: event.connector_id.clone(),
|
||||
mcp_app_resource_uri: event.mcp_app_resource_uri.clone(),
|
||||
link_id: event.link_id.clone(),
|
||||
app_name: event.app_name.clone(),
|
||||
action_name: event.action_name.clone(),
|
||||
plugin_id: event.plugin_id.clone(),
|
||||
read_only_hint: event.read_only_hint,
|
||||
status: if event.is_success() {
|
||||
McpToolCallStatus::Completed
|
||||
} else {
|
||||
McpToolCallStatus::Failed
|
||||
},
|
||||
result,
|
||||
error,
|
||||
duration: Some(event.duration),
|
||||
}),
|
||||
None,
|
||||
))
|
||||
}
|
||||
EventMsg::WebSearchEnd(event) => Some((
|
||||
TurnItem::WebSearch(WebSearchItem {
|
||||
id: event.call_id.clone(),
|
||||
query: event.query.clone(),
|
||||
action: event.action.clone(),
|
||||
results: event.results.clone(),
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
EventMsg::ImageGenerationEnd(event) => Some((
|
||||
TurnItem::ImageGeneration(ImageGenerationItem {
|
||||
id: event.call_id.clone(),
|
||||
status: event.status.clone(),
|
||||
revised_prompt: event.revised_prompt.clone(),
|
||||
result: event.result.clone(),
|
||||
saved_path: event.saved_path.clone(),
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
EventMsg::ContextCompacted(_) => Some((
|
||||
TurnItem::ContextCompaction(ContextCompactionItem {
|
||||
id: next_item_id()?,
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
EventMsg::EnteredReviewMode(event) => Some((
|
||||
TurnItem::EnteredReviewMode(EnteredReviewModeItem {
|
||||
id: match event.item_id.clone() {
|
||||
Some(id) => id,
|
||||
None => next_item_id()?,
|
||||
},
|
||||
target: event.target.clone(),
|
||||
user_facing_hint: event
|
||||
.user_facing_hint
|
||||
.clone()
|
||||
.unwrap_or_else(|| "Review requested.".to_string()),
|
||||
}),
|
||||
event.turn_id.clone(),
|
||||
)),
|
||||
EventMsg::ExitedReviewMode(event) => Some((
|
||||
TurnItem::ExitedReviewMode(ExitedReviewModeItem {
|
||||
id: match event.item_id.clone() {
|
||||
Some(id) => id,
|
||||
None => next_item_id()?,
|
||||
},
|
||||
review_output: event.review_output.clone(),
|
||||
}),
|
||||
event.turn_id.clone(),
|
||||
)),
|
||||
EventMsg::SubAgentActivity(event) => Some((
|
||||
TurnItem::SubAgentActivity(SubAgentActivityItem {
|
||||
id: event.event_id.clone(),
|
||||
kind: event.kind,
|
||||
agent_thread_id: event.agent_thread_id,
|
||||
agent_path: event.agent_path.clone(),
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
EventMsg::ExecCommandEnd(event) => Some((
|
||||
TurnItem::CommandExecution(CommandExecutionItem {
|
||||
id: event.call_id.clone(),
|
||||
plugin_id: event.plugin_id.clone(),
|
||||
script_path: event.script_path.clone(),
|
||||
process_id: event.process_id.clone(),
|
||||
command: event.command.clone(),
|
||||
cwd: event.cwd.clone(),
|
||||
parsed_cmd: event.parsed_cmd.clone(),
|
||||
source: event.source,
|
||||
interaction_input: event.interaction_input.clone(),
|
||||
status: event.status.clone().into(),
|
||||
stdout: (!event.stdout.is_empty()).then(|| event.stdout.clone()),
|
||||
stderr: (!event.stderr.is_empty()).then(|| event.stderr.clone()),
|
||||
aggregated_output: (!event.aggregated_output.is_empty())
|
||||
.then(|| event.aggregated_output.clone()),
|
||||
exit_code: Some(event.exit_code),
|
||||
duration: Some(event.duration),
|
||||
formatted_output: (!event.formatted_output.is_empty())
|
||||
.then(|| event.formatted_output.clone()),
|
||||
}),
|
||||
Some(event.turn_id.clone()),
|
||||
)),
|
||||
EventMsg::DynamicToolCallResponse(event) => Some((
|
||||
TurnItem::DynamicToolCall(DynamicToolCallItem {
|
||||
id: event.call_id.clone(),
|
||||
namespace: event.namespace.clone(),
|
||||
tool: event.tool.clone(),
|
||||
arguments: event.arguments.clone(),
|
||||
status: if event.success {
|
||||
DynamicToolCallStatus::Completed
|
||||
} else {
|
||||
DynamicToolCallStatus::Failed
|
||||
},
|
||||
content_items: Some(event.content_items.clone()),
|
||||
success: Some(event.success),
|
||||
error: event.error.clone(),
|
||||
duration: Some(event.duration),
|
||||
}),
|
||||
Some(event.turn_id.clone()),
|
||||
)),
|
||||
_ => None,
|
||||
};
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Makes historical rollout JSON parse like today's protocol types.
|
||||
//!
|
||||
//! Legacy rollouts span a bunch of old wire shapes: renamed fields, old sandbox policy layouts,
|
||||
//! string timestamps, legacy command cwd paths, retired events, and payloads that only deserialize
|
||||
//! correctly after going through `serde_json::Value`.
|
||||
//!
|
||||
//! This module only applies narrowly scoped compatibility rewrites for shapes we know existed in
|
||||
//! persisted rollouts. It does not decide turn boundaries, rollback behavior, or what gets written
|
||||
//! into paginated history.
|
||||
|
||||
use chrono::DateTime;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_utils_path_uri::LegacyAppPathString;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Parse one legacy rollout line after applying narrowly scoped compatibility
|
||||
/// rewrites for historical wire shapes that current protocol types no longer
|
||||
/// accept.
|
||||
pub(super) fn parse_legacy_rollout_line(bytes: &[u8]) -> Result<Option<RolloutLine>, String> {
|
||||
if bytes.iter().all(u8::is_ascii_whitespace) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Deserializing through Value is intentional. Some historical numeric
|
||||
// payloads fail serde_json's streaming enum path but deserialize correctly
|
||||
// once their tagged object shape has been materialized.
|
||||
let mut value = serde_json::from_slice::<Value>(bytes).map_err(|error| error.to_string())?;
|
||||
if should_skip_retired_record(&value) {
|
||||
return Ok(None);
|
||||
}
|
||||
normalize_legacy_turn_context(&mut value);
|
||||
normalize_legacy_sandbox_policy(&mut value);
|
||||
normalize_legacy_rate_limit_resets(&mut value);
|
||||
normalize_legacy_review_entry(&mut value);
|
||||
normalize_legacy_command_cwd(&mut value)?;
|
||||
serde_json::from_value(value)
|
||||
.map(Some)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn should_skip_retired_record(value: &Value) -> bool {
|
||||
matches!(
|
||||
event_type(value),
|
||||
Some("guardian_assessment" | "thread_name_updated" | "undo_completed")
|
||||
) || (rollout_type(value) == Some("response_item")
|
||||
&& value
|
||||
.get("payload")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|payload| payload.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
== Some("ghost_snapshot"))
|
||||
}
|
||||
|
||||
fn normalize_legacy_turn_context(value: &mut Value) {
|
||||
if rollout_type(value) != Some("turn_context") {
|
||||
return;
|
||||
}
|
||||
let Some(payload) = payload_object_mut(value) else {
|
||||
return;
|
||||
};
|
||||
let Some(collaboration_mode) = payload
|
||||
.get_mut("collaboration_mode")
|
||||
.and_then(Value::as_object_mut)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if collaboration_mode.contains_key("settings") {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut settings = Map::new();
|
||||
for key in ["model", "reasoning_effort", "developer_instructions"] {
|
||||
if let Some(value) = collaboration_mode.get(key) {
|
||||
settings.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
if !settings.is_empty() {
|
||||
collaboration_mode.insert("settings".to_string(), Value::Object(settings));
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_legacy_review_entry(value: &mut Value) {
|
||||
if event_type(value) != Some("entered_review_mode") {
|
||||
return;
|
||||
}
|
||||
let Some(payload) = payload_object_mut(value) else {
|
||||
return;
|
||||
};
|
||||
if payload.contains_key("target") {
|
||||
return;
|
||||
}
|
||||
let Some(prompt) = payload.get("prompt").and_then(Value::as_str) else {
|
||||
return;
|
||||
};
|
||||
payload.insert(
|
||||
"target".to_string(),
|
||||
serde_json::json!({
|
||||
"type": "custom",
|
||||
"instructions": prompt,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
fn normalize_legacy_rate_limit_resets(value: &mut Value) {
|
||||
if event_type(value) != Some("token_count") {
|
||||
return;
|
||||
}
|
||||
let Some(rate_limits) = payload_object_mut(value)
|
||||
.and_then(|payload| payload.get_mut("rate_limits"))
|
||||
.and_then(Value::as_object_mut)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
for window_name in ["primary", "secondary"] {
|
||||
let Some(window) = rate_limits
|
||||
.get_mut(window_name)
|
||||
.and_then(Value::as_object_mut)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(resets_at) = window.get("resets_at").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(resets_at) = DateTime::parse_from_rfc3339(resets_at) else {
|
||||
continue;
|
||||
};
|
||||
window.insert("resets_at".to_string(), Value::from(resets_at.timestamp()));
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_legacy_sandbox_policy(value: &mut Value) {
|
||||
if rollout_type(value) != Some("turn_context") {
|
||||
return;
|
||||
}
|
||||
let Some(sandbox_policy) = payload_object_mut(value)
|
||||
.and_then(|payload| payload.get_mut("sandbox_policy"))
|
||||
.and_then(Value::as_object_mut)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if sandbox_policy.contains_key("type") {
|
||||
return;
|
||||
}
|
||||
if let Some(mode) = sandbox_policy.get("mode").cloned() {
|
||||
sandbox_policy.insert("type".to_string(), mode);
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_legacy_command_cwd(value: &mut Value) -> Result<(), String> {
|
||||
if !matches!(
|
||||
event_type(value),
|
||||
Some("exec_command_begin" | "exec_command_end")
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(payload) = payload_object_mut(value) else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(cwd) = payload.get("cwd").and_then(Value::as_str) else {
|
||||
return Ok(());
|
||||
};
|
||||
if cwd.starts_with("file:") {
|
||||
return Ok(());
|
||||
}
|
||||
let cwd = PathUri::try_from(LegacyAppPathString::from_string(cwd))
|
||||
.map_err(|error| format!("invalid legacy command cwd: {error}"))?;
|
||||
payload.insert(
|
||||
"cwd".to_string(),
|
||||
serde_json::to_value(cwd).map_err(|error| error.to_string())?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rollout_type(value: &Value) -> Option<&str> {
|
||||
value.get("type").and_then(Value::as_str)
|
||||
}
|
||||
|
||||
fn event_type(value: &Value) -> Option<&str> {
|
||||
if rollout_type(value) != Some("event_msg") {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.get("payload")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|payload| payload.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
}
|
||||
|
||||
fn payload_object_mut(value: &mut Value) -> Option<&mut Map<String, Value>> {
|
||||
value.get_mut("payload").and_then(Value::as_object_mut)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "line_parser_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,228 @@
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
use super::parse_legacy_rollout_line;
|
||||
|
||||
fn line(payload_type: &str, payload: serde_json::Value) -> Vec<u8> {
|
||||
serde_json::to_vec(&json!({
|
||||
"timestamp": "2025-01-03T12:00:00Z",
|
||||
"type": payload_type,
|
||||
"payload": payload,
|
||||
}))
|
||||
.expect("serialize fixture")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_legacy_numeric_event_payloads_through_value() {
|
||||
let bytes = line(
|
||||
"event_msg",
|
||||
json!({
|
||||
"type": "token_count",
|
||||
"info": null,
|
||||
"rate_limits": {
|
||||
"primary": {
|
||||
"used_percent": 2,
|
||||
"window_minutes": 300,
|
||||
"resets_at": 1_770_414_841,
|
||||
},
|
||||
"secondary": {
|
||||
"used_percent": 12,
|
||||
"window_minutes": 10_080,
|
||||
"resets_at": 1_770_698_702,
|
||||
},
|
||||
"credits": {
|
||||
"has_credits": false,
|
||||
"unlimited": false,
|
||||
"balance": null,
|
||||
},
|
||||
"plan_type": null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy token count")
|
||||
.expect("keep legacy token count");
|
||||
assert!(matches!(
|
||||
parsed.item,
|
||||
RolloutItem::EventMsg(EventMsg::TokenCount(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_legacy_rate_limit_reset_timestamps() {
|
||||
let bytes = line(
|
||||
"event_msg",
|
||||
json!({
|
||||
"type": "token_count",
|
||||
"info": null,
|
||||
"rate_limits": {
|
||||
"primary": {
|
||||
"used_percent": 2,
|
||||
"window_minutes": 300,
|
||||
"resets_at": "2025-10-19T08:51:37.876641+00:00",
|
||||
},
|
||||
"secondary": null,
|
||||
"credits": null,
|
||||
"plan_type": null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy reset timestamp")
|
||||
.expect("keep legacy token count");
|
||||
assert!(matches!(
|
||||
parsed.item,
|
||||
RolloutItem::EventMsg(EventMsg::TokenCount(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_legacy_turn_context_collaboration_mode() {
|
||||
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
|
||||
let bytes = line(
|
||||
"turn_context",
|
||||
json!({
|
||||
"cwd": cwd,
|
||||
"approval_policy": "never",
|
||||
"sandbox_policy": {"type": "danger-full-access"},
|
||||
"model": "gpt-test",
|
||||
"personality": null,
|
||||
"collaboration_mode": {
|
||||
"mode": "plan",
|
||||
"model": "gpt-test",
|
||||
"reasoning_effort": null,
|
||||
"developer_instructions": null,
|
||||
},
|
||||
"effort": null,
|
||||
"summary": "auto",
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy turn context")
|
||||
.expect("keep legacy turn context");
|
||||
let RolloutItem::TurnContext(context) = parsed.item else {
|
||||
panic!("expected turn context");
|
||||
};
|
||||
assert_eq!(
|
||||
context
|
||||
.collaboration_mode
|
||||
.expect("collaboration mode")
|
||||
.model(),
|
||||
"gpt-test"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_legacy_turn_context_sandbox_policy() {
|
||||
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
|
||||
let bytes = line(
|
||||
"turn_context",
|
||||
json!({
|
||||
"cwd": cwd,
|
||||
"approval_policy": "never",
|
||||
"sandbox_policy": {"mode": "danger-full-access"},
|
||||
"model": "gpt-test",
|
||||
"personality": null,
|
||||
"effort": null,
|
||||
"summary": "auto",
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy sandbox policy")
|
||||
.expect("keep legacy turn context");
|
||||
assert!(matches!(parsed.item, RolloutItem::TurnContext(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_legacy_review_entry_prompt() {
|
||||
let bytes = line(
|
||||
"event_msg",
|
||||
json!({
|
||||
"type": "entered_review_mode",
|
||||
"prompt": "review these changes",
|
||||
"user_facing_hint": "Review requested.",
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy review entry")
|
||||
.expect("keep legacy review entry");
|
||||
assert!(matches!(
|
||||
parsed.item,
|
||||
RolloutItem::EventMsg(EventMsg::EnteredReviewMode(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_legacy_plain_command_cwd() {
|
||||
let cwd = std::env::temp_dir().to_string_lossy().into_owned();
|
||||
let bytes = line(
|
||||
"event_msg",
|
||||
json!({
|
||||
"type": "exec_command_end",
|
||||
"call_id": "call-1",
|
||||
"turn_id": "turn-1",
|
||||
"command": ["echo", "ok"],
|
||||
"cwd": cwd,
|
||||
"parsed_cmd": [],
|
||||
"source": "agent",
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"aggregated_output": "",
|
||||
"exit_code": 0,
|
||||
"duration": {"secs": 0, "nanos": 0},
|
||||
"formatted_output": "",
|
||||
"status": "completed",
|
||||
}),
|
||||
);
|
||||
|
||||
let parsed = parse_legacy_rollout_line(&bytes)
|
||||
.expect("parse legacy command")
|
||||
.expect("keep legacy command");
|
||||
assert!(matches!(
|
||||
parsed.item,
|
||||
RolloutItem::EventMsg(EventMsg::ExecCommandEnd(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_only_known_retired_events() {
|
||||
for event_type in [
|
||||
"guardian_assessment",
|
||||
"thread_name_updated",
|
||||
"undo_completed",
|
||||
] {
|
||||
let bytes = line("event_msg", json!({"type": event_type}));
|
||||
assert!(
|
||||
parse_legacy_rollout_line(&bytes)
|
||||
.expect("inspect retired event")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
let unknown = line("event_msg", json!({"type": "unknown_legacy_event"}));
|
||||
assert!(parse_legacy_rollout_line(&unknown).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_legacy_ghost_snapshots() {
|
||||
let ghost_snapshot = line(
|
||||
"response_item",
|
||||
json!({
|
||||
"type": "ghost_snapshot",
|
||||
"ghost_commit": {"id": "legacy"},
|
||||
}),
|
||||
);
|
||||
|
||||
assert!(
|
||||
parse_legacy_rollout_line(&ghost_snapshot)
|
||||
.expect("inspect ghost snapshot")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
//! Filesystem helpers for safely publishing migrated rollouts.
|
||||
//!
|
||||
//! This module owns temporary rollout paths, durable `.pending` journals, compressed rollout
|
||||
//! staging, cleanup, and parent-directory syncs. The migration orchestrator decides when to call
|
||||
//! these helpers; this module keeps the low-level file operations in one place.
|
||||
//!
|
||||
//! A migration can crash between publishing JSONL and finishing SQLite metadata. The journal is
|
||||
//! the durable handoff between those steps, so cleanup must only remove it once the paginated
|
||||
//! rollout and SQLite state are both complete.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fs::Permissions;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
use codex_protocol::ThreadId;
|
||||
|
||||
use super::migration_error;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
const MIGRATION_JOURNAL_DIRECTORY: &str = "rollout-migrations";
|
||||
|
||||
pub(super) fn migration_journal_path(codex_home: &Path, thread_id: ThreadId) -> PathBuf {
|
||||
codex_home
|
||||
.join(MIGRATION_JOURNAL_DIRECTORY)
|
||||
.join(format!("{thread_id}.pending"))
|
||||
}
|
||||
|
||||
pub(super) async fn pending_migration_thread_ids(
|
||||
codex_home: &Path,
|
||||
) -> ThreadStoreResult<HashSet<ThreadId>> {
|
||||
let mut entries = match tokio::fs::read_dir(codex_home.join(MIGRATION_JOURNAL_DIRECTORY)).await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(HashSet::new()),
|
||||
Err(error) => return Err(migration_error(error)),
|
||||
};
|
||||
let mut thread_ids = HashSet::new();
|
||||
while let Some(entry) = entries.next_entry().await.map_err(migration_error)? {
|
||||
if !entry.file_type().await.map_err(migration_error)?.is_file() {
|
||||
continue;
|
||||
}
|
||||
let filename = entry.file_name();
|
||||
let Some(thread_id) = filename
|
||||
.to_str()
|
||||
.and_then(|filename| filename.strip_suffix(".pending"))
|
||||
.and_then(|thread_id| ThreadId::from_string(thread_id).ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
thread_ids.insert(thread_id);
|
||||
}
|
||||
Ok(thread_ids)
|
||||
}
|
||||
|
||||
pub(super) fn staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
|
||||
staged_path(rollout_path, "paginated")
|
||||
}
|
||||
|
||||
pub(super) fn decompressed_staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
|
||||
staged_path(rollout_path, "decompressed")
|
||||
}
|
||||
|
||||
pub(super) fn compressed_staged_rollout_path(rollout_path: &Path) -> ThreadStoreResult<PathBuf> {
|
||||
staged_path(rollout_path, "paginated.zst")
|
||||
}
|
||||
|
||||
fn staged_path(rollout_path: &Path, suffix: &str) -> ThreadStoreResult<PathBuf> {
|
||||
let filename = rollout_path
|
||||
.file_name()
|
||||
.and_then(|filename| filename.to_str())
|
||||
.ok_or_else(|| migration_error("rollout path has no valid filename"))?;
|
||||
Ok(rollout_path.with_file_name(format!(".{filename}.{suffix}.tmp")))
|
||||
}
|
||||
|
||||
pub(super) async fn decompress_rollout_to_path(
|
||||
compressed_path: &Path,
|
||||
plain_path: &Path,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let compressed_path = compressed_path.to_path_buf();
|
||||
let plain_path = plain_path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || -> io::Result<()> {
|
||||
let input = std::fs::File::open(compressed_path)?;
|
||||
let mut decoder = zstd::stream::read::Decoder::new(input)?;
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
options.create(true).truncate(true).write(true);
|
||||
#[cfg(unix)]
|
||||
options.mode(0o600);
|
||||
let mut output = options.open(plain_path)?;
|
||||
#[cfg(unix)]
|
||||
output.set_permissions(Permissions::from_mode(0o600))?;
|
||||
io::copy(&mut decoder, &mut output)?;
|
||||
output.flush()
|
||||
})
|
||||
.await
|
||||
.map_err(migration_error)?
|
||||
.map_err(migration_error)
|
||||
}
|
||||
|
||||
pub(super) async fn compress_rollout_to_path(
|
||||
plain_path: &Path,
|
||||
compressed_path: &Path,
|
||||
permissions: Permissions,
|
||||
modified_at: Option<SystemTime>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let plain_path = plain_path.to_path_buf();
|
||||
let compressed_path = compressed_path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || -> io::Result<()> {
|
||||
let mut input = std::fs::File::open(plain_path)?;
|
||||
let output = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&compressed_path)?;
|
||||
let mut encoder = zstd::stream::write::Encoder::new(output, 3)?;
|
||||
io::copy(&mut input, &mut encoder)?;
|
||||
let output = encoder.finish()?;
|
||||
output.set_permissions(permissions)?;
|
||||
if let Some(modified_at) = modified_at {
|
||||
output.set_times(std::fs::FileTimes::new().set_modified(modified_at))?;
|
||||
}
|
||||
output.sync_all()?;
|
||||
|
||||
let input = std::fs::File::open(compressed_path)?;
|
||||
let mut decoder = zstd::stream::read::Decoder::new(input)?;
|
||||
io::copy(&mut decoder, &mut io::sink())?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(migration_error)?
|
||||
.map_err(migration_error)
|
||||
}
|
||||
|
||||
pub(super) async fn remove_file_if_present(path: &Path) -> ThreadStoreResult<()> {
|
||||
if tokio::fs::try_exists(path).await.map_err(migration_error)? {
|
||||
tokio::fs::remove_file(path)
|
||||
.await
|
||||
.map_err(migration_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn write_migration_journal(path: &Path) -> ThreadStoreResult<()> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| migration_error("rollout migration journal has no parent directory"))?;
|
||||
let parent_already_exists = tokio::fs::try_exists(parent)
|
||||
.await
|
||||
.map_err(migration_error)?;
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.map_err(migration_error)?;
|
||||
if !parent_already_exists {
|
||||
sync_parent_directory(parent).await?;
|
||||
}
|
||||
let file = tokio::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(path)
|
||||
.await
|
||||
.map_err(migration_error)?;
|
||||
file.sync_all().await.map_err(migration_error)?;
|
||||
drop(file);
|
||||
sync_parent_directory(path).await
|
||||
}
|
||||
|
||||
pub(super) async fn sync_parent_directory(path: &Path) -> ThreadStoreResult<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| migration_error("rollout path has no parent directory"))?
|
||||
.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || std::fs::File::open(parent)?.sync_all())
|
||||
.await
|
||||
.map_err(migration_error)?
|
||||
.map_err(migration_error)?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = path;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user