mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
feat(rollout): persist TurnItems for paginated thread rollouts (#30188)
## Description This PR makes new threads with `history_mode = "paginated"` persist `ItemCompleted(item: <turn_item>)` in their rollout JSONL file. Legacy threads keep persisting the existing legacy events. Because the format is selected per thread, a rollout is either legacy or paginated; we do not need to support mixed rollouts containing both representations. This PR depends on [#31473](https://github.com/openai/codex/pull/31473). ## Why Paginated thread history needs stable turn/item IDs and completed item snapshots so the later SQLite projector can materialize appended rollout JSONL without rebuilding the whole thread. Keeping the legacy persistence policy unchanged avoids changing historical rollouts or the readers that still consume them. ## What changed - Made rollout filtering history-mode aware. Paginated threads keep completed canonical `ItemCompleted` events and drop their redundant legacy projections; legacy threads keep the existing event set. - Made forks inherit the source thread history mode, so copied legacy history is never filtered as paginated. - Made paginated threads assign IDs to locally-created response items even when `Feature::ItemIds` is off, and reject streamed output items that arrive without server IDs. - Updated legacy turn replay, rollout list/search, and SQLite metadata extraction to understand completed canonical user-message items.
This commit is contained in:
1 parent
5892c7b69d
commit
2342b2c2a6
26 files changed
+497
-220
No files matched your search
@@ -1,16 +1,16 @@
|
||||
use crate::model::ThreadMetadata;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::TurnContextItem;
|
||||
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::strip_user_message_prefix;
|
||||
use codex_protocol::protocol::user_message_preview;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
const IMAGE_ONLY_USER_MESSAGE_PLACEHOLDER: &str = "[Image]";
|
||||
|
||||
/// Apply a rollout item to the metadata structure.
|
||||
pub fn apply_rollout_item(
|
||||
metadata: &mut ThreadMetadata,
|
||||
@@ -39,6 +39,11 @@ pub fn rollout_item_affects_thread_metadata(item: &RolloutItem) -> bool {
|
||||
RolloutItem::EventMsg(
|
||||
EventMsg::TokenCount(_) | EventMsg::UserMessage(_) | EventMsg::ThreadGoalUpdated(_),
|
||||
) => true,
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(event))
|
||||
if matches!(event.item, TurnItem::UserMessage(_)) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
RolloutItem::EventMsg(_)
|
||||
| RolloutItem::ResponseItem(_)
|
||||
| RolloutItem::InterAgentCommunication(_)
|
||||
@@ -96,16 +101,11 @@ fn apply_event_msg(metadata: &mut ThreadMetadata, event: &EventMsg) {
|
||||
}
|
||||
}
|
||||
EventMsg::UserMessage(user) => {
|
||||
let preview = user_message_preview(user);
|
||||
if metadata.first_user_message.is_none() {
|
||||
metadata.first_user_message = preview.clone();
|
||||
}
|
||||
set_preview_if_empty(metadata, preview);
|
||||
if metadata.title.is_empty() {
|
||||
let title = strip_user_message_prefix(user.message.as_str());
|
||||
if !title.is_empty() {
|
||||
metadata.title = title.to_string();
|
||||
}
|
||||
apply_user_message(metadata, user);
|
||||
}
|
||||
EventMsg::ItemCompleted(event) => {
|
||||
if let TurnItem::UserMessage(user) = &event.item {
|
||||
apply_user_message(metadata, &user.as_legacy_user_message_event());
|
||||
}
|
||||
}
|
||||
EventMsg::ThreadGoalUpdated(event) => {
|
||||
@@ -120,35 +120,26 @@ fn apply_event_msg(metadata: &mut ThreadMetadata, event: &EventMsg) {
|
||||
|
||||
fn apply_response_item(_metadata: &mut ThreadMetadata, _item: &ResponseItem) {}
|
||||
|
||||
fn apply_user_message(metadata: &mut ThreadMetadata, user: &UserMessageEvent) {
|
||||
let preview = user_message_preview(user);
|
||||
if metadata.first_user_message.is_none() {
|
||||
metadata.first_user_message = preview.clone();
|
||||
}
|
||||
set_preview_if_empty(metadata, preview);
|
||||
if metadata.title.is_empty() {
|
||||
let title = strip_user_message_prefix(user.message.as_str());
|
||||
if !title.is_empty() {
|
||||
metadata.title = title.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn set_preview_if_empty(metadata: &mut ThreadMetadata, preview: Option<String>) {
|
||||
if metadata.preview.is_none() {
|
||||
metadata.preview = preview;
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_user_message_prefix(text: &str) -> &str {
|
||||
match text.find(USER_MESSAGE_BEGIN) {
|
||||
Some(idx) => text[idx + USER_MESSAGE_BEGIN.len()..].trim(),
|
||||
None => text.trim(),
|
||||
}
|
||||
}
|
||||
|
||||
fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
|
||||
let message = strip_user_message_prefix(user.message.as_str());
|
||||
if !message.is_empty() {
|
||||
return Some(message.to_string());
|
||||
}
|
||||
if user
|
||||
.images
|
||||
.as_ref()
|
||||
.is_some_and(|images| !images.is_empty())
|
||||
|| !user.local_images.is_empty()
|
||||
{
|
||||
return Some(IMAGE_ONLY_USER_MESSAGE_PLACEHOLDER.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(crate) fn enum_to_string<T: Serialize>(value: &T) -> String {
|
||||
match serde_json::to_value(value) {
|
||||
Ok(Value::String(s)) => s,
|
||||
@@ -164,12 +155,15 @@ mod tests {
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
@@ -182,6 +176,7 @@ mod tests {
|
||||
use codex_protocol::protocol::TurnContextItem;
|
||||
use codex_protocol::protocol::USER_MESSAGE_BEGIN;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::path::PathBuf;
|
||||
@@ -229,6 +224,29 @@ mod tests {
|
||||
assert_eq!(metadata.title, "actual user request");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_user_message_items_set_title_and_first_user_message() {
|
||||
let mut metadata = metadata_for_test();
|
||||
let item = RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
thread_id: ThreadId::default(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
item: TurnItem::UserMessage(UserMessageItem::new(&[UserInput::Text {
|
||||
text: format!("{USER_MESSAGE_BEGIN} actual user request"),
|
||||
text_elements: Vec::new(),
|
||||
}])),
|
||||
completed_at_ms: 0,
|
||||
}));
|
||||
|
||||
apply_rollout_item(&mut metadata, &item, "test-provider");
|
||||
|
||||
assert_eq!(
|
||||
metadata.first_user_message.as_deref(),
|
||||
Some("actual user request")
|
||||
);
|
||||
assert_eq!(metadata.preview.as_deref(), Some("actual user request"));
|
||||
assert_eq!(metadata.title, "actual user request");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_msg_image_only_user_message_sets_image_placeholder_preview() {
|
||||
let mut metadata = metadata_for_test();
|
||||
@@ -243,14 +261,8 @@ mod tests {
|
||||
|
||||
apply_rollout_item(&mut metadata, &item, "test-provider");
|
||||
|
||||
assert_eq!(
|
||||
metadata.first_user_message.as_deref(),
|
||||
Some(super::IMAGE_ONLY_USER_MESSAGE_PLACEHOLDER)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.preview.as_deref(),
|
||||
Some(super::IMAGE_ONLY_USER_MESSAGE_PLACEHOLDER)
|
||||
);
|
||||
assert_eq!(metadata.first_user_message.as_deref(), Some("[Image]"));
|
||||
assert_eq!(metadata.preview.as_deref(), Some("[Image]"));
|
||||
assert_eq!(metadata.title, "");
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user