Persist realtime events in the thread timeline (#40508)

## Why

Clients need a bounded view of realtime conversations that preserves the order of speech, agent work, and turn lifecycle events without loading the full thread history.

## What changed

- Persist realtime session boundaries, transcript segments, and promoted agent items in paginated thread rollouts.
- Add the experimental `thread/timeline/list` API to page ordinary items, realtime items, and turn boundaries in canonical rollout order. Include `activeRealtimeSessionAtPageStart` so each page can be rendered independently.
- Emit typed `thread/realtime/item/started`, `thread/realtime/item/transcript/delta`, and `thread/realtime/item/completed` notifications around durable realtime items.
- Keep the existing `thread/items/list` response and legacy thread history behavior unchanged.

## Testing

- Cover mixed timeline pagination, realtime-session state at page boundaries, rollout replay, subagent history prefixes, transcript streaming, steering, and promoted agent artifacts.

GitOrigin-RevId: faa2c420192d63e060e8fb32446e3c797422656e
This commit is contained in:
guinness-oai authored and copyberry committed 2026-08-25 01:03:20 +00:00
1 parent 80e871309a
commit 41ca88e95c
80 files changed
+5140 -47

No files matched your search

@@ -1014,6 +1014,12 @@ client_request_definitions! {
serialization: thread_id(params.thread_id),
response: v2::ThreadRealtimeStopResponse,
},
#[experimental("thread/timeline/list")]
ThreadTimelineList => "thread/timeline/list" {
params: v2::ThreadTimelineListParams,
serialization: thread_id(params.thread_id),
response: v2::ThreadTimelineListResponse,
},
#[experimental("thread/realtime/listVoices")]
ThreadRealtimeListVoices => "thread/realtime/listVoices" {
params: v2::ThreadRealtimeListVoicesParams,
@@ -1917,6 +1923,12 @@ server_notification_definitions! {
ThreadRealtimeStarted => "thread/realtime/started" (v2::ThreadRealtimeStartedNotification),
#[experimental("thread/realtime/itemAdded")]
ThreadRealtimeItemAdded => "thread/realtime/itemAdded" (v2::ThreadRealtimeItemAddedNotification),
#[experimental("thread/realtime/item/started")]
ThreadRealtimeItemStarted => "thread/realtime/item/started" (v2::ThreadRealtimeItemStartedNotification),
#[experimental("thread/realtime/item/transcript/delta")]
ThreadRealtimeItemTranscriptDelta => "thread/realtime/item/transcript/delta" (v2::ThreadRealtimeItemTranscriptDeltaNotification),
#[experimental("thread/realtime/item/completed")]
ThreadRealtimeItemCompleted => "thread/realtime/item/completed" (v2::ThreadRealtimeItemCompletedNotification),
#[experimental("thread/realtime/transcript/delta")]
ThreadRealtimeTranscriptDelta => "thread/realtime/transcript/delta" (v2::ThreadRealtimeTranscriptDeltaNotification),
#[experimental("thread/realtime/transcript/done")]
@@ -401,6 +401,7 @@ impl ThreadHistoryBuilder {
| RolloutItem::InterAgentCommunicationMetadata { .. }
| RolloutItem::TurnContext(_)
| RolloutItem::WorldState(_)
| RolloutItem::RealtimeItem(_)
| RolloutItem::SecurityRiskScore(_)
| RolloutItem::SessionMeta(_) => {}
}
@@ -82,6 +82,7 @@ pub fn project_rollout_line(line: &RolloutLine) -> ThreadHistoryChangeSet {
| RolloutItem::Compacted(_)
| RolloutItem::TurnContext(_)
| RolloutItem::WorldState(_)
| RolloutItem::RealtimeItem(_)
| RolloutItem::SecurityRiskScore(_)
| RolloutItem::EventMsg(_) => ThreadHistoryChangeSet::default(),
}
@@ -7,11 +7,139 @@ use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeOutputModality;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_protocol::realtime::BemItemPresentation as CoreBemItemPresentation;
use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
use std::collections::BTreeMap;
/// EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeItem {
pub id: String,
pub realtime_session_id: String,
#[serde(flatten)]
pub content: ThreadRealtimeItemContent,
}
/// EXPERIMENTAL - durable facts describing realtime speech and promoted work.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(
tag = "type",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeItemContent {
RealtimeSessionStarted,
TranscriptSegment {
role: ThreadRealtimeTranscriptRole,
text: String,
},
BemItemPromoted {
turn_id: String,
item_id: String,
presentation: ThreadRealtimeBemItemPresentation,
},
RealtimeSessionClosed {
outcome: ThreadRealtimeSessionOutcome,
},
}
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(
tag = "type",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeBemItemPresentation {
WholeItem,
InlineMarkdown,
InlineVisualization { index: u32 },
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeTranscriptRole {
User,
Assistant,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeSessionOutcome {
Ended,
Failed,
}
impl From<CoreRealtimeItem> for ThreadRealtimeItem {
fn from(item: CoreRealtimeItem) -> Self {
let CoreRealtimeItem {
id,
realtime_session_id,
content,
} = item;
let content = match content {
CoreRealtimeItemContent::RealtimeSessionStarted => {
ThreadRealtimeItemContent::RealtimeSessionStarted
}
CoreRealtimeItemContent::TranscriptSegment { role, text } => {
ThreadRealtimeItemContent::TranscriptSegment {
role: match role {
CoreRealtimeTranscriptRole::User => ThreadRealtimeTranscriptRole::User,
CoreRealtimeTranscriptRole::Assistant => {
ThreadRealtimeTranscriptRole::Assistant
}
},
text,
}
}
CoreRealtimeItemContent::BemItemPromoted {
turn_id,
item_id,
presentation,
} => ThreadRealtimeItemContent::BemItemPromoted {
turn_id,
item_id,
presentation: match presentation {
CoreBemItemPresentation::WholeItem => {
ThreadRealtimeBemItemPresentation::WholeItem
}
CoreBemItemPresentation::InlineMarkdown => {
ThreadRealtimeBemItemPresentation::InlineMarkdown
}
CoreBemItemPresentation::InlineVisualization { index } => {
ThreadRealtimeBemItemPresentation::InlineVisualization { index }
}
},
},
CoreRealtimeItemContent::RealtimeSessionClosed { outcome } => {
ThreadRealtimeItemContent::RealtimeSessionClosed {
outcome: match outcome {
CoreRealtimeSessionOutcome::Ended => ThreadRealtimeSessionOutcome::Ended,
CoreRealtimeSessionOutcome::Failed => ThreadRealtimeSessionOutcome::Failed,
},
}
}
};
Self {
id,
realtime_session_id,
content,
}
}
}
/// EXPERIMENTAL - thread realtime audio chunk.
#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
@@ -263,6 +391,34 @@ pub struct ThreadRealtimeItemAddedNotification {
pub item: JsonValue,
}
/// EXPERIMENTAL - a realtime timeline item started before its content streams.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeItemStartedNotification {
pub thread_id: String,
pub item: ThreadRealtimeItem,
}
/// EXPERIMENTAL - text appended to an active realtime transcript item.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeItemTranscriptDeltaNotification {
pub thread_id: String,
pub item_id: String,
pub delta: String,
}
/// EXPERIMENTAL - a realtime timeline item published after canonical commit.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeItemCompletedNotification {
pub thread_id: String,
pub item: ThreadRealtimeItem,
}
/// EXPERIMENTAL - flat transcript delta emitted whenever realtime
/// transcript text changes.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
@@ -6,12 +6,15 @@ use super::SandboxPolicy;
use super::Thread;
use super::ThreadHistoryMode;
use super::ThreadItem;
use super::ThreadRealtimeItem;
use super::ThreadSection;
use super::ThreadSectionAppearance;
use super::ThreadSource;
use super::Turn;
use super::TurnEnvironmentParams;
use super::TurnError;
use super::TurnItemsView;
use super::TurnStatus;
use super::UserInput;
use super::shared::v2_enum_from_core;
use crate::JsonSchema;
@@ -1747,6 +1750,75 @@ pub struct ThreadItemsListResponse {
pub backwards_cursor: Option<String>,
}
/// EXPERIMENTAL - list ordinary and realtime thread history in rollout order.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadTimelineListParams {
pub thread_id: String,
#[ts(optional = nullable)]
pub cursor: Option<String>,
#[ts(optional = nullable)]
pub limit: Option<u32>,
}
/// EXPERIMENTAL - one item or turn boundary in canonical rollout order.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(
tag = "type",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadTimelineEntry {
Item {
#[ts(type = "number")]
position: u64,
#[schemars(rename = "turnId")]
#[serde(rename = "turnId")]
#[ts(rename = "turnId")]
turn_id: String,
item: Box<ThreadItem>,
},
Realtime {
#[ts(type = "number")]
position: u64,
item: ThreadRealtimeItem,
},
TurnStarted {
#[ts(type = "number")]
position: u64,
#[ts(rename = "turnId")]
turn_id: String,
#[ts(rename = "startedAt", type = "number | null")]
started_at: Option<i64>,
},
TurnCompleted {
#[ts(type = "number")]
position: u64,
#[ts(rename = "turnId")]
turn_id: String,
status: TurnStatus,
error: Option<TurnError>,
#[ts(rename = "startedAt", type = "number | null")]
started_at: Option<i64>,
#[ts(rename = "completedAt", type = "number | null")]
completed_at: Option<i64>,
#[ts(rename = "durationMs", type = "number | null")]
duration_ms: Option<i64>,
},
}
/// EXPERIMENTAL - a bounded timeline page with its resolved opening voice state.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadTimelineListResponse {
pub data: Vec<ThreadTimelineEntry>,
pub next_cursor: Option<String>,
pub active_realtime_session_at_page_start: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]