mirror of
https://github.com/openai/codex.git
synced 2026-11-18 23:02:23 +00:00
Persist realtime events in the thread timeline (#40508)
## Why Clients need a bounded view of realtime conversations that preserves the order of speech, agent work, and turn lifecycle events without loading the full thread history. ## What changed - Persist realtime session boundaries, transcript segments, and promoted agent items in paginated thread rollouts. - Add the experimental `thread/timeline/list` API to page ordinary items, realtime items, and turn boundaries in canonical rollout order. Include `activeRealtimeSessionAtPageStart` so each page can be rendered independently. - Emit typed `thread/realtime/item/started`, `thread/realtime/item/transcript/delta`, and `thread/realtime/item/completed` notifications around durable realtime items. - Keep the existing `thread/items/list` response and legacy thread history behavior unchanged. ## Testing - Cover mixed timeline pagination, realtime-session state at page boundaries, rollout replay, subagent history prefixes, transcript streaming, steering, and promoted agent artifacts. GitOrigin-RevId: faa2c420192d63e060e8fb32446e3c797422656e
This commit is contained in:
1 parent
80e871309a
commit
41ca88e95c
80 files changed
+5140
-47
No files matched your search
@@ -1014,6 +1014,12 @@ client_request_definitions! {
|
||||
serialization: thread_id(params.thread_id),
|
||||
response: v2::ThreadRealtimeStopResponse,
|
||||
},
|
||||
#[experimental("thread/timeline/list")]
|
||||
ThreadTimelineList => "thread/timeline/list" {
|
||||
params: v2::ThreadTimelineListParams,
|
||||
serialization: thread_id(params.thread_id),
|
||||
response: v2::ThreadTimelineListResponse,
|
||||
},
|
||||
#[experimental("thread/realtime/listVoices")]
|
||||
ThreadRealtimeListVoices => "thread/realtime/listVoices" {
|
||||
params: v2::ThreadRealtimeListVoicesParams,
|
||||
@@ -1917,6 +1923,12 @@ server_notification_definitions! {
|
||||
ThreadRealtimeStarted => "thread/realtime/started" (v2::ThreadRealtimeStartedNotification),
|
||||
#[experimental("thread/realtime/itemAdded")]
|
||||
ThreadRealtimeItemAdded => "thread/realtime/itemAdded" (v2::ThreadRealtimeItemAddedNotification),
|
||||
#[experimental("thread/realtime/item/started")]
|
||||
ThreadRealtimeItemStarted => "thread/realtime/item/started" (v2::ThreadRealtimeItemStartedNotification),
|
||||
#[experimental("thread/realtime/item/transcript/delta")]
|
||||
ThreadRealtimeItemTranscriptDelta => "thread/realtime/item/transcript/delta" (v2::ThreadRealtimeItemTranscriptDeltaNotification),
|
||||
#[experimental("thread/realtime/item/completed")]
|
||||
ThreadRealtimeItemCompleted => "thread/realtime/item/completed" (v2::ThreadRealtimeItemCompletedNotification),
|
||||
#[experimental("thread/realtime/transcript/delta")]
|
||||
ThreadRealtimeTranscriptDelta => "thread/realtime/transcript/delta" (v2::ThreadRealtimeTranscriptDeltaNotification),
|
||||
#[experimental("thread/realtime/transcript/done")]
|
||||
|
||||
@@ -401,6 +401,7 @@ impl ThreadHistoryBuilder {
|
||||
| RolloutItem::InterAgentCommunicationMetadata { .. }
|
||||
| RolloutItem::TurnContext(_)
|
||||
| RolloutItem::WorldState(_)
|
||||
| RolloutItem::RealtimeItem(_)
|
||||
| RolloutItem::SecurityRiskScore(_)
|
||||
| RolloutItem::SessionMeta(_) => {}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ pub fn project_rollout_line(line: &RolloutLine) -> ThreadHistoryChangeSet {
|
||||
| RolloutItem::Compacted(_)
|
||||
| RolloutItem::TurnContext(_)
|
||||
| RolloutItem::WorldState(_)
|
||||
| RolloutItem::RealtimeItem(_)
|
||||
| RolloutItem::SecurityRiskScore(_)
|
||||
| RolloutItem::EventMsg(_) => ThreadHistoryChangeSet::default(),
|
||||
}
|
||||
|
||||
@@ -7,11 +7,139 @@ use codex_protocol::protocol::RealtimeConversationVersion;
|
||||
use codex_protocol::protocol::RealtimeOutputModality;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::RealtimeVoicesList;
|
||||
use codex_protocol::realtime::BemItemPresentation as CoreBemItemPresentation;
|
||||
use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
|
||||
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
|
||||
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
|
||||
use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeItem {
|
||||
pub id: String,
|
||||
pub realtime_session_id: String,
|
||||
#[serde(flatten)]
|
||||
pub content: ThreadRealtimeItemContent,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - durable facts describing realtime speech and promoted work.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all = "camelCase",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeItemContent {
|
||||
RealtimeSessionStarted,
|
||||
TranscriptSegment {
|
||||
role: ThreadRealtimeTranscriptRole,
|
||||
text: String,
|
||||
},
|
||||
BemItemPromoted {
|
||||
turn_id: String,
|
||||
item_id: String,
|
||||
presentation: ThreadRealtimeBemItemPresentation,
|
||||
},
|
||||
RealtimeSessionClosed {
|
||||
outcome: ThreadRealtimeSessionOutcome,
|
||||
},
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all = "camelCase",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeBemItemPresentation {
|
||||
WholeItem,
|
||||
InlineMarkdown,
|
||||
InlineVisualization { index: u32 },
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeTranscriptRole {
|
||||
User,
|
||||
Assistant,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeSessionOutcome {
|
||||
Ended,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl From<CoreRealtimeItem> for ThreadRealtimeItem {
|
||||
fn from(item: CoreRealtimeItem) -> Self {
|
||||
let CoreRealtimeItem {
|
||||
id,
|
||||
realtime_session_id,
|
||||
content,
|
||||
} = item;
|
||||
let content = match content {
|
||||
CoreRealtimeItemContent::RealtimeSessionStarted => {
|
||||
ThreadRealtimeItemContent::RealtimeSessionStarted
|
||||
}
|
||||
CoreRealtimeItemContent::TranscriptSegment { role, text } => {
|
||||
ThreadRealtimeItemContent::TranscriptSegment {
|
||||
role: match role {
|
||||
CoreRealtimeTranscriptRole::User => ThreadRealtimeTranscriptRole::User,
|
||||
CoreRealtimeTranscriptRole::Assistant => {
|
||||
ThreadRealtimeTranscriptRole::Assistant
|
||||
}
|
||||
},
|
||||
text,
|
||||
}
|
||||
}
|
||||
CoreRealtimeItemContent::BemItemPromoted {
|
||||
turn_id,
|
||||
item_id,
|
||||
presentation,
|
||||
} => ThreadRealtimeItemContent::BemItemPromoted {
|
||||
turn_id,
|
||||
item_id,
|
||||
presentation: match presentation {
|
||||
CoreBemItemPresentation::WholeItem => {
|
||||
ThreadRealtimeBemItemPresentation::WholeItem
|
||||
}
|
||||
CoreBemItemPresentation::InlineMarkdown => {
|
||||
ThreadRealtimeBemItemPresentation::InlineMarkdown
|
||||
}
|
||||
CoreBemItemPresentation::InlineVisualization { index } => {
|
||||
ThreadRealtimeBemItemPresentation::InlineVisualization { index }
|
||||
}
|
||||
},
|
||||
},
|
||||
CoreRealtimeItemContent::RealtimeSessionClosed { outcome } => {
|
||||
ThreadRealtimeItemContent::RealtimeSessionClosed {
|
||||
outcome: match outcome {
|
||||
CoreRealtimeSessionOutcome::Ended => ThreadRealtimeSessionOutcome::Ended,
|
||||
CoreRealtimeSessionOutcome::Failed => ThreadRealtimeSessionOutcome::Failed,
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
Self {
|
||||
id,
|
||||
realtime_session_id,
|
||||
content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - thread realtime audio chunk.
|
||||
#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -263,6 +391,34 @@ pub struct ThreadRealtimeItemAddedNotification {
|
||||
pub item: JsonValue,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - a realtime timeline item started before its content streams.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeItemStartedNotification {
|
||||
pub thread_id: String,
|
||||
pub item: ThreadRealtimeItem,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - text appended to an active realtime transcript item.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeItemTranscriptDeltaNotification {
|
||||
pub thread_id: String,
|
||||
pub item_id: String,
|
||||
pub delta: String,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - a realtime timeline item published after canonical commit.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadRealtimeItemCompletedNotification {
|
||||
pub thread_id: String,
|
||||
pub item: ThreadRealtimeItem,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - flat transcript delta emitted whenever realtime
|
||||
/// transcript text changes.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
|
||||
@@ -6,12 +6,15 @@ use super::SandboxPolicy;
|
||||
use super::Thread;
|
||||
use super::ThreadHistoryMode;
|
||||
use super::ThreadItem;
|
||||
use super::ThreadRealtimeItem;
|
||||
use super::ThreadSection;
|
||||
use super::ThreadSectionAppearance;
|
||||
use super::ThreadSource;
|
||||
use super::Turn;
|
||||
use super::TurnEnvironmentParams;
|
||||
use super::TurnError;
|
||||
use super::TurnItemsView;
|
||||
use super::TurnStatus;
|
||||
use super::UserInput;
|
||||
use super::shared::v2_enum_from_core;
|
||||
use crate::JsonSchema;
|
||||
@@ -1747,6 +1750,75 @@ pub struct ThreadItemsListResponse {
|
||||
pub backwards_cursor: Option<String>,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - list ordinary and realtime thread history in rollout order.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadTimelineListParams {
|
||||
pub thread_id: String,
|
||||
#[ts(optional = nullable)]
|
||||
pub cursor: Option<String>,
|
||||
#[ts(optional = nullable)]
|
||||
pub limit: Option<u32>,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - one item or turn boundary in canonical rollout order.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
rename_all = "camelCase",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadTimelineEntry {
|
||||
Item {
|
||||
#[ts(type = "number")]
|
||||
position: u64,
|
||||
#[schemars(rename = "turnId")]
|
||||
#[serde(rename = "turnId")]
|
||||
#[ts(rename = "turnId")]
|
||||
turn_id: String,
|
||||
item: Box<ThreadItem>,
|
||||
},
|
||||
Realtime {
|
||||
#[ts(type = "number")]
|
||||
position: u64,
|
||||
item: ThreadRealtimeItem,
|
||||
},
|
||||
TurnStarted {
|
||||
#[ts(type = "number")]
|
||||
position: u64,
|
||||
#[ts(rename = "turnId")]
|
||||
turn_id: String,
|
||||
#[ts(rename = "startedAt", type = "number | null")]
|
||||
started_at: Option<i64>,
|
||||
},
|
||||
TurnCompleted {
|
||||
#[ts(type = "number")]
|
||||
position: u64,
|
||||
#[ts(rename = "turnId")]
|
||||
turn_id: String,
|
||||
status: TurnStatus,
|
||||
error: Option<TurnError>,
|
||||
#[ts(rename = "startedAt", type = "number | null")]
|
||||
started_at: Option<i64>,
|
||||
#[ts(rename = "completedAt", type = "number | null")]
|
||||
completed_at: Option<i64>,
|
||||
#[ts(rename = "durationMs", type = "number | null")]
|
||||
duration_ms: Option<i64>,
|
||||
},
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - a bounded timeline page with its resolved opening voice state.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct ThreadTimelineListResponse {
|
||||
pub data: Vec<ThreadTimelineEntry>,
|
||||
pub next_cursor: Option<String>,
|
||||
pub active_realtime_session_at_page_start: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
|
||||
Reference in new issue
Block a user