diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 6772a70680..50dc26a8eb 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -5060,6 +5060,65 @@ ], "type": "object" }, + "ThreadRealtimeBemItemPresentation": { + "description": "EXPERIMENTAL - how an existing agent item appears in a realtime conversation.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "wholeItem" + ], + "title": "WholeItemThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WholeItemThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "type": { + "enum": [ + "inlineMarkdown" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "index": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "inlineVisualization" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "index", + "type" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentation", + "type": "object" + } + ] + }, "ThreadRealtimeClosedNotification": { "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", "properties": { @@ -5094,6 +5153,112 @@ ], "type": "object" }, + "ThreadRealtimeItem": { + "description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "realtimeSessionStarted" + ], + "title": "RealtimeSessionStartedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "RealtimeSessionStartedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "role": { + "$ref": "#/definitions/ThreadRealtimeTranscriptRole" + }, + "text": { + "type": "string" + }, + "type": { + "enum": [ + "transcriptSegment" + ], + "title": "TranscriptSegmentThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "role", + "text", + "type" + ], + "title": "TranscriptSegmentThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "item_id": { + "type": "string" + }, + "presentation": { + "$ref": "#/definitions/ThreadRealtimeBemItemPresentation" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "bemItemPromoted" + ], + "title": "BemItemPromotedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "item_id", + "presentation", + "turn_id", + "type" + ], + "title": "BemItemPromotedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "outcome": { + "$ref": "#/definitions/ThreadRealtimeSessionOutcome" + }, + "type": { + "enum": [ + "realtimeSessionClosed" + ], + "title": "RealtimeSessionClosedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "outcome", + "type" + ], + "title": "RealtimeSessionClosedThreadRealtimeItem", + "type": "object" + } + ], + "properties": { + "id": { + "type": "string" + }, + "realtimeSessionId": { + "type": "string" + } + }, + "required": [ + "id", + "realtimeSessionId" + ], + "type": "object" + }, "ThreadRealtimeItemAddedNotification": { "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", "properties": { @@ -5108,6 +5273,58 @@ ], "type": "object" }, + "ThreadRealtimeItemCompletedNotification": { + "description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeItemStartedNotification": { + "description": "EXPERIMENTAL - a realtime timeline item started before its content streams.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeItemTranscriptDeltaNotification": { + "description": "EXPERIMENTAL - text appended to an active realtime transcript item.", + "properties": { + "delta": { + "type": "string" + }, + "itemId": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "itemId", + "threadId" + ], + "type": "object" + }, "ThreadRealtimeOutputAudioDeltaNotification": { "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", "properties": { @@ -5140,6 +5357,13 @@ ], "type": "object" }, + "ThreadRealtimeSessionOutcome": { + "enum": [ + "ended", + "failed" + ], + "type": "string" + }, "ThreadRealtimeStartedNotification": { "description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.", "properties": { @@ -5204,6 +5428,13 @@ ], "type": "object" }, + "ThreadRealtimeTranscriptRole": { + "enum": [ + "user", + "assistant" + ], + "type": "string" + }, "ThreadRevertedNotification": { "properties": { "threadId": { @@ -7504,6 +7735,66 @@ "title": "Thread/realtime/itemAddedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/started" + ], + "title": "Thread/realtime/item/startedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemStartedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/startedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/transcript/delta" + ], + "title": "Thread/realtime/item/transcript/deltaNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemTranscriptDeltaNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/completed" + ], + "title": "Thread/realtime/item/completedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemCompletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/completedNotification", + "type": "object" + }, { "properties": { "method": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index e20a9351c6..55d8b4749b 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5683,6 +5683,66 @@ "title": "Thread/realtime/itemAddedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/started" + ], + "title": "Thread/realtime/item/startedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeItemStartedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/startedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/transcript/delta" + ], + "title": "Thread/realtime/item/transcript/deltaNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeItemTranscriptDeltaNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/completed" + ], + "title": "Thread/realtime/item/completedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeItemCompletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/completedNotification", + "type": "object" + }, { "properties": { "method": { @@ -21518,6 +21578,65 @@ ], "type": "object" }, + "ThreadRealtimeBemItemPresentation": { + "description": "EXPERIMENTAL - how an existing agent item appears in a realtime conversation.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "wholeItem" + ], + "title": "WholeItemThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WholeItemThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "type": { + "enum": [ + "inlineMarkdown" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "index": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "inlineVisualization" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "index", + "type" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentation", + "type": "object" + } + ] + }, "ThreadRealtimeClosedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", @@ -21572,6 +21691,112 @@ ], "type": "object" }, + "ThreadRealtimeItem": { + "description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "realtimeSessionStarted" + ], + "title": "RealtimeSessionStartedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "RealtimeSessionStartedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "role": { + "$ref": "#/definitions/v2/ThreadRealtimeTranscriptRole" + }, + "text": { + "type": "string" + }, + "type": { + "enum": [ + "transcriptSegment" + ], + "title": "TranscriptSegmentThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "role", + "text", + "type" + ], + "title": "TranscriptSegmentThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "item_id": { + "type": "string" + }, + "presentation": { + "$ref": "#/definitions/v2/ThreadRealtimeBemItemPresentation" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "bemItemPromoted" + ], + "title": "BemItemPromotedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "item_id", + "presentation", + "turn_id", + "type" + ], + "title": "BemItemPromotedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "outcome": { + "$ref": "#/definitions/v2/ThreadRealtimeSessionOutcome" + }, + "type": { + "enum": [ + "realtimeSessionClosed" + ], + "title": "RealtimeSessionClosedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "outcome", + "type" + ], + "title": "RealtimeSessionClosedThreadRealtimeItem", + "type": "object" + } + ], + "properties": { + "id": { + "type": "string" + }, + "realtimeSessionId": { + "type": "string" + } + }, + "required": [ + "id", + "realtimeSessionId" + ], + "type": "object" + }, "ThreadRealtimeItemAddedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", @@ -21588,6 +21813,64 @@ "title": "ThreadRealtimeItemAddedNotification", "type": "object" }, + "ThreadRealtimeItemCompletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.", + "properties": { + "item": { + "$ref": "#/definitions/v2/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemCompletedNotification", + "type": "object" + }, + "ThreadRealtimeItemStartedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - a realtime timeline item started before its content streams.", + "properties": { + "item": { + "$ref": "#/definitions/v2/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemStartedNotification", + "type": "object" + }, + "ThreadRealtimeItemTranscriptDeltaNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - text appended to an active realtime transcript item.", + "properties": { + "delta": { + "type": "string" + }, + "itemId": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "itemId", + "threadId" + ], + "title": "ThreadRealtimeItemTranscriptDeltaNotification", + "type": "object" + }, "ThreadRealtimeOutputAudioDeltaNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", @@ -21624,6 +21907,13 @@ "title": "ThreadRealtimeSdpNotification", "type": "object" }, + "ThreadRealtimeSessionOutcome": { + "enum": [ + "ended", + "failed" + ], + "type": "string" + }, "ThreadRealtimeStartTransport": { "description": "EXPERIMENTAL - transport used by thread realtime.", "oneOf": [ @@ -21757,6 +22047,13 @@ "title": "ThreadRealtimeTranscriptDoneNotification", "type": "object" }, + "ThreadRealtimeTranscriptRole": { + "enum": [ + "user", + "assistant" + ], + "type": "string" + }, "ThreadResumeInitialTurnsPageParams": { "properties": { "itemsView": { @@ -22752,6 +23049,161 @@ "title": "ThreadStatusChangedNotification", "type": "object" }, + "ThreadTimelineEntry": { + "description": "EXPERIMENTAL - one item or turn boundary in canonical rollout order.", + "oneOf": [ + { + "properties": { + "item": { + "$ref": "#/definitions/v2/ThreadItem" + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turnId": { + "type": "string" + }, + "type": { + "enum": [ + "item" + ], + "title": "ItemThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "item", + "position", + "turnId", + "type" + ], + "title": "ItemThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "item": { + "$ref": "#/definitions/v2/ThreadRealtimeItem" + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "realtime" + ], + "title": "RealtimeThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "item", + "position", + "type" + ], + "title": "RealtimeThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "started_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "turnStarted" + ], + "title": "TurnStartedThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "position", + "turn_id", + "type" + ], + "title": "TurnStartedThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "completed_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "duration_ms": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/v2/TurnError" + }, + { + "type": "null" + } + ] + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "started_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "status": { + "$ref": "#/definitions/v2/TurnStatus" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "turnCompleted" + ], + "title": "TurnCompletedThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "position", + "status", + "turn_id", + "type" + ], + "title": "TurnCompletedThreadTimelineEntry", + "type": "object" + } + ] + }, "ThreadTokenUsage": { "properties": { "last": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 6afde55a16..5d6b9441a8 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -16424,6 +16424,66 @@ "title": "Thread/realtime/itemAddedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/started" + ], + "title": "Thread/realtime/item/startedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemStartedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/startedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/transcript/delta" + ], + "title": "Thread/realtime/item/transcript/deltaNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemTranscriptDeltaNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/transcript/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/item/completed" + ], + "title": "Thread/realtime/item/completedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemCompletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/item/completedNotification", + "type": "object" + }, { "properties": { "method": { @@ -19270,6 +19330,65 @@ ], "type": "object" }, + "ThreadRealtimeBemItemPresentation": { + "description": "EXPERIMENTAL - how an existing agent item appears in a realtime conversation.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "wholeItem" + ], + "title": "WholeItemThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WholeItemThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "type": { + "enum": [ + "inlineMarkdown" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "index": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "inlineVisualization" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "index", + "type" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentation", + "type": "object" + } + ] + }, "ThreadRealtimeClosedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", @@ -19324,6 +19443,112 @@ ], "type": "object" }, + "ThreadRealtimeItem": { + "description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "realtimeSessionStarted" + ], + "title": "RealtimeSessionStartedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "RealtimeSessionStartedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "role": { + "$ref": "#/definitions/ThreadRealtimeTranscriptRole" + }, + "text": { + "type": "string" + }, + "type": { + "enum": [ + "transcriptSegment" + ], + "title": "TranscriptSegmentThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "role", + "text", + "type" + ], + "title": "TranscriptSegmentThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "item_id": { + "type": "string" + }, + "presentation": { + "$ref": "#/definitions/ThreadRealtimeBemItemPresentation" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "bemItemPromoted" + ], + "title": "BemItemPromotedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "item_id", + "presentation", + "turn_id", + "type" + ], + "title": "BemItemPromotedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "outcome": { + "$ref": "#/definitions/ThreadRealtimeSessionOutcome" + }, + "type": { + "enum": [ + "realtimeSessionClosed" + ], + "title": "RealtimeSessionClosedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "outcome", + "type" + ], + "title": "RealtimeSessionClosedThreadRealtimeItem", + "type": "object" + } + ], + "properties": { + "id": { + "type": "string" + }, + "realtimeSessionId": { + "type": "string" + } + }, + "required": [ + "id", + "realtimeSessionId" + ], + "type": "object" + }, "ThreadRealtimeItemAddedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", @@ -19340,6 +19565,64 @@ "title": "ThreadRealtimeItemAddedNotification", "type": "object" }, + "ThreadRealtimeItemCompletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemCompletedNotification", + "type": "object" + }, + "ThreadRealtimeItemStartedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - a realtime timeline item started before its content streams.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemStartedNotification", + "type": "object" + }, + "ThreadRealtimeItemTranscriptDeltaNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - text appended to an active realtime transcript item.", + "properties": { + "delta": { + "type": "string" + }, + "itemId": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "itemId", + "threadId" + ], + "title": "ThreadRealtimeItemTranscriptDeltaNotification", + "type": "object" + }, "ThreadRealtimeOutputAudioDeltaNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", @@ -19376,6 +19659,13 @@ "title": "ThreadRealtimeSdpNotification", "type": "object" }, + "ThreadRealtimeSessionOutcome": { + "enum": [ + "ended", + "failed" + ], + "type": "string" + }, "ThreadRealtimeStartTransport": { "description": "EXPERIMENTAL - transport used by thread realtime.", "oneOf": [ @@ -19509,6 +19799,13 @@ "title": "ThreadRealtimeTranscriptDoneNotification", "type": "object" }, + "ThreadRealtimeTranscriptRole": { + "enum": [ + "user", + "assistant" + ], + "type": "string" + }, "ThreadResumeInitialTurnsPageParams": { "properties": { "itemsView": { @@ -20504,6 +20801,161 @@ "title": "ThreadStatusChangedNotification", "type": "object" }, + "ThreadTimelineEntry": { + "description": "EXPERIMENTAL - one item or turn boundary in canonical rollout order.", + "oneOf": [ + { + "properties": { + "item": { + "$ref": "#/definitions/ThreadItem" + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "turnId": { + "type": "string" + }, + "type": { + "enum": [ + "item" + ], + "title": "ItemThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "item", + "position", + "turnId", + "type" + ], + "title": "ItemThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "realtime" + ], + "title": "RealtimeThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "item", + "position", + "type" + ], + "title": "RealtimeThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "started_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "turnStarted" + ], + "title": "TurnStartedThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "position", + "turn_id", + "type" + ], + "title": "TurnStartedThreadTimelineEntry", + "type": "object" + }, + { + "properties": { + "completed_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "duration_ms": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/TurnError" + }, + { + "type": "null" + } + ] + }, + "position": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "started_at": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "status": { + "$ref": "#/definitions/TurnStatus" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "turnCompleted" + ], + "title": "TurnCompletedThreadTimelineEntryType", + "type": "string" + } + }, + "required": [ + "position", + "status", + "turn_id", + "type" + ], + "title": "TurnCompletedThreadTimelineEntry", + "type": "object" + } + ] + }, "ThreadTokenUsage": { "properties": { "last": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json new file mode 100644 index 0000000000..b24e3169ef --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json @@ -0,0 +1,199 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ThreadRealtimeBemItemPresentation": { + "description": "EXPERIMENTAL - how an existing agent item appears in a realtime conversation.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "wholeItem" + ], + "title": "WholeItemThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WholeItemThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "type": { + "enum": [ + "inlineMarkdown" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "index": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "inlineVisualization" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "index", + "type" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentation", + "type": "object" + } + ] + }, + "ThreadRealtimeItem": { + "description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "realtimeSessionStarted" + ], + "title": "RealtimeSessionStartedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "RealtimeSessionStartedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "role": { + "$ref": "#/definitions/ThreadRealtimeTranscriptRole" + }, + "text": { + "type": "string" + }, + "type": { + "enum": [ + "transcriptSegment" + ], + "title": "TranscriptSegmentThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "role", + "text", + "type" + ], + "title": "TranscriptSegmentThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "item_id": { + "type": "string" + }, + "presentation": { + "$ref": "#/definitions/ThreadRealtimeBemItemPresentation" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "bemItemPromoted" + ], + "title": "BemItemPromotedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "item_id", + "presentation", + "turn_id", + "type" + ], + "title": "BemItemPromotedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "outcome": { + "$ref": "#/definitions/ThreadRealtimeSessionOutcome" + }, + "type": { + "enum": [ + "realtimeSessionClosed" + ], + "title": "RealtimeSessionClosedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "outcome", + "type" + ], + "title": "RealtimeSessionClosedThreadRealtimeItem", + "type": "object" + } + ], + "properties": { + "id": { + "type": "string" + }, + "realtimeSessionId": { + "type": "string" + } + }, + "required": [ + "id", + "realtimeSessionId" + ], + "type": "object" + }, + "ThreadRealtimeSessionOutcome": { + "enum": [ + "ended", + "failed" + ], + "type": "string" + }, + "ThreadRealtimeTranscriptRole": { + "enum": [ + "user", + "assistant" + ], + "type": "string" + } + }, + "description": "EXPERIMENTAL - a realtime timeline item published after canonical commit.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemCompletedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json new file mode 100644 index 0000000000..74971aa932 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json @@ -0,0 +1,199 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ThreadRealtimeBemItemPresentation": { + "description": "EXPERIMENTAL - how an existing agent item appears in a realtime conversation.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "wholeItem" + ], + "title": "WholeItemThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "WholeItemThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "type": { + "enum": [ + "inlineMarkdown" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "InlineMarkdownThreadRealtimeBemItemPresentation", + "type": "object" + }, + { + "properties": { + "index": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": { + "enum": [ + "inlineVisualization" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentationType", + "type": "string" + } + }, + "required": [ + "index", + "type" + ], + "title": "InlineVisualizationThreadRealtimeBemItemPresentation", + "type": "object" + } + ] + }, + "ThreadRealtimeItem": { + "description": "EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline.", + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "realtimeSessionStarted" + ], + "title": "RealtimeSessionStartedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "RealtimeSessionStartedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "role": { + "$ref": "#/definitions/ThreadRealtimeTranscriptRole" + }, + "text": { + "type": "string" + }, + "type": { + "enum": [ + "transcriptSegment" + ], + "title": "TranscriptSegmentThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "role", + "text", + "type" + ], + "title": "TranscriptSegmentThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "item_id": { + "type": "string" + }, + "presentation": { + "$ref": "#/definitions/ThreadRealtimeBemItemPresentation" + }, + "turn_id": { + "type": "string" + }, + "type": { + "enum": [ + "bemItemPromoted" + ], + "title": "BemItemPromotedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "item_id", + "presentation", + "turn_id", + "type" + ], + "title": "BemItemPromotedThreadRealtimeItem", + "type": "object" + }, + { + "properties": { + "outcome": { + "$ref": "#/definitions/ThreadRealtimeSessionOutcome" + }, + "type": { + "enum": [ + "realtimeSessionClosed" + ], + "title": "RealtimeSessionClosedThreadRealtimeItemType", + "type": "string" + } + }, + "required": [ + "outcome", + "type" + ], + "title": "RealtimeSessionClosedThreadRealtimeItem", + "type": "object" + } + ], + "properties": { + "id": { + "type": "string" + }, + "realtimeSessionId": { + "type": "string" + } + }, + "required": [ + "id", + "realtimeSessionId" + ], + "type": "object" + }, + "ThreadRealtimeSessionOutcome": { + "enum": [ + "ended", + "failed" + ], + "type": "string" + }, + "ThreadRealtimeTranscriptRole": { + "enum": [ + "user", + "assistant" + ], + "type": "string" + } + }, + "description": "EXPERIMENTAL - a realtime timeline item started before its content streams.", + "properties": { + "item": { + "$ref": "#/definitions/ThreadRealtimeItem" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemStartedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemTranscriptDeltaNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemTranscriptDeltaNotification.json new file mode 100644 index 0000000000..0354fcd408 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemTranscriptDeltaNotification.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - text appended to an active realtime transcript item.", + "properties": { + "delta": { + "type": "string" + }, + "itemId": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "delta", + "itemId", + "threadId" + ], + "title": "ThreadRealtimeItemTranscriptDeltaNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 65ca7eb274..1404530dda 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index a12fda2014..e95edd35a3 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index c0179dc46e..1ee34f8cae 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -59,6 +59,9 @@ import type { ThreadQueueChangedNotification } from "./v2/ThreadQueueChangedNoti import type { ThreadRealtimeClosedNotification } from "./v2/ThreadRealtimeClosedNotification"; import type { ThreadRealtimeErrorNotification } from "./v2/ThreadRealtimeErrorNotification"; import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeItemAddedNotification"; +import type { ThreadRealtimeItemCompletedNotification } from "./v2/ThreadRealtimeItemCompletedNotification"; +import type { ThreadRealtimeItemStartedNotification } from "./v2/ThreadRealtimeItemStartedNotification"; +import type { ThreadRealtimeItemTranscriptDeltaNotification } from "./v2/ThreadRealtimeItemTranscriptDeltaNotification"; import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification"; import type { ThreadRealtimeSdpNotification } from "./v2/ThreadRealtimeSdpNotification"; import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification"; @@ -82,4 +85,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "thread/reverted", "params": ThreadRevertedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/queue/changed", "params": ThreadQueueChangedNotification } | { "method": "project/changed", "params": ProjectChangedNotification } | { "method": "thread/project/updated", "params": ThreadProjectUpdatedNotification } | { "method": "thread/environment/connected", "params": EnvironmentConnectionNotification } | { "method": "thread/environment/disconnected", "params": EnvironmentConnectionNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "autoApprovalReview/strictReviewRequired", "params": StrictReviewRequiredNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "rawResponse/completed", "params": RawResponseCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "mcpServer/event/stream/notification", "params": McpServerEventStreamNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/progress", "params": ExternalAgentConfigImportProgressNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "model/safetyBuffering/updated", "params": ModelSafetyBufferingUpdatedNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "thread/reverted", "params": ThreadRevertedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/queue/changed", "params": ThreadQueueChangedNotification } | { "method": "project/changed", "params": ProjectChangedNotification } | { "method": "thread/project/updated", "params": ThreadProjectUpdatedNotification } | { "method": "thread/environment/connected", "params": EnvironmentConnectionNotification } | { "method": "thread/environment/disconnected", "params": EnvironmentConnectionNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "autoApprovalReview/strictReviewRequired", "params": StrictReviewRequiredNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "rawResponse/completed", "params": RawResponseCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "mcpServer/event/stream/notification", "params": McpServerEventStreamNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/progress", "params": ExternalAgentConfigImportProgressNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "model/safetyBuffering/updated", "params": ModelSafetyBufferingUpdatedNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/item/started", "params": ThreadRealtimeItemStartedNotification } | { "method": "thread/realtime/item/transcript/delta", "params": ThreadRealtimeItemTranscriptDeltaNotification } | { "method": "thread/realtime/item/completed", "params": ThreadRealtimeItemCompletedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotificationEnvelope.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotificationEnvelope.ts index a8c657713d..f6aa0d7d87 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotificationEnvelope.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotificationEnvelope.ts @@ -59,6 +59,9 @@ import type { ThreadQueueChangedNotification } from "./v2/ThreadQueueChangedNoti import type { ThreadRealtimeClosedNotification } from "./v2/ThreadRealtimeClosedNotification"; import type { ThreadRealtimeErrorNotification } from "./v2/ThreadRealtimeErrorNotification"; import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeItemAddedNotification"; +import type { ThreadRealtimeItemCompletedNotification } from "./v2/ThreadRealtimeItemCompletedNotification"; +import type { ThreadRealtimeItemStartedNotification } from "./v2/ThreadRealtimeItemStartedNotification"; +import type { ThreadRealtimeItemTranscriptDeltaNotification } from "./v2/ThreadRealtimeItemTranscriptDeltaNotification"; import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification"; import type { ThreadRealtimeSdpNotification } from "./v2/ThreadRealtimeSdpNotification"; import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification"; @@ -92,4 +95,4 @@ export type ServerNotificationEnvelope = { * Optional so clients can decode notifications from older app-server * versions. Current app-server versions always populate it. */ -emittedAtMs?: number, } & ({ "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "thread/reverted", "params": ThreadRevertedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/queue/changed", "params": ThreadQueueChangedNotification } | { "method": "project/changed", "params": ProjectChangedNotification } | { "method": "thread/project/updated", "params": ThreadProjectUpdatedNotification } | { "method": "thread/environment/connected", "params": EnvironmentConnectionNotification } | { "method": "thread/environment/disconnected", "params": EnvironmentConnectionNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "autoApprovalReview/strictReviewRequired", "params": StrictReviewRequiredNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "rawResponse/completed", "params": RawResponseCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "mcpServer/event/stream/notification", "params": McpServerEventStreamNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/progress", "params": ExternalAgentConfigImportProgressNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "model/safetyBuffering/updated", "params": ModelSafetyBufferingUpdatedNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }); +emittedAtMs?: number, } & ({ "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "thread/reverted", "params": ThreadRevertedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/queue/changed", "params": ThreadQueueChangedNotification } | { "method": "project/changed", "params": ProjectChangedNotification } | { "method": "thread/project/updated", "params": ThreadProjectUpdatedNotification } | { "method": "thread/environment/connected", "params": EnvironmentConnectionNotification } | { "method": "thread/environment/disconnected", "params": EnvironmentConnectionNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "autoApprovalReview/strictReviewRequired", "params": StrictReviewRequiredNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "rawResponse/completed", "params": RawResponseCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "mcpServer/event/stream/notification", "params": McpServerEventStreamNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/progress", "params": ExternalAgentConfigImportProgressNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "model/safetyBuffering/updated", "params": ModelSafetyBufferingUpdatedNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/item/started", "params": ThreadRealtimeItemStartedNotification } | { "method": "thread/realtime/item/transcript/delta", "params": ThreadRealtimeItemTranscriptDeltaNotification } | { "method": "thread/realtime/item/completed", "params": ThreadRealtimeItemCompletedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }); diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeBemItemPresentation.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeBemItemPresentation.ts new file mode 100644 index 0000000000..61a11939c0 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeBemItemPresentation.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - how an existing agent item appears in a realtime conversation. + */ +export type ThreadRealtimeBemItemPresentation = { "type": "wholeItem" } | { "type": "inlineMarkdown" } | { "type": "inlineVisualization", index: number, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts new file mode 100644 index 0000000000..5ac5de98a2 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts @@ -0,0 +1,11 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation"; +import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome"; +import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole"; + +/** + * EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline. + */ +export type ThreadRealtimeItem = { id: string, realtimeSessionId: string, } & ({ "type": "realtimeSessionStarted" } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, }); diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemCompletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemCompletedNotification.ts new file mode 100644 index 0000000000..2a77b0c45c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemCompletedNotification.ts @@ -0,0 +1,9 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ThreadRealtimeItem } from "./ThreadRealtimeItem"; + +/** + * EXPERIMENTAL - a realtime timeline item published after canonical commit. + */ +export type ThreadRealtimeItemCompletedNotification = { threadId: string, item: ThreadRealtimeItem, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemStartedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemStartedNotification.ts new file mode 100644 index 0000000000..cec5b8b113 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemStartedNotification.ts @@ -0,0 +1,9 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ThreadRealtimeItem } from "./ThreadRealtimeItem"; + +/** + * EXPERIMENTAL - a realtime timeline item started before its content streams. + */ +export type ThreadRealtimeItemStartedNotification = { threadId: string, item: ThreadRealtimeItem, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemTranscriptDeltaNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemTranscriptDeltaNotification.ts new file mode 100644 index 0000000000..550b55bbe9 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemTranscriptDeltaNotification.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - text appended to an active realtime transcript item. + */ +export type ThreadRealtimeItemTranscriptDeltaNotification = { threadId: string, itemId: string, delta: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionOutcome.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionOutcome.ts new file mode 100644 index 0000000000..c3c73dff3f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionOutcome.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadRealtimeSessionOutcome = "ended" | "failed"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptRole.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptRole.ts new file mode 100644 index 0000000000..d957a42e22 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeTranscriptRole.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadRealtimeTranscriptRole = "user" | "assistant"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadTimelineEntry.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadTimelineEntry.ts new file mode 100644 index 0000000000..f528548299 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadTimelineEntry.ts @@ -0,0 +1,12 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ThreadItem } from "./ThreadItem"; +import type { ThreadRealtimeItem } from "./ThreadRealtimeItem"; +import type { TurnError } from "./TurnError"; +import type { TurnStatus } from "./TurnStatus"; + +/** + * EXPERIMENTAL - one item or turn boundary in canonical rollout order. + */ +export type ThreadTimelineEntry = { "type": "item", position: number, turnId: string, item: ThreadItem, } | { "type": "realtime", position: number, item: ThreadRealtimeItem, } | { "type": "turnStarted", position: number, turnId: string, startedAt: number | null, } | { "type": "turnCompleted", position: number, turnId: string, status: TurnStatus, error: TurnError | null, startedAt: number | null, completedAt: number | null, durationMs: number | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 08a6b60ff8..2c6c12f075 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -491,16 +491,23 @@ export type { ThreadQueueChangedNotification } from "./ThreadQueueChangedNotific export type { ThreadReadParams } from "./ThreadReadParams"; export type { ThreadReadResponse } from "./ThreadReadResponse"; export type { ThreadRealtimeAudioChunk } from "./ThreadRealtimeAudioChunk"; +export type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation"; export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNotification"; export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification"; export type { ThreadRealtimeInitialItem } from "./ThreadRealtimeInitialItem"; +export type { ThreadRealtimeItem } from "./ThreadRealtimeItem"; export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification"; +export type { ThreadRealtimeItemCompletedNotification } from "./ThreadRealtimeItemCompletedNotification"; +export type { ThreadRealtimeItemStartedNotification } from "./ThreadRealtimeItemStartedNotification"; +export type { ThreadRealtimeItemTranscriptDeltaNotification } from "./ThreadRealtimeItemTranscriptDeltaNotification"; export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification"; export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification"; +export type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome"; export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport"; export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification"; export type { ThreadRealtimeTranscriptDeltaNotification } from "./ThreadRealtimeTranscriptDeltaNotification"; export type { ThreadRealtimeTranscriptDoneNotification } from "./ThreadRealtimeTranscriptDoneNotification"; +export type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole"; export type { ThreadResumeInitialTurnsPageParams } from "./ThreadResumeInitialTurnsPageParams"; export type { ThreadResumeParams } from "./ThreadResumeParams"; export type { ThreadResumeResponse } from "./ThreadResumeResponse"; @@ -536,6 +543,7 @@ export type { ThreadStartSource } from "./ThreadStartSource"; export type { ThreadStartedNotification } from "./ThreadStartedNotification"; export type { ThreadStatus } from "./ThreadStatus"; export type { ThreadStatusChangedNotification } from "./ThreadStatusChangedNotification"; +export type { ThreadTimelineEntry } from "./ThreadTimelineEntry"; export type { ThreadTokenUsage } from "./ThreadTokenUsage"; export type { ThreadTokenUsageUpdatedNotification } from "./ThreadTokenUsageUpdatedNotification"; export type { ThreadUnarchiveParams } from "./ThreadUnarchiveParams"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index ee62837e7c..f466ba7a81 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1014,6 +1014,12 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadRealtimeStopResponse, }, + #[experimental("thread/timeline/list")] + ThreadTimelineList => "thread/timeline/list" { + params: v2::ThreadTimelineListParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadTimelineListResponse, + }, #[experimental("thread/realtime/listVoices")] ThreadRealtimeListVoices => "thread/realtime/listVoices" { params: v2::ThreadRealtimeListVoicesParams, @@ -1917,6 +1923,12 @@ server_notification_definitions! { ThreadRealtimeStarted => "thread/realtime/started" (v2::ThreadRealtimeStartedNotification), #[experimental("thread/realtime/itemAdded")] ThreadRealtimeItemAdded => "thread/realtime/itemAdded" (v2::ThreadRealtimeItemAddedNotification), + #[experimental("thread/realtime/item/started")] + ThreadRealtimeItemStarted => "thread/realtime/item/started" (v2::ThreadRealtimeItemStartedNotification), + #[experimental("thread/realtime/item/transcript/delta")] + ThreadRealtimeItemTranscriptDelta => "thread/realtime/item/transcript/delta" (v2::ThreadRealtimeItemTranscriptDeltaNotification), + #[experimental("thread/realtime/item/completed")] + ThreadRealtimeItemCompleted => "thread/realtime/item/completed" (v2::ThreadRealtimeItemCompletedNotification), #[experimental("thread/realtime/transcript/delta")] ThreadRealtimeTranscriptDelta => "thread/realtime/transcript/delta" (v2::ThreadRealtimeTranscriptDeltaNotification), #[experimental("thread/realtime/transcript/done")] diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 566b99d12e..5e007e0cfd 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -401,6 +401,7 @@ impl ThreadHistoryBuilder { | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::SessionMeta(_) => {} } diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history_projection.rs b/codex-rs/app-server-protocol/src/protocol/thread_history_projection.rs index d742fd0b22..38a7c7e2f6 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history_projection.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history_projection.rs @@ -82,6 +82,7 @@ pub fn project_rollout_line(line: &RolloutLine) -> ThreadHistoryChangeSet { | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => ThreadHistoryChangeSet::default(), } diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 12355c96d3..4bba26856c 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -7,11 +7,139 @@ use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; +use codex_protocol::realtime::BemItemPresentation as CoreBemItemPresentation; +use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem; +use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent; +use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome; +use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; use std::collections::BTreeMap; +/// EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeItem { + pub id: String, + pub realtime_session_id: String, + #[serde(flatten)] + pub content: ThreadRealtimeItemContent, +} + +/// EXPERIMENTAL - durable facts describing realtime speech and promoted work. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadRealtimeItemContent { + RealtimeSessionStarted, + TranscriptSegment { + role: ThreadRealtimeTranscriptRole, + text: String, + }, + BemItemPromoted { + turn_id: String, + item_id: String, + presentation: ThreadRealtimeBemItemPresentation, + }, + RealtimeSessionClosed { + outcome: ThreadRealtimeSessionOutcome, + }, +} + +/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadRealtimeBemItemPresentation { + WholeItem, + InlineMarkdown, + InlineVisualization { index: u32 }, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadRealtimeTranscriptRole { + User, + Assistant, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadRealtimeSessionOutcome { + Ended, + Failed, +} + +impl From for ThreadRealtimeItem { + fn from(item: CoreRealtimeItem) -> Self { + let CoreRealtimeItem { + id, + realtime_session_id, + content, + } = item; + let content = match content { + CoreRealtimeItemContent::RealtimeSessionStarted => { + ThreadRealtimeItemContent::RealtimeSessionStarted + } + CoreRealtimeItemContent::TranscriptSegment { role, text } => { + ThreadRealtimeItemContent::TranscriptSegment { + role: match role { + CoreRealtimeTranscriptRole::User => ThreadRealtimeTranscriptRole::User, + CoreRealtimeTranscriptRole::Assistant => { + ThreadRealtimeTranscriptRole::Assistant + } + }, + text, + } + } + CoreRealtimeItemContent::BemItemPromoted { + turn_id, + item_id, + presentation, + } => ThreadRealtimeItemContent::BemItemPromoted { + turn_id, + item_id, + presentation: match presentation { + CoreBemItemPresentation::WholeItem => { + ThreadRealtimeBemItemPresentation::WholeItem + } + CoreBemItemPresentation::InlineMarkdown => { + ThreadRealtimeBemItemPresentation::InlineMarkdown + } + CoreBemItemPresentation::InlineVisualization { index } => { + ThreadRealtimeBemItemPresentation::InlineVisualization { index } + } + }, + }, + CoreRealtimeItemContent::RealtimeSessionClosed { outcome } => { + ThreadRealtimeItemContent::RealtimeSessionClosed { + outcome: match outcome { + CoreRealtimeSessionOutcome::Ended => ThreadRealtimeSessionOutcome::Ended, + CoreRealtimeSessionOutcome::Failed => ThreadRealtimeSessionOutcome::Failed, + }, + } + } + }; + Self { + id, + realtime_session_id, + content, + } + } +} + /// EXPERIMENTAL - thread realtime audio chunk. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] @@ -263,6 +391,34 @@ pub struct ThreadRealtimeItemAddedNotification { pub item: JsonValue, } +/// EXPERIMENTAL - a realtime timeline item started before its content streams. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeItemStartedNotification { + pub thread_id: String, + pub item: ThreadRealtimeItem, +} + +/// EXPERIMENTAL - text appended to an active realtime transcript item. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeItemTranscriptDeltaNotification { + pub thread_id: String, + pub item_id: String, + pub delta: String, +} + +/// EXPERIMENTAL - a realtime timeline item published after canonical commit. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeItemCompletedNotification { + pub thread_id: String, + pub item: ThreadRealtimeItem, +} + /// EXPERIMENTAL - flat transcript delta emitted whenever realtime /// transcript text changes. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index 75fe5091b1..fc70dbe6f2 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -6,12 +6,15 @@ use super::SandboxPolicy; use super::Thread; use super::ThreadHistoryMode; use super::ThreadItem; +use super::ThreadRealtimeItem; use super::ThreadSection; use super::ThreadSectionAppearance; use super::ThreadSource; use super::Turn; use super::TurnEnvironmentParams; +use super::TurnError; use super::TurnItemsView; +use super::TurnStatus; use super::UserInput; use super::shared::v2_enum_from_core; use crate::JsonSchema; @@ -1747,6 +1750,75 @@ pub struct ThreadItemsListResponse { pub backwards_cursor: Option, } +/// EXPERIMENTAL - list ordinary and realtime thread history in rollout order. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTimelineListParams { + pub thread_id: String, + #[ts(optional = nullable)] + pub cursor: Option, + #[ts(optional = nullable)] + pub limit: Option, +} + +/// EXPERIMENTAL - one item or turn boundary in canonical rollout order. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +#[ts(tag = "type", rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadTimelineEntry { + Item { + #[ts(type = "number")] + position: u64, + #[schemars(rename = "turnId")] + #[serde(rename = "turnId")] + #[ts(rename = "turnId")] + turn_id: String, + item: Box, + }, + Realtime { + #[ts(type = "number")] + position: u64, + item: ThreadRealtimeItem, + }, + TurnStarted { + #[ts(type = "number")] + position: u64, + #[ts(rename = "turnId")] + turn_id: String, + #[ts(rename = "startedAt", type = "number | null")] + started_at: Option, + }, + TurnCompleted { + #[ts(type = "number")] + position: u64, + #[ts(rename = "turnId")] + turn_id: String, + status: TurnStatus, + error: Option, + #[ts(rename = "startedAt", type = "number | null")] + started_at: Option, + #[ts(rename = "completedAt", type = "number | null")] + completed_at: Option, + #[ts(rename = "durationMs", type = "number | null")] + duration_ms: Option, + }, +} + +/// EXPERIMENTAL - a bounded timeline page with its resolved opening voice state. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTimelineListResponse { + pub data: Vec, + pub next_cursor: Option, + pub active_realtime_session_at_page_start: Option, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 09b40ca94d..df16c36fd5 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -218,6 +218,7 @@ Example with notification opt-out: - `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user`, `developer`, or `assistant` (experimental); returns `{}`. Older clients that omit `role` default to `user`. Parent-owned Multi-Agent V2 subagents reject this request. - `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`. Parent-owned Multi-Agent V2 subagents reject this request. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. Parent-owned Multi-Agent V2 subagents reject this request. +- `thread/timeline/list` — page ordinary turn items, durable realtime facts, and turn boundaries together in rollout order (experimental). Entries are tagged `item`, `realtime`, `turnStarted`, or `turnCompleted`. Turn boundaries carry lifecycle metadata without duplicating the turn's items; completed boundaries also cover interrupted and failed turns. Each response contains an opaque continuation cursor and `activeRealtimeSessionAtPageStart`, allowing clients to render any bounded page without loading earlier thread history. Entries at the same rollout position have stable ordering and can span pages. Existing `thread/items/list` remains unchanged. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start`. Inline reviews emit `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. Detached reviews stream ordinary turn items on the new review thread. Parent-owned Multi-Agent V2 subagents reject both inline and detached reviews. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). - `command/exec/write` — write base64-decoded stdin bytes to a running `command/exec` session or close stdin; returns `{}`. @@ -1565,7 +1566,9 @@ All filesystem paths in this section must be absolute. Event notifications are the server-initiated event stream for thread lifecycles, turn lifecycles, and the items within them. After you start or resume a thread, keep reading stdout for `thread/started`, `thread/archived`, `thread/unarchived`, `thread/closed`, `turn/*`, and `item/*` notifications. -Thread realtime uses a separate thread-scoped notification surface. `thread/realtime/*` notifications are ephemeral transport events, not `ThreadItem`s, and are not returned by `thread/read`, `thread/resume`, or `thread/fork`. +Thread realtime publishes thread-scoped timeline item lifecycle notifications for paginated threads alongside its existing realtime notifications. Completed timeline items are durably interleaved with ordinary turn items by `thread/timeline/list`. Neither surface changes `ThreadItem`, `thread/read`, `thread/resume`, or `thread/fork`; clients ignore notification methods they do not recognize. + +Each realtime item has an `id`, a `realtimeSessionId`, and one of four types: `realtimeSessionStarted`, `transcriptSegment`, `bemItemPromoted`, or `realtimeSessionClosed`. A `bemItemPromoted` item references an existing backing-agent item by `turnId` and `itemId`; its `presentation` is `wholeItem`, `inlineMarkdown`, or `inlineVisualization` with an `index`. Recoverable configuration and initialization warnings use the existing `configWarning` notification: `{ summary, details?, path?, range? }`. App-server may emit it during initialization for config parsing and related setup diagnostics, or to the requesting connection during `thread/start` when that thread's exec-policy rules fail to parse. @@ -1600,6 +1603,9 @@ The thread realtime API emits thread-scoped notifications for session lifecycle - `thread/realtime/itemAdded` — `{ threadId, item }` for raw non-audio realtime items that do not have a dedicated typed app-server notification, including `handoff_request` (experimental). `item` is forwarded as raw JSON while the upstream websocket item schema remains unstable. - `thread/realtime/transcript/delta` — `{ threadId, role, delta }` for live realtime transcript deltas (experimental). - `thread/realtime/transcript/done` — `{ threadId, role, text }` when realtime emits the final full text for a transcript part (experimental). +- `thread/realtime/item/started` — `{ threadId, item }` when a realtime item begins. Session boundaries and artifacts complete immediately; transcript segment IDs remain stable through streaming and persistence (experimental). +- `thread/realtime/item/transcript/delta` — `{ threadId, itemId, delta }` for text appended to a started transcript segment (experimental). +- `thread/realtime/item/completed` — `{ threadId, item }` after a session boundary, transcript segment, or promoted backing-agent artifact has been durably committed (experimental). - `thread/realtime/outputAudio/delta` — `{ threadId, audio }` for streamed output audio chunks (experimental). `audio` uses camelCase fields (`data`, `sampleRate`, `numChannels`, `samplesPerChannel`). - `thread/realtime/error` — `{ threadId, message }` when realtime encounters a transport or backend error (experimental). - `thread/realtime/closed` — `{ threadId, reason }` when the realtime transport closes (experimental). diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index e253481a98..3aedcf9500 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -121,6 +121,8 @@ mod models; mod models_refresh_worker; mod otel_reloader; mod outgoing_message; +mod realtime_event_handling; +mod realtime_history; mod request_processors; mod request_serialization; mod server_request_error; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 5d60a6c6fd..f9c6e24b87 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1485,6 +1485,9 @@ impl MessageProcessor { .thread_realtime_stop(&request_id, params) .await } + ClientRequest::ThreadTimelineList { params, .. } => { + self.thread_processor.thread_timeline_list(params).await + } ClientRequest::ThreadRealtimeListVoices { params: _, .. } => { self.turn_processor.thread_realtime_list_voices().await } diff --git a/codex-rs/app-server/src/realtime_event_handling.rs b/codex-rs/app-server/src/realtime_event_handling.rs new file mode 100644 index 0000000000..cc4a29e4ec --- /dev/null +++ b/codex-rs/app-server/src/realtime_event_handling.rs @@ -0,0 +1,91 @@ +use crate::outgoing_message::ThreadScopedOutgoingMessageSender; +use crate::realtime_history::RealtimeEventEffects; +use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::ThreadRealtimeItemCompletedNotification; +use codex_app_server_protocol::ThreadRealtimeItemStartedNotification; +use codex_app_server_protocol::ThreadRealtimeItemTranscriptDeltaNotification; +use codex_core::CodexThread; +use codex_protocol::ThreadId; +use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_rollout::RolloutItem; +use tracing::warn; + +pub(crate) async fn apply_realtime_event_effects( + conversation: &CodexThread, + outgoing: &ThreadScopedOutgoingMessageSender, + thread_id: ThreadId, + effects: RealtimeEventEffects, +) { + let thread_id = thread_id.to_string(); + + if let Some(stream) = effects.transcript_stream { + if let Some(item) = stream.started_item { + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeItemStarted( + ThreadRealtimeItemStartedNotification { + thread_id: thread_id.clone(), + item: item.into(), + }, + )) + .await; + } + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeItemTranscriptDelta( + ThreadRealtimeItemTranscriptDeltaNotification { + thread_id: thread_id.clone(), + item_id: stream.item_id, + delta: stream.delta, + }, + )) + .await; + } + + if let Err(error) = + persist_realtime_items(conversation, outgoing, &thread_id, effects.items).await + { + warn!(thread_id, "failed to persist realtime history: {error}"); + } +} + +pub(crate) async fn persist_realtime_items( + conversation: &CodexThread, + outgoing: &ThreadScopedOutgoingMessageSender, + thread_id: &str, + items: Vec, +) -> Result<(), String> { + if items.is_empty() { + return Ok(()); + } + conversation + .append_rollout_items( + &items + .iter() + .cloned() + .map(RolloutItem::RealtimeItem) + .collect::>(), + ) + .await + .map_err(|error| format!("failed to persist realtime history: {error}"))?; + for item in items { + if !matches!(&item.content, RealtimeItemContent::TranscriptSegment { .. }) { + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeItemStarted( + ThreadRealtimeItemStartedNotification { + thread_id: thread_id.to_string(), + item: item.clone().into(), + }, + )) + .await; + } + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeItemCompleted( + ThreadRealtimeItemCompletedNotification { + thread_id: thread_id.to_string(), + item: item.into(), + }, + )) + .await; + } + Ok(()) +} diff --git a/codex-rs/app-server/src/realtime_history.rs b/codex-rs/app-server/src/realtime_history.rs new file mode 100644 index 0000000000..507632f8e0 --- /dev/null +++ b/codex-rs/app-server/src/realtime_history.rs @@ -0,0 +1,534 @@ +use codex_protocol::items::AgentMessageContent; +use codex_protocol::items::DynamicToolCallStatus; +use codex_protocol::items::McpToolCallStatus; +use codex_protocol::items::TurnItem; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::RealtimeEvent; +use codex_protocol::protocol::RealtimeTranscriptDelta; +use codex_protocol::protocol::RealtimeTranscriptDone; +use codex_protocol::protocol::SubAgentActivityKind; +use codex_protocol::realtime::BemItemPresentation; +use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_protocol::realtime::RealtimeSessionOutcome; +use codex_protocol::realtime::RealtimeTranscriptRole; +use codex_protocol::user_input::UserInput; +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; +use uuid::Uuid; + +const INLINE_MARKDOWN_DIRECTIVE: &str = "::codex-realtime-inline{}"; +const INLINE_VISUALIZATION_DIRECTIVE: &str = "::codex-inline-vis{"; +const VISUALIZE_DIRECTIVE: &str = "visualize{"; +const BACKTICK_FENCE: &str = "```"; +const TILDE_FENCE: &str = "~~~"; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ActiveSegment { + session_id: String, + id: String, + role: RealtimeTranscriptRole, + text: String, +} + +#[derive(Debug, Default)] +struct ActiveTranscriptSegments { + user: Option, + assistant: Option, + first_active_role: Option, +} + +impl ActiveTranscriptSegments { + fn slot_mut(&mut self, role: RealtimeTranscriptRole) -> &mut Option { + match role { + RealtimeTranscriptRole::User => &mut self.user, + RealtimeTranscriptRole::Assistant => &mut self.assistant, + } + } + + fn take(&mut self, role: RealtimeTranscriptRole) -> Option { + let segment = self.slot_mut(role).take(); + if self.first_active_role == Some(role) { + self.first_active_role = self + .user + .as_ref() + .or(self.assistant.as_ref()) + .map(|segment| segment.role); + } + segment + } +} + +#[derive(Debug)] +struct StreamingAgentMessage { + item_id: String, + text: String, +} + +#[derive(Debug)] +pub(crate) struct RealtimeTranscriptStream { + pub(crate) started_item: Option, + pub(crate) item_id: String, + pub(crate) delta: String, +} + +#[derive(Debug, Default)] +pub(crate) struct RealtimeEventEffects { + pub(crate) items: Vec, + pub(crate) transcript_stream: Option, +} + +#[derive(Clone, Copy)] +enum Continuation { + Continue, + Finish, +} + +/// Retains only live session state; durable history is served by the rollout index. +#[derive(Debug, Default)] +pub(crate) struct RealtimeHistoryState { + active_session_id: Option, + active_segments: ActiveTranscriptSegments, + streaming_agent_message: Option, + realtime_session_by_bem_turn: HashMap, + promoted_bem_presentation_keys: HashSet, + pending_handoffs: VecDeque, + failed: bool, +} + +impl RealtimeHistoryState { + pub(crate) fn should_seal_user_input(&self, input: &[UserInput]) -> bool { + self.active_session_id.is_some() + && [&self.active_segments.user, &self.active_segments.assistant] + .into_iter() + .flatten() + .any(|segment| !segment.text.is_empty()) + && !matches!(input, [UserInput::Text { text, .. }] if { + let text = text.trim(); + text.starts_with("") + && text.ends_with("") + }) + } + + pub(crate) fn seal_user_input(&mut self, input: &[UserInput]) -> Vec { + if !self.should_seal_user_input(input) { + return Vec::new(); + } + let mut items = Vec::new(); + self.seal_segments(&mut items, Continuation::Continue); + items + } + + pub(crate) fn should_observe(&self, event: &EventMsg) -> bool { + matches!(event, EventMsg::RealtimeConversationStarted(_)) + || (self.active_session_id.is_some() + && matches!( + event, + EventMsg::RealtimeConversationRealtime(_) + | EventMsg::RealtimeConversationClosed(_) + | EventMsg::TurnStarted(_) + | EventMsg::ItemStarted(_) + | EventMsg::ItemCompleted(_) + | EventMsg::AgentMessageContentDelta(_) + )) + || match event { + EventMsg::ItemStarted(event) => self + .realtime_session_by_bem_turn + .contains_key(&event.turn_id), + EventMsg::ItemCompleted(event) => self + .realtime_session_by_bem_turn + .contains_key(&event.turn_id), + EventMsg::AgentMessageContentDelta(event) => self + .realtime_session_by_bem_turn + .contains_key(&event.turn_id), + EventMsg::TurnStarted(_) => !self.pending_handoffs.is_empty(), + _ => false, + } + } + + pub(crate) fn observe( + &mut self, + event: &EventMsg, + active_turn_id: Option<&str>, + ) -> RealtimeEventEffects { + let mut items = Vec::new(); + let mut transcript_stream = None; + match event { + EventMsg::RealtimeConversationStarted(event) => { + let session_id = event + .realtime_session_id + .clone() + .unwrap_or_else(|| Uuid::now_v7().to_string()); + if self.active_session_id.as_deref() != Some(session_id.as_str()) { + self.seal_segments(&mut items, Continuation::Finish); + self.active_session_id = Some(session_id.clone()); + self.failed = false; + items.push(RealtimeItem { + id: Uuid::now_v7().to_string(), + realtime_session_id: session_id.clone(), + content: RealtimeItemContent::RealtimeSessionStarted, + }); + } + if let Some(turn_id) = active_turn_id { + self.realtime_session_by_bem_turn + .insert(turn_id.to_string(), session_id); + } + } + EventMsg::RealtimeConversationRealtime(event) => match &event.payload { + RealtimeEvent::InputTranscriptDelta(delta) => { + transcript_stream = self.add_delta(RealtimeTranscriptRole::User, delta); + } + RealtimeEvent::OutputTranscriptDelta(delta) => { + transcript_stream = self.add_delta(RealtimeTranscriptRole::Assistant, delta); + } + RealtimeEvent::InputTranscriptDone(done) => { + transcript_stream = + self.finish_segment(&mut items, RealtimeTranscriptRole::User, done); + } + RealtimeEvent::OutputTranscriptDone(done) => { + transcript_stream = + self.finish_segment(&mut items, RealtimeTranscriptRole::Assistant, done); + } + RealtimeEvent::HandoffRequested(_) => { + if let Some(session_id) = &self.active_session_id { + self.pending_handoffs.push_back(session_id.clone()); + } + } + RealtimeEvent::Error(_) => self.failed = true, + _ => {} + }, + EventMsg::TurnStarted(event) => { + if let Some(session_id) = self + .pending_handoffs + .pop_front() + .or_else(|| self.active_session_id.clone()) + { + self.realtime_session_by_bem_turn + .entry(event.turn_id.clone()) + .or_insert(session_id); + } + } + EventMsg::ItemStarted(event) => { + if let TurnItem::UserMessage(item) = &event.item { + items.extend(self.seal_user_input(&item.content)); + } + self.observe_item( + &mut items, + &event.turn_id, + &event.item, + /*completed*/ false, + ); + } + EventMsg::ItemCompleted(event) => { + if let TurnItem::UserMessage(item) = &event.item { + items.extend(self.seal_user_input(&item.content)); + } + self.observe_item( + &mut items, + &event.turn_id, + &event.item, + /*completed*/ true, + ); + if self + .streaming_agent_message + .as_ref() + .is_some_and(|message| message.item_id == event.item.id()) + { + self.streaming_agent_message = None; + } + } + EventMsg::AgentMessageContentDelta(event) => { + let message = + self.streaming_agent_message + .get_or_insert_with(|| StreamingAgentMessage { + item_id: event.item_id.clone(), + text: String::new(), + }); + if message.item_id != event.item_id { + message.item_id.clone_from(&event.item_id); + message.text.clear(); + } + message.text.push_str(&event.delta); + let text = message.text.clone(); + self.observe_assistant_message(&mut items, &event.turn_id, &event.item_id, &text); + } + EventMsg::RealtimeConversationClosed(_) => { + if let Some(session_id) = self.active_session_id.take() { + self.seal_segments(&mut items, Continuation::Finish); + items.push(RealtimeItem { + id: Uuid::now_v7().to_string(), + realtime_session_id: session_id, + content: RealtimeItemContent::RealtimeSessionClosed { + outcome: if self.failed { + RealtimeSessionOutcome::Failed + } else { + RealtimeSessionOutcome::Ended + }, + }, + }); + self.failed = false; + } + } + _ => {} + } + RealtimeEventEffects { + items, + transcript_stream, + } + } + + fn observe_item( + &mut self, + items: &mut Vec, + turn_id: &str, + item: &TurnItem, + completed: bool, + ) { + match item { + TurnItem::AgentMessage(message) => { + let text = message + .content + .iter() + .map(|content| match content { + AgentMessageContent::Text { text } => text.as_str(), + }) + .collect::(); + if !completed { + self.streaming_agent_message = Some(StreamingAgentMessage { + item_id: message.id.clone(), + text: text.clone(), + }); + } + self.observe_assistant_message(items, turn_id, &message.id, &text); + } + TurnItem::ImageGeneration(image) => { + self.add_promotion(items, turn_id, &image.id, BemItemPresentation::WholeItem); + } + TurnItem::Extension(extension) + if serde_json::to_value(extension) + .is_ok_and(|item| item["kind"] == "image_gen.generation") => + { + self.add_promotion( + items, + turn_id, + extension.id(), + BemItemPresentation::WholeItem, + ); + } + TurnItem::SubAgentActivity(activity) + if completed && activity.kind == SubAgentActivityKind::Started => + { + self.add_promotion(items, turn_id, &activity.id, BemItemPresentation::WholeItem); + } + TurnItem::DynamicToolCall(call) + if self.active_session_id.is_some() + && completed + && call.status == DynamicToolCallStatus::Completed + && call.success == Some(true) => + { + self.add_promotion(items, turn_id, &call.id, BemItemPresentation::WholeItem); + } + TurnItem::McpToolCall(call) + if self.active_session_id.is_some() + && completed + && call.server == "codex_app" + && call.status == McpToolCallStatus::Completed => + { + self.add_promotion(items, turn_id, &call.id, BemItemPresentation::WholeItem); + } + _ => {} + } + } + + fn observe_assistant_message( + &mut self, + items: &mut Vec, + turn_id: &str, + item_id: &str, + text: &str, + ) { + let mut lines = text.trim_start().lines(); + let mut first = lines.next().unwrap_or_default(); + if first.starts_with('[') + && let Some((_, content)) = first.split_once(']') + { + first = content.trim_start(); + if first.is_empty() { + first = lines.next().unwrap_or_default(); + } + } + if first == INLINE_MARKDOWN_DIRECTIVE && text.contains('\n') { + self.add_promotion(items, turn_id, item_id, BemItemPresentation::InlineMarkdown); + return; + } + + let mut in_fence = false; + let mut visualization_index = 0; + for line in text.lines() { + let trimmed = line.trim_start(); + if trimmed.starts_with(BACKTICK_FENCE) || trimmed.starts_with(TILDE_FENCE) { + in_fence = !in_fence; + continue; + } + if !in_fence + && (trimmed.starts_with(INLINE_VISUALIZATION_DIRECTIVE) + || trimmed.starts_with(VISUALIZE_DIRECTIVE)) + { + self.add_promotion( + items, + turn_id, + item_id, + BemItemPresentation::InlineVisualization { + index: visualization_index, + }, + ); + visualization_index += 1; + } + } + } + + fn add_promotion( + &mut self, + items: &mut Vec, + turn_id: &str, + item_id: &str, + presentation: BemItemPresentation, + ) { + let Some(realtime_session_id) = self.realtime_session_by_bem_turn.get(turn_id).cloned() + else { + return; + }; + let presentation_key = match &presentation { + BemItemPresentation::WholeItem => format!("{item_id}:whole-item"), + BemItemPresentation::InlineMarkdown => format!("{item_id}:inline-markdown"), + BemItemPresentation::InlineVisualization { index } => { + format!("{item_id}:inline-visualization:{index}") + } + }; + if !self.promoted_bem_presentation_keys.insert(presentation_key) { + return; + } + self.seal_segments(items, Continuation::Continue); + items.push(RealtimeItem { + id: Uuid::now_v7().to_string(), + realtime_session_id, + content: RealtimeItemContent::BemItemPromoted { + turn_id: turn_id.to_string(), + item_id: item_id.to_string(), + presentation, + }, + }); + } + + fn add_delta( + &mut self, + role: RealtimeTranscriptRole, + delta: &RealtimeTranscriptDelta, + ) -> Option { + let session_id = self.active_session_id.clone()?; + self.active_segments.first_active_role.get_or_insert(role); + let segment = self + .active_segments + .slot_mut(role) + .get_or_insert_with(|| ActiveSegment { + session_id, + id: Uuid::now_v7().to_string(), + role, + text: String::new(), + }); + let started_item = segment.text.is_empty().then(|| RealtimeItem { + id: segment.id.clone(), + realtime_session_id: segment.session_id.clone(), + content: RealtimeItemContent::TranscriptSegment { + role: segment.role, + text: String::new(), + }, + }); + segment.text.push_str(&delta.delta); + Some(RealtimeTranscriptStream { + started_item, + item_id: segment.id.clone(), + delta: delta.delta.clone(), + }) + } + + fn finish_segment( + &mut self, + items: &mut Vec, + role: RealtimeTranscriptRole, + done: &RealtimeTranscriptDone, + ) -> Option { + let Some(segment) = self.active_segments.take(role) else { + if done.text.is_empty() { + return None; + } + let stream = self.add_delta( + role, + &RealtimeTranscriptDelta { + delta: done.text.clone(), + }, + ); + if let Some(segment) = self.active_segments.take(role) { + self.seal_active_segment(items, segment, Continuation::Finish); + } + return stream; + }; + // A split leaves an empty continuation; the upstream final may repeat + // text that was already committed before the split. + self.seal_active_segment(items, segment, Continuation::Finish); + None + } + + fn seal_segments(&mut self, items: &mut Vec, continuation: Continuation) { + let mut segments = std::mem::take(&mut self.active_segments); + let roles = match segments.first_active_role { + Some(RealtimeTranscriptRole::Assistant) => [ + RealtimeTranscriptRole::Assistant, + RealtimeTranscriptRole::User, + ], + Some(RealtimeTranscriptRole::User) | None => [ + RealtimeTranscriptRole::User, + RealtimeTranscriptRole::Assistant, + ], + }; + for role in roles { + if let Some(segment) = segments.take(role) { + self.seal_active_segment(items, segment, continuation); + } + } + } + + fn seal_active_segment( + &mut self, + items: &mut Vec, + segment: ActiveSegment, + continuation: Continuation, + ) { + if !segment.text.is_empty() { + items.push(RealtimeItem { + id: segment.id, + realtime_session_id: segment.session_id.clone(), + content: RealtimeItemContent::TranscriptSegment { + role: segment.role, + text: segment.text, + }, + }); + } + if matches!(continuation, Continuation::Continue) { + self.active_segments + .first_active_role + .get_or_insert(segment.role); + *self.active_segments.slot_mut(segment.role) = Some(ActiveSegment { + session_id: segment.session_id, + id: Uuid::now_v7().to_string(), + role: segment.role, + text: String::new(), + }); + } + } +} + +#[cfg(test)] +#[path = "realtime_history_tests.rs"] +mod tests; diff --git a/codex-rs/app-server/src/realtime_history_tests.rs b/codex-rs/app-server/src/realtime_history_tests.rs new file mode 100644 index 0000000000..ad16bcff45 --- /dev/null +++ b/codex-rs/app-server/src/realtime_history_tests.rs @@ -0,0 +1,541 @@ +use super::*; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::items::AgentMessageItem; +use codex_protocol::items::DynamicToolCallItem; +use codex_protocol::items::ImageGenerationItem; +use codex_protocol::items::McpToolCallItem; +use codex_protocol::items::SubAgentActivityItem; +use codex_protocol::protocol::AgentMessageContentDeltaEvent; +use codex_protocol::protocol::ItemCompletedEvent; +use codex_protocol::protocol::ItemStartedEvent; +use codex_protocol::protocol::RealtimeConversationClosedEvent; +use codex_protocol::protocol::RealtimeConversationRealtimeEvent; +use codex_protocol::protocol::RealtimeConversationStartedEvent; +use codex_protocol::protocol::RealtimeConversationVersion; +use pretty_assertions::assert_eq; +use uuid::Uuid; + +fn started_state() -> RealtimeHistoryState { + let mut state = RealtimeHistoryState::default(); + state.observe( + &EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent { + realtime_session_id: Some("voice-1".to_string()), + version: RealtimeConversationVersion::V2, + }), + Some("turn-1"), + ); + state +} + +fn observe_realtime( + state: &mut RealtimeHistoryState, + payload: RealtimeEvent, +) -> RealtimeEventEffects { + state.observe( + &EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { payload }), + /*active_turn_id*/ None, + ) +} + +fn assistant_delta(item_id: &str, delta: &str) -> EventMsg { + EventMsg::AgentMessageContentDelta(AgentMessageContentDeltaEvent { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + item_id: item_id.to_string(), + delta: delta.to_string(), + }) +} + +fn completed_item(item: TurnItem) -> EventMsg { + EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".to_string(), + item, + started_at_ms: None, + completed_at_ms: 0, + }) +} + +fn dynamic_tool_call(id: &str, status: DynamicToolCallStatus) -> TurnItem { + TurnItem::DynamicToolCall(DynamicToolCallItem { + id: id.to_string(), + namespace: Some("another_client".to_string()), + tool: "any_dynamic_tool".to_string(), + arguments: serde_json::Value::Null, + status, + content_items: None, + success: Some(status == DynamicToolCallStatus::Completed), + error: None, + duration: None, + }) +} + +fn mcp_tool_call(id: &str, server: &str, status: McpToolCallStatus) -> TurnItem { + TurnItem::McpToolCall(McpToolCallItem { + id: id.to_string(), + server: server.to_string(), + tool: "create_thread".to_string(), + arguments: serde_json::Value::Null, + connector_id: None, + mcp_app_resource_uri: None, + link_id: None, + app_name: None, + action_name: None, + plugin_id: None, + read_only_hint: None, + status, + result: None, + error: None, + duration: None, + }) +} + +fn contents(effects: RealtimeEventEffects) -> Vec { + effects + .items + .into_iter() + .map(|item| { + assert_eq!(Uuid::parse_str(&item.id).unwrap().get_version_num(), 7); + item.content + }) + .collect() +} + +#[test] +fn promotes_backing_agent_artifacts_once_without_a_client_request() { + let mut state = started_state(); + let first_delta = assistant_delta("message-1", "[analysis] ::codex-realtime-inline{}"); + assert!( + state + .observe(&first_delta, /*active_turn_id*/ None) + .items + .is_empty() + ); + + let second_delta = assistant_delta("message-1", "\nVisible explanation"); + assert_eq!( + contents(state.observe(&second_delta, /*active_turn_id*/ None)), + vec![RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "message-1".to_string(), + presentation: BemItemPresentation::InlineMarkdown, + }] + ); + + let completed = completed_item(TurnItem::AgentMessage(AgentMessageItem { + id: "message-1".to_string(), + content: vec![AgentMessageContent::Text { + text: "[analysis] ::codex-realtime-inline{}\nVisible explanation".to_string(), + }], + phase: None, + memory_citation: None, + delivery: None, + })); + assert!( + state + .observe(&completed, /*active_turn_id*/ None) + .items + .is_empty() + ); + + let next_message = assistant_delta("message-2", "::codex-realtime-inline{}\nNext message"); + assert_eq!( + contents(state.observe(&next_message, /*active_turn_id*/ None)), + vec![RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "message-2".to_string(), + presentation: BemItemPresentation::InlineMarkdown, + }] + ); + + let image = EventMsg::ItemStarted(ItemStartedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".to_string(), + item: TurnItem::ImageGeneration(ImageGenerationItem { + id: "image-1".to_string(), + status: "in_progress".to_string(), + revised_prompt: None, + result: String::new(), + saved_path: None, + }), + started_at_ms: 0, + }); + let subagent = completed_item(TurnItem::SubAgentActivity(SubAgentActivityItem { + id: "subagent-1".to_string(), + kind: SubAgentActivityKind::Started, + agent_thread_id: ThreadId::new(), + agent_path: AgentPath::root(), + })); + for (event, item_id) in [(image, "image-1"), (subagent, "subagent-1")] { + assert_eq!( + contents(state.observe(&event, /*active_turn_id*/ None)), + vec![RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: item_id.to_string(), + presentation: BemItemPresentation::WholeItem, + }] + ); + } +} + +#[test] +fn promotes_distinct_visualizations_once_and_ignores_markdown_fences() { + let mut state = started_state(); + let text = "```\n::codex-inline-vis{file=hidden}\n```\n~~~\nvisualize{file=also-hidden}\n~~~\n::codex-inline-vis{file=first}\nvisualize{file=second}"; + let message = assistant_delta("message-1", text); + assert_eq!( + contents(state.observe(&message, /*active_turn_id*/ None)), + (0..2) + .map(|index| RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "message-1".to_string(), + presentation: BemItemPresentation::InlineVisualization { index }, + }) + .collect::>() + ); + + let completed = completed_item(TurnItem::AgentMessage(AgentMessageItem { + id: "message-1".to_string(), + content: vec![AgentMessageContent::Text { + text: text.to_string(), + }], + phase: None, + memory_citation: None, + delivery: None, + })); + assert!( + state + .observe(&completed, /*active_turn_id*/ None) + .items + .is_empty() + ); +} + +#[test] +fn streams_stable_segment_items_and_ignores_upstream_transcript_revisions() { + let mut state = started_state(); + let effects = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "hello".to_string(), + }), + ); + assert!(effects.items.is_empty()); + let stream = effects.transcript_stream.expect("streaming delta"); + let started = stream.started_item.expect("segment start"); + assert_eq!(stream.item_id, started.id); + assert_eq!(Uuid::parse_str(&started.id).unwrap().get_version_num(), 7); + assert_eq!(stream.delta, "hello"); + + let done = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDone(RealtimeTranscriptDone { + text: "a substantially different upstream revision".to_string(), + }), + ); + assert_eq!( + done.items, + vec![RealtimeItem { + id: started.id, + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "hello".to_string(), + }, + }], + ); + assert!(done.transcript_stream.is_none()); +} + +#[test] +fn emits_the_complete_item_lifecycle_when_only_a_final_transcript_arrives() { + let mut state = started_state(); + let effects = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDone(RealtimeTranscriptDone { + text: "final transcript".to_string(), + }), + ); + let stream = effects.transcript_stream.expect("final transcript stream"); + let started = stream.started_item.expect("transcript item started"); + assert_eq!(stream.item_id, started.id); + assert_eq!(stream.delta, "final transcript"); + assert_eq!( + effects.items, + vec![RealtimeItem { + id: started.id, + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "final transcript".to_string(), + }, + }] + ); +} + +#[test] +fn preserves_first_transcript_activity_order_when_both_speakers_are_active() { + for roles in [ + [ + RealtimeTranscriptRole::Assistant, + RealtimeTranscriptRole::User, + ], + [ + RealtimeTranscriptRole::User, + RealtimeTranscriptRole::Assistant, + ], + ] { + let mut state = started_state(); + for role in roles { + let delta = RealtimeTranscriptDelta { + delta: format!("{role:?} speech"), + }; + observe_realtime( + &mut state, + match role { + RealtimeTranscriptRole::User => RealtimeEvent::InputTranscriptDelta(delta), + RealtimeTranscriptRole::Assistant => { + RealtimeEvent::OutputTranscriptDelta(delta) + } + }, + ); + } + let items = state + .observe( + &EventMsg::RealtimeConversationClosed(RealtimeConversationClosedEvent { + reason: None, + }), + /*active_turn_id*/ None, + ) + .items; + let observed_roles = items + .iter() + .filter_map(|item| match item.content { + RealtimeItemContent::TranscriptSegment { role, .. } => Some(role), + _ => None, + }) + .collect::>(); + assert_eq!(observed_roles, roles); + } +} + +#[test] +fn does_not_replay_a_final_transcript_after_a_promotion_split() { + let mut state = started_state(); + observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "Already spoken".to_string(), + }), + ); + let promotion = completed_item(dynamic_tool_call( + "successful-tool", + DynamicToolCallStatus::Completed, + )); + assert_eq!( + state + .observe(&promotion, /*active_turn_id*/ None) + .items + .len(), + 2 + ); + + let done = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDone(RealtimeTranscriptDone { + text: "Already spoken, with an upstream revision".to_string(), + }), + ); + assert!(done.items.is_empty()); + assert!(done.transcript_stream.is_none()); +} + +#[test] +fn generates_distinct_boundary_ids_for_reused_realtime_sessions() { + let mut state = RealtimeHistoryState::default(); + let mut boundary_ids = Vec::new(); + for _ in 0..2 { + let started = state.observe( + &EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent { + realtime_session_id: Some("reused-session".to_string()), + version: RealtimeConversationVersion::V2, + }), + /*active_turn_id*/ None, + ); + let closed = state.observe( + &EventMsg::RealtimeConversationClosed(RealtimeConversationClosedEvent { reason: None }), + /*active_turn_id*/ None, + ); + for item in started.items.into_iter().chain(closed.items) { + assert_eq!(item.realtime_session_id, "reused-session"); + assert_eq!(Uuid::parse_str(&item.id).unwrap().get_version_num(), 7); + boundary_ids.push(item.id); + } + } + let unique_ids = boundary_ids + .iter() + .collect::>(); + assert_eq!(unique_ids.len(), 4); + + let fallback = state.observe( + &EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent { + realtime_session_id: None, + version: RealtimeConversationVersion::V2, + }), + /*active_turn_id*/ None, + ); + assert_eq!(fallback.items.len(), 1); + assert_eq!( + Uuid::parse_str(&fallback.items[0].realtime_session_id) + .unwrap() + .get_version_num(), + 7 + ); +} + +#[test] +fn promotes_successful_codex_app_mcp_calls_and_splits_active_transcripts() { + let mut state = started_state(); + observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "Before tool.".to_string(), + }), + ); + for (server, status) in [ + ("other_server", McpToolCallStatus::Completed), + ("codex_app", McpToolCallStatus::InProgress), + ("codex_app", McpToolCallStatus::Failed), + ] { + assert!( + state + .observe( + &completed_item(mcp_tool_call("not-promoted", server, status)), + /*active_turn_id*/ None, + ) + .items + .is_empty() + ); + } + let completed = completed_item(mcp_tool_call( + "app-tool", + "codex_app", + McpToolCallStatus::Completed, + )); + assert_eq!( + contents(state.observe(&completed, /*active_turn_id*/ None)), + vec![ + RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "Before tool.".to_string(), + }, + RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "app-tool".to_string(), + presentation: BemItemPresentation::WholeItem, + }, + ] + ); + assert!( + state + .observe(&completed, /*active_turn_id*/ None) + .items + .is_empty() + ); +} + +#[test] +fn promotes_successful_dynamic_tools_and_splits_active_transcripts() { + let mut state = started_state(); + let first = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "Before tool. ".to_string(), + }), + ) + .transcript_stream + .expect("first transcript stream"); + + let failed = completed_item(dynamic_tool_call( + "failed-tool", + DynamicToolCallStatus::Failed, + )); + assert!( + state + .observe(&failed, /*active_turn_id*/ None) + .items + .is_empty() + ); + + let completed = completed_item(dynamic_tool_call( + "successful-tool", + DynamicToolCallStatus::Completed, + )); + let items = state.observe(&completed, /*active_turn_id*/ None).items; + assert_eq!(items.len(), 2); + assert_eq!( + items[0], + RealtimeItem { + id: first.item_id.clone(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "Before tool. ".to_string(), + }, + } + ); + assert_eq!( + items[1].content, + RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "successful-tool".to_string(), + presentation: BemItemPresentation::WholeItem, + } + ); + assert_eq!(Uuid::parse_str(&items[1].id).unwrap().get_version_num(), 7); + + let second = observe_realtime( + &mut state, + RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "After tool.".to_string(), + }), + ) + .transcript_stream + .expect("second transcript stream"); + assert_ne!(first.item_id, second.item_id); + assert!(second.started_item.is_some()); + assert!( + state + .observe(&completed, /*active_turn_id*/ None) + .items + .is_empty() + ); + + let closed = state.observe( + &EventMsg::RealtimeConversationClosed(RealtimeConversationClosedEvent { reason: None }), + /*active_turn_id*/ None, + ); + assert_eq!(closed.items[0].id, second.item_id); + let late = completed_item(dynamic_tool_call( + "late-tool", + DynamicToolCallStatus::Completed, + )); + assert!( + state + .observe(&late, /*active_turn_id*/ None) + .items + .is_empty() + ); + + state.observe( + &EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent { + realtime_session_id: Some("voice-2".to_string()), + version: RealtimeConversationVersion::V2, + }), + Some("turn-1"), + ); + let resumed = state.observe(&late, /*active_turn_id*/ None).items; + assert_eq!(resumed.len(), 1); + assert_eq!(resumed[0].realtime_session_id, "voice-2"); +} diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 28d48e299f..9c0f6b7567 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -284,6 +284,8 @@ use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStartedNotification; use codex_app_server_protocol::ThreadStatus; +use codex_app_server_protocol::ThreadTimelineListParams; +use codex_app_server_protocol::ThreadTimelineListResponse; use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadTurnsListResponse; use codex_app_server_protocol::ThreadUnarchiveParams; @@ -481,6 +483,7 @@ use codex_thread_store::GitInfoPatch as StoreGitInfoPatch; use codex_thread_store::ItemSortKey as StoreItemSortKey; use codex_thread_store::ListItemsParams as StoreListItemsParams; use codex_thread_store::ListThreadsParams as StoreListThreadsParams; +use codex_thread_store::ListTimelineParams as StoreListTimelineParams; use codex_thread_store::ListTurnsParams as StoreListTurnsParams; use codex_thread_store::LoadThreadHistoryParams as StoreLoadThreadHistoryParams; use codex_thread_store::LocalThreadStore; diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 2cbf81d085..7060ec4f32 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -1,8 +1,12 @@ use super::*; use crate::extensions::send_thread_warning; +use crate::realtime_event_handling::apply_realtime_event_effects; +use crate::realtime_event_handling::persist_realtime_items; +use crate::realtime_history::RealtimeEventEffects; use codex_app_server_protocol::ThreadQueueChangedNotification; use codex_extension_api::ThreadIdleCause; use codex_protocol::config_types::MultiAgentMode; +use codex_protocol::protocol::ThreadHistoryMode; pub(super) const THREAD_UNLOADING_DELAY: Duration = Duration::from_secs(30 * 60); @@ -242,8 +246,10 @@ pub(super) async fn ensure_listener_task_running( &environments, ) .await; - let thread_settings_baseline = - thread_settings_from_config_snapshot(&conversation.config_snapshot().await); + let config_snapshot = conversation.config_snapshot().await; + let realtime_history_enabled = + matches!(config_snapshot.history_mode, ThreadHistoryMode::Paginated); + let thread_settings_baseline = thread_settings_from_config_snapshot(&config_snapshot); let (mut listener_command_rx, listener_generation) = { let mut thread_state = thread_state.lock().await; if thread_state.listener_matches(&conversation) { @@ -325,10 +331,20 @@ pub(super) async fn ensure_listener_task_running( // Track the event before emitting any typed translations // so thread-local state such as raw event opt-in stays // synchronized with the conversation. - let raw_events_enabled = { + let (raw_events_enabled, realtime_effects) = { let mut thread_state = thread_state.lock().await; thread_state.track_current_turn_event(&event.id, &event.msg); - thread_state.experimental_raw_events + let realtime_effects = if realtime_history_enabled + && thread_state.realtime_history.should_observe(&event.msg) + { + let active_turn_id = thread_state.active_turn_snapshot().map(|turn| turn.id); + thread_state + .realtime_history + .observe(&event.msg, active_turn_id.as_deref()) + } else { + RealtimeEventEffects::default() + }; + (thread_state.experimental_raw_events, realtime_effects) }; if matches!( &event.msg, @@ -346,6 +362,14 @@ pub(super) async fn ensure_listener_task_running( conversation_id, ); + apply_realtime_event_effects( + conversation.as_ref(), + &thread_outgoing, + conversation_id, + realtime_effects, + ) + .await; + apply_bespoke_event_handling( event.clone(), conversation_id, @@ -558,6 +582,32 @@ pub(super) async fn handle_thread_listener_command( .await; let _ = completion_tx.send(()); } + ThreadListenerCommand::SealRealtimeUserInput { + input, + completion_tx, + } => { + let items = thread_state + .lock() + .await + .realtime_history + .seal_user_input(&input); + let subscribed_connection_ids = thread_state_manager + .subscribed_connection_ids(conversation_id) + .await; + let thread_outgoing = ThreadScopedOutgoingMessageSender::new( + outgoing.clone(), + subscribed_connection_ids, + conversation_id, + ); + let result = persist_realtime_items( + conversation.as_ref(), + &thread_outgoing, + &conversation_id.to_string(), + items, + ) + .await; + let _ = completion_tx.send(result); + } } } diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index be6c016a11..686d5dd201 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -870,6 +870,35 @@ impl ThreadRequestProcessor { .map(|response| Some(response.into())) } + pub(crate) async fn thread_timeline_list( + &self, + params: ThreadTimelineListParams, + ) -> Result, JSONRPCErrorError> { + let thread_id = ThreadId::from_string(¶ms.thread_id) + .map_err(|error| invalid_request(format!("invalid thread id: {error}")))?; + let page = self + .thread_store + .list_timeline(StoreListTimelineParams { + thread_id, + cursor: params.cursor, + page_size: params + .limit + .map(|limit| limit as usize) + .unwrap_or(THREAD_ITEMS_DEFAULT_LIMIT) + .clamp(1, THREAD_ITEMS_MAX_LIMIT), + }) + .await + .map_err(paginated_history_list_error)?; + Ok(Some( + ThreadTimelineListResponse { + data: page.items, + next_cursor: page.next_cursor, + active_realtime_session_at_page_start: page.active_realtime_session_at_page_start, + } + .into(), + )) + } + pub(crate) async fn thread_shell_command( &self, request_id: &ConnectionRequestId, diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index c74d279df5..5dfd2c5701 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -528,6 +528,9 @@ impl TurnRequestProcessor { }, ) .await?; + self.seal_realtime_transcript_before_user_input(thread_id, &mapped_items) + .await?; + let submission = thread .start_or_steer_turn( TurnInputRequest::new(TurnInput::UserInput { @@ -901,12 +904,12 @@ impl TurnRequestProcessor { request_id: &ConnectionRequestId, params: TurnSteerParams, ) -> Result { - let (_, thread) = self - .load_thread(¶ms.thread_id) - .await - .inspect_err(|error| { - self.track_error_response(request_id, error, /*error_type*/ None); - })?; + let (thread_id, thread) = + self.load_thread(¶ms.thread_id) + .await + .inspect_err(|error| { + self.track_error_response(request_id, error, /*error_type*/ None); + })?; self.ensure_direct_input_allowed(request_id, thread.as_ref()) .await?; @@ -932,6 +935,9 @@ impl TurnRequestProcessor { .collect(); let additional_context = map_additional_context(params.additional_context); + self.seal_realtime_transcript_before_user_input(thread_id, &mapped_items) + .await?; + let submission = thread .steer_turn( TurnInputRequest::new(TurnInput::UserInput { @@ -1027,6 +1033,37 @@ impl TurnRequestProcessor { Ok(TurnSteerResponse { turn_id }) } + async fn seal_realtime_transcript_before_user_input( + &self, + thread_id: ThreadId, + input: &[CoreInputItem], + ) -> Result<(), JSONRPCErrorError> { + let thread_state = self.thread_state_manager.thread_state(thread_id).await; + if !thread_state + .lock() + .await + .realtime_history + .should_seal_user_input(input) + { + return Ok(()); + } + let listener = self + .thread_state_manager + .current_listener_command_tx(thread_id) + .ok_or_else(|| internal_error("thread listener is not running"))?; + let (completion_tx, completion_rx) = tokio::sync::oneshot::channel(); + listener + .send(ThreadListenerCommand::SealRealtimeUserInput { + input: input.to_vec(), + completion_tx, + }) + .map_err(|_| internal_error("thread listener is not running"))?; + completion_rx + .await + .map_err(|_| internal_error("thread listener stopped before sealing realtime input"))? + .map_err(internal_error) + } + async fn prepare_realtime_conversation_thread( &self, request_id: &ConnectionRequestId, diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index 3795638cdd..402783151e 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -1,5 +1,6 @@ use crate::outgoing_message::ConnectionId; use crate::outgoing_message::ConnectionRequestId; +use crate::realtime_history::RealtimeHistoryState; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadGoal; use codex_app_server_protocol::ThreadHistoryBuilder; @@ -17,6 +18,7 @@ use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::models::MessagePhase; use codex_protocol::protocol::EventMsg; +use codex_protocol::user_input::UserInput; use codex_rollout::RolloutItem; use codex_rollout::state_db::StateDbHandle; use codex_utils_path_uri::LegacyAppPathString; @@ -81,6 +83,10 @@ pub(crate) enum ThreadListenerCommand { request_id: RequestId, completion_tx: oneshot::Sender<()>, }, + SealRealtimeUserInput { + input: Vec, + completion_tx: oneshot::Sender>, + }, } /// Per-conversation accumulation of the latest states e.g. error message while a turn runs. @@ -104,6 +110,7 @@ pub(crate) struct ThreadState { pub(crate) cancel_tx: Option>, pub(crate) experimental_raw_events: bool, pub(crate) listener_generation: u64, + pub(crate) realtime_history: RealtimeHistoryState, last_thread_settings: Option, listener_command_tx: Option>, current_turn_history: ThreadHistoryBuilder, diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 4a154d0606..b6ff5f97cc 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -113,6 +113,7 @@ use codex_app_server_protocol::ThreadSettingsUpdateParams; use codex_app_server_protocol::ThreadShellCommandParams; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::ThreadTimelineListParams; use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadUnarchiveParams; use codex_app_server_protocol::ThreadUnsubscribeParams; @@ -1191,6 +1192,14 @@ impl TestAppServer { self.send_request("thread/realtime/stop", params).await } + pub async fn send_thread_timeline_list_request( + &mut self, + params: ThreadTimelineListParams, + ) -> anyhow::Result { + self.send_request("thread/timeline/list", Some(serde_json::to_value(params)?)) + .await + } + pub async fn send_thread_realtime_list_voices_request( &mut self, params: ThreadRealtimeListVoicesParams, diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index ba7043a5a1..5f9eaeff4c 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -110,6 +110,7 @@ mod thread_settings_update; mod thread_shell_command; mod thread_start; mod thread_status; +mod thread_timeline; mod thread_unarchive; mod thread_unsubscribe; mod turn_interrupt; diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 38684e79d4..9cf86981bf 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -11,6 +11,7 @@ use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::LoginAccountResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadHistoryMode; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; @@ -22,7 +23,12 @@ use codex_app_server_protocol::ThreadRealtimeAudioChunk; use codex_app_server_protocol::ThreadRealtimeClosedNotification; use codex_app_server_protocol::ThreadRealtimeErrorNotification; use codex_app_server_protocol::ThreadRealtimeInitialItem; +use codex_app_server_protocol::ThreadRealtimeItem; use codex_app_server_protocol::ThreadRealtimeItemAddedNotification; +use codex_app_server_protocol::ThreadRealtimeItemCompletedNotification; +use codex_app_server_protocol::ThreadRealtimeItemContent; +use codex_app_server_protocol::ThreadRealtimeItemStartedNotification; +use codex_app_server_protocol::ThreadRealtimeItemTranscriptDeltaNotification; use codex_app_server_protocol::ThreadRealtimeListVoicesParams; use codex_app_server_protocol::ThreadRealtimeListVoicesResponse; use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification; @@ -37,10 +43,15 @@ use codex_app_server_protocol::ThreadRealtimeTranscriptDeltaNotification; use codex_app_server_protocol::ThreadRealtimeTranscriptDoneNotification; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::ThreadTimelineEntry; +use codex_app_server_protocol::ThreadTimelineListParams; +use codex_app_server_protocol::ThreadTimelineListResponse; use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::TurnSteerParams; +use codex_app_server_protocol::TurnSteerResponse; use codex_app_server_protocol::UserInput as V2UserInput; use codex_features::Feature; use codex_protocol::protocol::CodexResponseHandoffMode; @@ -70,6 +81,7 @@ use std::time::Duration; use tempfile::TempDir; use test_case::test_case; use tokio::time::timeout; +use uuid::Uuid; use wiremock::Match; use wiremock::Mock; use wiremock::MockServer; @@ -643,6 +655,119 @@ fn v2_background_agent_tool_call(call_id: &str, prompt: &str) -> Value { }) } +#[tokio::test] +async fn realtime_conversation_streams_timeline_items() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; + let realtime_server = start_websocket_server(vec![vec![vec![ + session_updated("voice-1"), + json!({ "type": "response.output_text.delta", "delta": "hello" }), + json!({ + "type": "response.output_text.done", + "text": "a substantially different final revision" + }), + ]]]) + .await; + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &responses_server.uri(), + realtime_server.uri(), + /*realtime_enabled*/ true, + StartupContextConfig::Generated, + )?; + + let mut mcp = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .build() + .await?; + mcp.initialize().await?; + login_with_api_key(&mut mcp, "sk-test-key").await?; + + let thread_request = mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + history_mode: Some(ThreadHistoryMode::Paginated), + ..Default::default() + }) + .await?; + let thread: ThreadStartResponse = + timeout(DEFAULT_TIMEOUT, mcp.read_response(thread_request)).await??; + let request = mcp + .send_thread_realtime_start_request(ThreadRealtimeStartParams { + thread_id: thread.thread.id.clone(), + client_managed_handoffs: None, + delegation_ack_filler: None, + flush_transcript_tail_on_session_end: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, + codex_response_handoff_mode: None, + codex_response_handoff_channel_prefixes: None, + model: None, + output_modality: RealtimeOutputModality::Audio, + include_startup_context: None, + initial_items: None, + realtime_start_instructions: None, + realtime_end_instructions: None, + prompt: None, + realtime_session_id: None, + transport: None, + version: None, + voice: None, + }) + .await?; + let _: ThreadRealtimeStartResponse = + timeout(DEFAULT_TIMEOUT, mcp.read_response(request)).await??; + + let session_started = read_notification::( + &mut mcp, + "thread/realtime/item/started", + ) + .await?; + let session_completed = read_notification::( + &mut mcp, + "thread/realtime/item/completed", + ) + .await?; + assert_eq!(session_started.item, session_completed.item); + assert_eq!( + Uuid::parse_str(&session_started.item.id)?.get_version_num(), + 7 + ); + assert!(matches!( + session_completed.item.content, + ThreadRealtimeItemContent::RealtimeSessionStarted + )); + + let started = read_notification::( + &mut mcp, + "thread/realtime/item/started", + ) + .await?; + let delta = read_notification::( + &mut mcp, + "thread/realtime/item/transcript/delta", + ) + .await?; + let completed = read_notification::( + &mut mcp, + "thread/realtime/item/completed", + ) + .await?; + assert_eq!(Uuid::parse_str(&started.item.id)?.get_version_num(), 7); + assert_ne!(started.item.id, session_started.item.id); + assert_eq!(delta.item_id, started.item.id); + assert_eq!(completed.item.id, started.item.id); + assert_eq!(delta.delta, "hello"); + assert!(matches!( + completed.item.content, + ThreadRealtimeItemContent::TranscriptSegment { text, .. } if text == "hello" + )); + + realtime_server.shutdown().await; + Ok(()) +} + #[tokio::test] async fn realtime_conversation_streams_v2_notifications() -> Result<()> { skip_if_no_network!(Ok(())); @@ -729,7 +854,10 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { login_with_api_key(&mut mcp, "sk-test-key").await?; let thread_start_request_id = mcp - .send_thread_start_request_with_auto_env(ThreadStartParams::default()) + .send_thread_start_request_with_auto_env(ThreadStartParams { + history_mode: Some(ThreadHistoryMode::Paginated), + ..Default::default() + }) .await?; let thread_start: ThreadStartResponse = timeout(DEFAULT_TIMEOUT, mcp.read_response(thread_start_request_id)).await??; @@ -910,6 +1038,46 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { assert_eq!(closed.thread_id, output_audio.thread_id); assert_eq!(closed.reason.as_deref(), Some("error")); + let history_request = mcp + .send_thread_timeline_list_request(ThreadTimelineListParams { + thread_id: thread_start.thread.id.clone(), + cursor: None, + limit: Some(100), + }) + .await?; + let history: ThreadTimelineListResponse = + timeout(DEFAULT_TIMEOUT, mcp.read_response(history_request)).await??; + assert!(matches!( + history.data.first(), + Some(ThreadTimelineEntry::Realtime { + item: ThreadRealtimeItem { + content: ThreadRealtimeItemContent::RealtimeSessionStarted, + .. + }, + .. + }) + )); + assert!(history.data.iter().any(|entry| matches!( + entry, + ThreadTimelineEntry::Realtime { + item: ThreadRealtimeItem { + content: ThreadRealtimeItemContent::TranscriptSegment { .. }, + .. + }, + .. + } + ))); + assert!(matches!( + history.data.last(), + Some(ThreadTimelineEntry::Realtime { + item: ThreadRealtimeItem { + content: ThreadRealtimeItemContent::RealtimeSessionClosed { .. }, + .. + }, + .. + }) + )); + let connections = realtime_server.connections(); assert_eq!(connections.len(), 1); let connection = &connections[0]; @@ -984,6 +1152,201 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn realtime_timeline_splits_accepted_steering_and_persists_promoted_artifacts() -> Result<()> +{ + skip_if_no_network!(Ok(())); + + let responses_server = responses::start_mock_server().await; + let (gate_tx, gate_rx) = mpsc::channel(); + Mock::given(method("POST")) + .and(path_regex(".*/responses$")) + .respond_with(GatedSseResponse { + gate_rx: Mutex::new(Some(gate_rx)), + response: responses::sse(vec![ + responses::ev_response_created("response-1"), + responses::ev_assistant_message( + "promoted-message", + "::codex-realtime-inline{}\nVisible artifact", + ), + responses::ev_completed("response-1"), + ]), + }) + .expect(2) + .mount(&responses_server) + .await; + + let mut harness = RealtimeE2eHarness::new_with_main_loop_responses_server( + RealtimeTestVersion::V2, + responses_server, + realtime_sideband(vec![open_realtime_sideband_connection(vec![ + vec![session_updated("voice-steering")], + vec![json!({ + "type": "response.output_text.delta", + "delta": "Spoken before steering" + })], + ])]), + ) + .await?; + let thread_request = harness + .mcp + .send_thread_start_request_with_auto_env(ThreadStartParams { + history_mode: Some(ThreadHistoryMode::Paginated), + ..Default::default() + }) + .await?; + let thread: ThreadStartResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(thread_request)).await??; + harness.thread_id = thread.thread.id; + let realtime_session_id = harness + .start_websocket_realtime() + .await? + .realtime_session_id + .context("realtime started notification should include a session ID")?; + + let turn_request = harness + .mcp + .send_turn_start_request(TurnStartParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "Start work".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn: TurnStartResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(turn_request)).await??; + harness + .read_notification::("turn/started") + .await?; + + harness + .append_text(harness.thread_id.clone(), "Trigger speech") + .await?; + harness + .read_notification::( + "thread/realtime/transcript/delta", + ) + .await?; + + let steering_request = harness + .mcp + .send_turn_steer_request(TurnSteerParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "Accepted steering".to_string(), + text_elements: Vec::new(), + }], + expected_turn_id: turn.turn.id.clone(), + additional_context: None, + client_user_message_id: Some("accepted-steer".to_string()), + responsesapi_client_metadata: None, + }) + .await?; + let accepted: TurnSteerResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(steering_request)).await??; + assert_eq!(accepted.turn_id, turn.turn.id); + + let _ = gate_tx.send(()); + harness + .read_notification::("turn/completed") + .await?; + let page_request = harness + .mcp + .send_thread_timeline_list_request(ThreadTimelineListParams { + thread_id: harness.thread_id.clone(), + cursor: None, + limit: Some(100), + }) + .await?; + let page: ThreadTimelineListResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(page_request)).await??; + let transcript_index = page + .data + .iter() + .position(|entry| { + matches!( + entry, + ThreadTimelineEntry::Realtime { + item: ThreadRealtimeItem { + content: ThreadRealtimeItemContent::TranscriptSegment { text, .. }, + .. + }, + .. + } if text == "Spoken before steering" + ) + }) + .context("accepted steering should seal the active transcript")?; + let steering_index = page + .data + .iter() + .position(|entry| { + matches!( + entry, + ThreadTimelineEntry::Item { + item, + .. + } if matches!( + item.as_ref(), + ThreadItem::UserMessage { client_id, .. } + if client_id.as_deref() == Some("accepted-steer") + ) + ) + }) + .context("accepted steering should be included in the timeline")?; + assert!(transcript_index < steering_index); + assert!(page.data.iter().any(|entry| matches!( + entry, + ThreadTimelineEntry::Realtime { + item: ThreadRealtimeItem { + content: ThreadRealtimeItemContent::BemItemPromoted { item_id, .. }, + .. + }, + .. + } if item_id == "promoted-message" + ))); + for entry in &page.data { + if let ThreadTimelineEntry::Realtime { item, .. } = entry { + assert_eq!(Uuid::parse_str(&item.id)?.get_version_num(), 7); + } + } + + let bounded_request = harness + .mcp + .send_thread_timeline_list_request(ThreadTimelineListParams { + thread_id: harness.thread_id.clone(), + cursor: None, + limit: Some(1), + }) + .await?; + let bounded: ThreadTimelineListResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(bounded_request)).await??; + assert_eq!(bounded.data.len(), 1); + assert_eq!( + bounded.active_realtime_session_at_page_start.as_deref(), + Some(realtime_session_id.as_str()) + ); + let older_request = harness + .mcp + .send_thread_timeline_list_request(ThreadTimelineListParams { + thread_id: harness.thread_id.clone(), + cursor: Some(bounded.next_cursor.context("bounded timeline cursor")?), + limit: Some(1), + }) + .await?; + let older: ThreadTimelineListResponse = + timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(older_request)).await??; + assert_eq!(older.data.len(), 1); + assert_eq!( + older.active_realtime_session_at_page_start.as_deref(), + Some(realtime_session_id.as_str()) + ); + + harness.shutdown().await; + Ok(()) +} + #[tokio::test] async fn realtime_start_can_skip_startup_context() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/app-server/tests/suite/v2/thread_timeline.rs b/codex-rs/app-server/tests/suite/v2/thread_timeline.rs new file mode 100644 index 0000000000..6c799a4dbb --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/thread_timeline.rs @@ -0,0 +1,224 @@ +use anyhow::Result; +use app_test_support::MockResponsesConfig; +use app_test_support::TestAppServer; +use app_test_support::create_mock_responses_server_repeating_assistant; +use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::SortDirection; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadItemsListParams; +use codex_app_server_protocol::ThreadItemsListResponse; +use codex_app_server_protocol::ThreadRealtimeItemContent; +use codex_app_server_protocol::ThreadTimelineEntry; +use codex_app_server_protocol::ThreadTimelineListParams; +use codex_app_server_protocol::ThreadTimelineListResponse; +use codex_protocol::ThreadId; +use codex_protocol::items::TurnItem; +use codex_protocol::items::UserMessageItem; +use codex_protocol::models::BaseInstructions; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ItemCompletedEvent; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_protocol::protocol::TurnCompleteEvent; +use codex_protocol::protocol::TurnStartedEvent; +use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_protocol::realtime::RealtimeTranscriptRole; +use codex_protocol::user_input::UserInput; +use codex_rollout::RolloutItem; +use codex_thread_store::AppendThreadItemsParams; +use codex_thread_store::CreateThreadParams; +use codex_thread_store::LocalThreadStore; +use codex_thread_store::LocalThreadStoreConfig; +use codex_thread_store::PersistContext; +use codex_thread_store::ThreadPersistenceMetadata; +use codex_thread_store::ThreadStore; +use codex_utils_absolute_path::test_support::PathExt; +use pretty_assertions::assert_eq; +use tempfile::TempDir; +use uuid::Uuid; + +#[tokio::test] +async fn timeline_pages_mix_items_and_resolve_the_opening_realtime_session() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + MockResponsesConfig::new(&server.uri()).write(codex_home.path())?; + + let thread_id = ThreadId::default(); + let sqlite = codex_state::SqliteConfig::new_for_testing(codex_home.path().abs()); + let state_db = + codex_state::StateRuntime::init(sqlite.clone(), "mock_provider".to_string()).await?; + let store = LocalThreadStore::new( + LocalThreadStoreConfig { + codex_home: codex_home.path().to_path_buf(), + sqlite, + default_model_provider_id: "mock_provider".to_string(), + }, + Some(state_db), + ); + store + .create_thread(CreateThreadParams { + session_id: thread_id.into(), + thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Cli, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode: ThreadHistoryMode::Paginated, + history_base: None, + subagent_history_start_ordinal: None, + initial_window_id: Uuid::now_v7().to_string(), + metadata: ThreadPersistenceMetadata { + cwd: Some(codex_home.path().to_path_buf()), + model_provider: "mock_provider".to_string(), + memory_mode: ThreadMemoryMode::Enabled, + }, + }) + .await?; + store + .persist_thread(thread_id, PersistContext::Standard) + .await?; + store + .append_items(AppendThreadItemsParams { + thread_id, + items: vec![ + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice-1:started".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }), + RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: "turn-1".to_string(), + trace_id: None, + started_at: Some(10), + model_context_window: None, + collaboration_mode_kind: Default::default(), + })), + RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id, + turn_id: "turn-1".to_string(), + item: TurnItem::UserMessage(UserMessageItem { + id: "user-1".to_string(), + client_id: None, + content: vec![UserInput::Text { + text: "check staging".to_string(), + text_elements: Vec::new(), + }], + }), + started_at_ms: Some(0), + completed_at_ms: 1, + })), + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice-1:transcript-1".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "Checking staging".to_string(), + }, + }), + RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: "turn-1".to_string(), + last_agent_message: None, + error: None, + started_at: Some(10), + completed_at: Some(12), + duration_ms: Some(2000), + time_to_first_token_ms: None, + })), + ], + }) + .await?; + store.shutdown_thread(thread_id).await?; + + let mut app_server = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .without_auto_env() + .build_initialized() + .await?; + let page: ThreadTimelineListResponse = app_server + .request(|request_id| ClientRequest::ThreadTimelineList { + request_id, + params: ThreadTimelineListParams { + thread_id: thread_id.to_string(), + cursor: None, + limit: Some(3), + }, + }) + .await?; + assert_eq!(page.data.len(), 3); + assert!(matches!( + &page.data[0], + ThreadTimelineEntry::Item { turn_id, item, .. } + if turn_id == "turn-1" + && matches!(item.as_ref(), ThreadItem::UserMessage { .. }) + )); + assert!(matches!( + &page.data[1], + ThreadTimelineEntry::Realtime { item, .. } + if matches!( + &item.content, + ThreadRealtimeItemContent::TranscriptSegment { text, .. } + if text == "Checking staging" + ) + )); + assert_eq!( + page.active_realtime_session_at_page_start.as_deref(), + Some("voice-1") + ); + assert!(matches!( + &page.data[2], + ThreadTimelineEntry::TurnCompleted { turn_id, duration_ms: Some(2000), .. } + if turn_id == "turn-1" + )); + + let older: ThreadTimelineListResponse = app_server + .request(|request_id| ClientRequest::ThreadTimelineList { + request_id, + params: ThreadTimelineListParams { + thread_id: thread_id.to_string(), + cursor: page.next_cursor, + limit: Some(2), + }, + }) + .await?; + assert_eq!(older.data.len(), 2); + assert!(matches!( + &older.data[0], + ThreadTimelineEntry::Realtime { item, .. } + if matches!( + item.content, + ThreadRealtimeItemContent::RealtimeSessionStarted + ) + )); + assert!(matches!( + &older.data[1], + ThreadTimelineEntry::TurnStarted { turn_id, started_at: Some(10), .. } + if turn_id == "turn-1" + )); + assert_eq!(older.active_realtime_session_at_page_start, None); + assert_eq!(older.next_cursor, None); + + let ordinary: ThreadItemsListResponse = app_server + .request(|request_id| ClientRequest::ThreadItemsList { + request_id, + params: ThreadItemsListParams { + thread_id: thread_id.to_string(), + turn_id: None, + cursor: None, + limit: Some(10), + sort_direction: Some(SortDirection::Asc), + }, + }) + .await?; + assert_eq!(ordinary.data.len(), 1); + assert_eq!(ordinary.data[0].turn_id, "turn-1"); + + Ok(()) +} diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rs index ca6e43bc05..3a9e2533bb 100644 --- a/codex-rs/core/src/agent/control/spawn.rs +++ b/codex-rs/core/src/agent/control/spawn.rs @@ -87,7 +87,8 @@ fn keep_forked_rollout_item(item: &RolloutItem, preserve_reference_context_item: | ResponseItem::ContextCompaction { .. } | ResponseItem::Other => false, }, - RolloutItem::InterAgentCommunication(_) + RolloutItem::RealtimeItem(_) + | RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::SecurityRiskScore(_) => false, // Full-history forks preserve the cached prompt prefix and can keep diffing @@ -1001,6 +1002,7 @@ impl AgentControl { } true } + RolloutItem::RealtimeItem(_) => false, RolloutItem::EventMsg(_) | RolloutItem::SessionMeta(_) | RolloutItem::TurnContext(_) diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 1ddc5c8875..e5b7212ca5 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -272,6 +272,7 @@ async fn persisted_originator(thread: &CodexThread) -> String { | RolloutItem::EventMsg(_) | RolloutItem::Compacted(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::TurnContext(_) => None, }) diff --git a/codex-rs/core/src/session/rollout_reconstruction.rs b/codex-rs/core/src/session/rollout_reconstruction.rs index dae13c97fb..3896fa2e2e 100644 --- a/codex-rs/core/src/session/rollout_reconstruction.rs +++ b/codex-rs/core/src/session/rollout_reconstruction.rs @@ -282,6 +282,7 @@ impl Session { } RolloutItem::EventMsg(_) | RolloutItem::SessionMeta(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::InterAgentCommunicationMetadata { .. } => {} } @@ -371,6 +372,7 @@ impl Session { } RolloutItem::EventMsg(_) | RolloutItem::TurnContext(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::WorldState(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::SessionMeta(_) => {} @@ -411,6 +413,7 @@ impl Session { | RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::TurnContext(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => { unreachable!("only world-state replay items are collected") diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 5c80eb594a..69c7f4be13 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1350,6 +1350,7 @@ impl Session { | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) => {} } } diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 239c05699b..e5b49d450c 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -3400,6 +3400,7 @@ async fn start_new_context_window_assigns_and_persists_item_ids() { | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) | RolloutItem::SecurityRiskScore(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::EventMsg(_) => None, }); assert_eq!( @@ -3478,6 +3479,7 @@ async fn record_initial_history_assigns_and_persists_id_for_forked_response_item | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) | RolloutItem::SecurityRiskScore(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::EventMsg(_) => None, }); let persisted_item = persisted_item.expect("forked response item should be persisted"); diff --git a/codex-rs/external-agent-migration/src/sessions/append.rs b/codex-rs/external-agent-migration/src/sessions/append.rs index e08f0607a8..d76a54b7ce 100644 --- a/codex-rs/external-agent-migration/src/sessions/append.rs +++ b/codex-rs/external-agent-migration/src/sessions/append.rs @@ -217,7 +217,9 @@ fn source_model_items(items: &[RolloutItem]) -> Option>> let mut append_start_index = None; for (index, item) in items.iter().enumerate() { match item { - RolloutItem::SessionMeta(_) | RolloutItem::InterAgentCommunicationMetadata { .. } => {} + RolloutItem::SessionMeta(_) + | RolloutItem::InterAgentCommunicationMetadata { .. } + | RolloutItem::RealtimeItem(_) => {} RolloutItem::ResponseItem(response_item) => { model_items.push(SourceModelItem { response_item: &response_item.item, @@ -255,6 +257,7 @@ fn history_model_items(items: &[RolloutItem]) -> Option> { match item { RolloutItem::SessionMeta(_) | RolloutItem::InterAgentCommunicationMetadata { .. } + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) => {} RolloutItem::ResponseItem(response_item) => model_items.push(&response_item.item), RolloutItem::EventMsg( diff --git a/codex-rs/history/src/lib.rs b/codex-rs/history/src/lib.rs index 23fc515d8e..6e15c01bc8 100644 --- a/codex-rs/history/src/lib.rs +++ b/codex-rs/history/src/lib.rs @@ -23,6 +23,7 @@ use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::WorldStateItem; +use codex_protocol::realtime::RealtimeItem; use codex_protocol::security_risk::SecurityRiskScore; use schemars::JsonSchema; use serde::Deserialize; @@ -96,12 +97,16 @@ pub enum RolloutItem { SessionMeta(SessionMetaLine), ResponseItem(ResponseItemEnvelope), InterAgentCommunication(InterAgentCommunication), - InterAgentCommunicationMetadata { trigger_turn: bool }, + InterAgentCommunicationMetadata { + trigger_turn: bool, + }, Compacted(CompactedItem), TurnContext(TurnContextItem), WorldState(WorldStateItem), SecurityRiskScore(SecurityRiskScore), EventMsg(EventMsg), + /// Sparse, model-invisible facts used to reconstruct realtime presentation. + RealtimeItem(RealtimeItem), } impl Serialize for RolloutItem { @@ -415,6 +420,7 @@ fn multi_agent_version_from_items( | RolloutItem::Compacted(_) | RolloutItem::WorldState(_) | RolloutItem::SecurityRiskScore(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::EventMsg(_) => None, }) }) diff --git a/codex-rs/history/src/rollout_payload.rs b/codex-rs/history/src/rollout_payload.rs index cc4621bf5a..8f65619363 100644 --- a/codex-rs/history/src/rollout_payload.rs +++ b/codex-rs/history/src/rollout_payload.rs @@ -5,6 +5,7 @@ use super::CompactedItem; use super::EventMsg; use super::InterAgentCommunication; use super::McpResourceOriginCheckpoint; +use super::RealtimeItem; use super::ResponseItem; use super::ResponseItemEnvelope; use super::RolloutItem; @@ -49,6 +50,9 @@ pub(super) enum RolloutItemWire<'a> { EventMsg { payload: Cow<'a, EventMsg>, }, + RealtimeItem { + payload: Cow<'a, RealtimeItem>, + }, } impl<'a> From<&'a RolloutItem> for RolloutItemWire<'a> { @@ -86,6 +90,9 @@ impl<'a> From<&'a RolloutItem> for RolloutItemWire<'a> { RolloutItem::EventMsg(payload) => Self::EventMsg { payload: Cow::Borrowed(payload), }, + RolloutItem::RealtimeItem(payload) => Self::RealtimeItem { + payload: Cow::Borrowed(payload), + }, } } } @@ -115,6 +122,7 @@ impl From> for RolloutItem { Self::SecurityRiskScore(payload.into_owned()) } RolloutItemWire::EventMsg { payload } => Self::EventMsg(payload.into_owned()), + RolloutItemWire::RealtimeItem { payload } => Self::RealtimeItem(payload.into_owned()), } } } diff --git a/codex-rs/history/src/tests.rs b/codex-rs/history/src/tests.rs index fbd115fb38..97b61cf511 100644 --- a/codex-rs/history/src/tests.rs +++ b/codex-rs/history/src/tests.rs @@ -367,6 +367,16 @@ fn rollout_item_variants_preserve_existing_payload_shapes() -> Result<()> { "type": "event_msg", "payload": { "type": "warning", "message": "heads up" }, }), + json!({ + "type": "realtime_item", + "payload": { + "id": "segment-1", + "realtime_session_id": "session-1", + "type": "transcript_segment", + "role": "assistant", + "text": "hello", + }, + }), ]; for expected in fixtures { @@ -381,7 +391,7 @@ fn rollout_item_variants_preserve_existing_payload_shapes() -> Result<()> { fn rollout_item_schema_matches_tagged_payload_and_sibling_metadata() -> Result<()> { let schema = serde_json::to_value(schemars::schema_for!(RolloutItem))?; let variants = schema["oneOf"].as_array().expect("rollout variants"); - assert_eq!(variants.len(), 9); + assert_eq!(variants.len(), 10); for variant in variants { let required = variant["required"].as_array().expect("required fields"); diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs index 215574cdf0..4d98599b93 100644 --- a/codex-rs/memories/write/src/phase1.rs +++ b/codex-rs/memories/write/src/phase1.rs @@ -416,6 +416,7 @@ mod job { | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::WorldState(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => None, diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 3a5abc9c9f..f52dbe0cdc 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -38,6 +38,7 @@ mod permission_profile_snapshot; pub mod permissions; pub mod plan_tool; pub mod protocol; +pub mod realtime; pub mod request_permissions; pub mod request_user_input; pub mod review_format; diff --git a/codex-rs/protocol/src/realtime.rs b/codex-rs/protocol/src/realtime.rs new file mode 100644 index 0000000000..8c35c3efc9 --- /dev/null +++ b/codex-rs/protocol/src/realtime.rs @@ -0,0 +1,59 @@ +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use ts_rs::TS; + +/// A realtime thread item persisted in the canonical rollout. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +pub struct RealtimeItem { + pub id: String, + pub realtime_session_id: String, + #[serde(flatten)] + pub content: RealtimeItemContent, +} + +/// The minimum facts needed to interleave realtime speech and agent work. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "snake_case")] +#[ts(tag = "type", rename_all = "snake_case")] +pub enum RealtimeItemContent { + RealtimeSessionStarted, + TranscriptSegment { + role: RealtimeTranscriptRole, + text: String, + }, + BemItemPromoted { + turn_id: String, + item_id: String, + presentation: BemItemPresentation, + }, + RealtimeSessionClosed { + outcome: RealtimeSessionOutcome, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum RealtimeSessionOutcome { + Ended, + Failed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum RealtimeTranscriptRole { + User, + Assistant, +} + +/// How an existing agent item is presented in the realtime conversation. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "snake_case")] +#[ts(tag = "type", rename_all = "snake_case")] +pub enum BemItemPresentation { + WholeItem, + InlineMarkdown, + InlineVisualization { index: u32 }, +} diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index a5b8040c48..f75622fced 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -1176,6 +1176,9 @@ async fn read_head_summary(path: &Path, head_limit: usize) -> io::Result { // Not included in `head`; skip. } + RolloutItem::RealtimeItem(_) => { + // Realtime presentation does not affect model-visible thread summaries. + } RolloutItem::Compacted(_) => { // Not included in `head`; skip. } @@ -1246,6 +1249,7 @@ pub async fn read_head_for_summary(path: &Path) -> io::Result {} } @@ -1299,6 +1303,7 @@ pub async fn read_session_meta_line(path: &Path) -> io::Result | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => {} } diff --git a/codex-rs/rollout/src/metadata.rs b/codex-rs/rollout/src/metadata.rs index 15b4344a3c..62407790ce 100644 --- a/codex-rs/rollout/src/metadata.rs +++ b/codex-rs/rollout/src/metadata.rs @@ -75,6 +75,7 @@ pub fn builder_from_items( | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => None, }) && let Some(builder) = builder_from_session_meta(session_meta, rollout_path) @@ -144,6 +145,7 @@ pub async fn extract_metadata_from_rollout( | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => None, }), diff --git a/codex-rs/rollout/src/model_context.rs b/codex-rs/rollout/src/model_context.rs index 06f245b39b..6a19e5f24e 100644 --- a/codex-rs/rollout/src/model_context.rs +++ b/codex-rs/rollout/src/model_context.rs @@ -153,6 +153,7 @@ impl ModelContextScan { RolloutItem::EventMsg(_) | RolloutItem::SessionMeta(_) | RolloutItem::InterAgentCommunicationMetadata { .. } + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_) => {} } diff --git a/codex-rs/rollout/src/persistence_metrics.rs b/codex-rs/rollout/src/persistence_metrics.rs index d345c1ae53..b26320cd8a 100644 --- a/codex-rs/rollout/src/persistence_metrics.rs +++ b/codex-rs/rollout/src/persistence_metrics.rs @@ -237,6 +237,20 @@ fn rollout_item_type(item: &RolloutItem) -> String { RolloutItem::TurnContext(_) => "turn_context".to_string(), RolloutItem::WorldState(_) => "world_state".to_string(), RolloutItem::SecurityRiskScore(_) => "security_risk_score".to_string(), + RolloutItem::RealtimeItem(item) => match &item.content { + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionStarted => { + "realtime.session_started".to_string() + } + codex_protocol::realtime::RealtimeItemContent::TranscriptSegment { .. } => { + "realtime.transcript_segment".to_string() + } + codex_protocol::realtime::RealtimeItemContent::BemItemPromoted { .. } => { + "realtime.bem_item_promoted".to_string() + } + codex_protocol::realtime::RealtimeItemContent::RealtimeSessionClosed { .. } => { + "realtime.session_closed".to_string() + } + }, RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => { format!("event.item_completed.{}", turn_item_type(&event.item)) } diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index 5c6c9e026f..2334852b19 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -13,6 +13,7 @@ pub fn is_persisted_rollout_item(item: &RolloutItem, history_mode: ThreadHistory RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } => true, RolloutItem::EventMsg(ev) => should_persist_event_msg(ev, history_mode), + RolloutItem::RealtimeItem(_) => matches!(history_mode, ThreadHistoryMode::Paginated), // Persist Codex executive markers so we can analyze flows (e.g., compaction, API turns). RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 6be4356c99..394a6d3e9d 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -2074,6 +2074,7 @@ async fn resume_candidate_matches_cwd( | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::Compacted(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => None, }) diff --git a/codex-rs/rollout/src/search.rs b/codex-rs/rollout/src/search.rs index 72f3e54606..946e2672fd 100644 --- a/codex-rs/rollout/src/search.rs +++ b/codex-rs/rollout/src/search.rs @@ -292,6 +292,7 @@ fn conversation_text_from_item(item: &RolloutItem) -> Option { | RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::Compacted(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_) => None, } diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 989075a23b..9b0c7f3375 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -27,6 +27,7 @@ pub fn apply_rollout_item( RolloutItem::Compacted(_) => {} RolloutItem::WorldState(_) => {} RolloutItem::SecurityRiskScore(_) => {} + RolloutItem::RealtimeItem(_) => {} } if metadata.model_provider.is_empty() { metadata.model_provider = default_provider.to_string(); @@ -53,6 +54,7 @@ pub fn rollout_item_affects_thread_metadata(item: &RolloutItem) -> bool { | RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::Compacted(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_) => false, } diff --git a/codex-rs/state/src/migrations_tests.rs b/codex-rs/state/src/migrations_tests.rs index fea8878d36..78c6929807 100644 --- a/codex-rs/state/src/migrations_tests.rs +++ b/codex-rs/state/src/migrations_tests.rs @@ -374,6 +374,109 @@ INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_ pool.close().await; } +#[tokio::test] +async fn realtime_items_preserve_older_thread_history_writers() { + let sqlite_home = crate::runtime::test_support::unique_temp_dir(); + tokio::fs::create_dir_all(&sqlite_home) + .await + .expect("sqlite home should be created"); + let _cleanup = scopeguard::guard(sqlite_home.clone(), |sqlite_home| { + let _ = std::fs::remove_dir_all(sqlite_home); + }); + let sqlite = crate::SqliteConfig::new_for_testing(sqlite_home.as_path().abs()); + let older_migrator = Migrator { + migrations: Cow::Owned( + THREAD_HISTORY_MIGRATOR + .migrations + .iter() + .filter(|migration| migration.version < 5) + .cloned() + .collect(), + ), + ignore_missing: true, + locking: THREAD_HISTORY_MIGRATOR.locking, + table_name: THREAD_HISTORY_MIGRATOR.table_name.clone(), + create_schemas: THREAD_HISTORY_MIGRATOR.create_schemas.clone(), + no_tx: THREAD_HISTORY_MIGRATOR.no_tx, + }; + let pool = sqlite + .open_thread_history_db(&older_migrator, /*telemetry_override*/ None) + .await + .expect("existing thread history migrations should apply"); + sqlx::query( + "INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_json) VALUES ('thread-1', 'turn-1', 'existing-item', 1, 100, '{}')", + ) + .execute(&pool) + .await + .expect("existing turn-scoped item should be inserted"); + + THREAD_HISTORY_MIGRATOR + .run(&pool) + .await + .expect("realtime item migration should apply"); + let turn_id_not_null = sqlx::query_scalar::<_, i64>( + "SELECT \"notnull\" FROM pragma_table_info('thread_items') WHERE name = 'turn_id'", + ) + .fetch_one(&pool) + .await + .expect("existing thread item schema should remain inspectable"); + assert_eq!(turn_id_not_null, 1); + + sqlx::query( + "INSERT INTO thread_realtime_items (thread_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json) VALUES ('thread-1', 'realtime-item', 2, 200, 'realtime_session_started', '{}')", + ) + .execute(&pool) + .await + .expect("thread-scoped realtime item should be inserted separately"); + sqlx::query( + "INSERT INTO thread_history_projection_state (thread_id, next_rollout_byte_offset, next_rollout_ordinal) VALUES ('thread-1', 0, 0)", + ) + .execute(&pool) + .await + .expect("thread projection checkpoint should be inserted"); + + let older_pool = sqlite + .open_thread_history_db(&older_migrator, /*telemetry_override*/ None) + .await + .expect("older binaries should tolerate the additive realtime migration"); + sqlx::query( + "INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, created_at_ms, item_json) VALUES ('thread-1', 'turn-1', 'older-writer-item', 3, 300, '{}')", + ) + .execute(&older_pool) + .await + .expect("older binaries should continue writing ordinary turn-scoped items"); + let ordinary_items = sqlx::query_as::<_, (String, String)>( + "SELECT item_id, turn_id FROM thread_items WHERE thread_id = ? ORDER BY rollout_ordinal", + ) + .bind("thread-1") + .fetch_all(&older_pool) + .await + .expect("older binaries should never observe turnless realtime items"); + assert_eq!( + ordinary_items, + vec![ + ("existing-item".to_string(), "turn-1".to_string()), + ("older-writer-item".to_string(), "turn-1".to_string()), + ] + ); + sqlx::query("DELETE FROM thread_history_projection_state WHERE thread_id = ?") + .bind("thread-1") + .execute(&older_pool) + .await + .expect("older binaries should delete their known projection checkpoint"); + let remaining_realtime_items = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM thread_realtime_items WHERE thread_id = ?", + ) + .bind("thread-1") + .fetch_one(&pool) + .await + .expect("read realtime items after an older writer deleted the thread"); + assert_eq!(remaining_realtime_items, 0); + + older_pool.close().await; + pool.close().await; +} + #[tokio::test] async fn agent_job_tables_are_dropped_when_upgrading() { let sqlite_home = crate::runtime::test_support::unique_temp_dir(); diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 4bb83cd169..237cc3b08b 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -1320,6 +1320,7 @@ pub(super) fn extract_memory_mode(items: &[RolloutItem]) -> Option { | RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::WorldState(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::EventMsg(_) => None, }) diff --git a/codex-rs/state/thread_history_migrations/0005_thread_realtime_items.sql b/codex-rs/state/thread_history_migrations/0005_thread_realtime_items.sql new file mode 100644 index 0000000000..9c85043651 --- /dev/null +++ b/codex-rs/state/thread_history_migrations/0005_thread_realtime_items.sql @@ -0,0 +1,22 @@ +CREATE TABLE thread_realtime_items ( + thread_id TEXT NOT NULL, + item_id TEXT NOT NULL, + rollout_ordinal INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + item_type TEXT NOT NULL, + item_json TEXT NOT NULL, + PRIMARY KEY (thread_id, item_id) +); + +CREATE UNIQUE INDEX idx_thread_realtime_items_page + ON thread_realtime_items(thread_id, rollout_ordinal); + +CREATE INDEX idx_thread_realtime_items_boundary + ON thread_realtime_items(thread_id, rollout_ordinal) + WHERE item_type IN ('realtime_session_started', 'realtime_session_closed'); + +CREATE TRIGGER thread_realtime_items_projection_cleanup + AFTER DELETE ON thread_history_projection_state +BEGIN + DELETE FROM thread_realtime_items WHERE thread_id = OLD.thread_id; +END; diff --git a/codex-rs/state/thread_history_migrations/0006_thread_turn_ends.sql b/codex-rs/state/thread_history_migrations/0006_thread_turn_ends.sql new file mode 100644 index 0000000000..b6c6ddd738 --- /dev/null +++ b/codex-rs/state/thread_history_migrations/0006_thread_turn_ends.sql @@ -0,0 +1,3 @@ +CREATE INDEX idx_thread_turns_end_page + ON thread_turns(thread_id, rollout_end_ordinal, turn_id) + WHERE rollout_end_ordinal IS NOT NULL; diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 70fc994d1a..236ef61c14 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -68,6 +68,7 @@ pub use types::ItemPage; pub use types::ItemSortKey; pub use types::ListItemsParams; pub use types::ListThreadsParams; +pub use types::ListTimelineParams; pub use types::ListTurnsParams; pub use types::LoadThreadHistoryParams; pub use types::MoveThreadToSectionParams; @@ -98,5 +99,6 @@ pub use types::ThreadPersistenceMetadata; pub use types::ThreadRelationFilter; pub use types::ThreadSearchPage; pub use types::ThreadSortKey; +pub use types::TimelinePage; pub use types::TurnPage; pub use types::UpdateThreadMetadataParams; diff --git a/codex-rs/thread-store/src/local/delete_thread.rs b/codex-rs/thread-store/src/local/delete_thread.rs index 74fafd5447..810153b800 100644 --- a/codex-rs/thread-store/src/local/delete_thread.rs +++ b/codex-rs/thread-store/src/local/delete_thread.rs @@ -648,6 +648,13 @@ mod tests { .execute(&pool) .await .expect("insert item"); + sqlx::query( + "INSERT INTO thread_realtime_items (thread_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json) VALUES (?, 'realtime-1', 3, 1, 'realtime_session_started', '{}')", + ) + .bind(thread_id_string.as_str()) + .execute(&pool) + .await + .expect("insert realtime item"); sqlx::query( "INSERT INTO thread_history_projection_state (thread_id, next_rollout_byte_offset, next_rollout_ordinal) VALUES (?, 3, 3)", ) @@ -668,21 +675,23 @@ mod tests { } )); assert!(rollout_path.exists()); - let counts = sqlx::query_as::<_, (i64, i64, i64)>( + let counts = sqlx::query_as::<_, (i64, i64, i64, i64)>( r#" SELECT (SELECT COUNT(*) FROM thread_turns WHERE thread_id = ?), (SELECT COUNT(*) FROM thread_items WHERE thread_id = ?), + (SELECT COUNT(*) FROM thread_realtime_items WHERE thread_id = ?), (SELECT COUNT(*) FROM thread_history_projection_state WHERE thread_id = ?) "#, ) .bind(thread_id_string.as_str()) .bind(thread_id_string.as_str()) .bind(thread_id_string.as_str()) + .bind(thread_id_string.as_str()) .fetch_one(&pool) .await .expect("read preserved history rows"); - assert_eq!(counts, (1, 1, 1)); + assert_eq!(counts, (1, 1, 1, 1)); } #[tokio::test] @@ -725,6 +734,13 @@ SELECT .execute(&pool) .await .expect("insert item"); + sqlx::query( + "INSERT INTO thread_realtime_items (thread_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json) VALUES (?, 'realtime-1', 3, 1, 'realtime_session_started', '{}')", + ) + .bind(thread_id_string.as_str()) + .execute(&pool) + .await + .expect("insert realtime item"); sqlx::query( "INSERT INTO thread_history_projection_state (thread_id, next_rollout_byte_offset, next_rollout_ordinal) VALUES (?, 3, 3)", ) @@ -759,21 +775,23 @@ SELECT .expect("delete thread"); assert!(!lock_path.exists()); - let counts = sqlx::query_as::<_, (i64, i64, i64)>( + let counts = sqlx::query_as::<_, (i64, i64, i64, i64)>( r#" SELECT (SELECT COUNT(*) FROM thread_turns WHERE thread_id = ?), (SELECT COUNT(*) FROM thread_items WHERE thread_id = ?), + (SELECT COUNT(*) FROM thread_realtime_items WHERE thread_id = ?), (SELECT COUNT(*) FROM thread_history_projection_state WHERE thread_id = ?) "#, ) .bind(thread_id_string.as_str()) .bind(thread_id_string.as_str()) .bind(thread_id_string.as_str()) + .bind(thread_id_string.as_str()) .fetch_one(&pool) .await .expect("read remaining history rows"); - assert_eq!(counts, (0, 0, 0)); + assert_eq!(counts, (0, 0, 0, 0)); } #[tokio::test] diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index b08a17d261..aa88ea0280 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -62,6 +62,7 @@ use crate::ListItemsParams; use crate::ListProjectsParams; use crate::ListThreadSectionsParams; use crate::ListThreadsParams; +use crate::ListTimelineParams; use crate::ListTurnsParams; use crate::LoadThreadHistoryParams; use crate::MoveProjectParams; @@ -92,6 +93,7 @@ use crate::ThreadStore; use crate::ThreadStoreError; use crate::ThreadStoreFuture; use crate::ThreadStoreResult; +use crate::TimelinePage; use crate::TurnPage; use crate::UpdateProjectParams; use crate::UpdateThreadMetadataParams; @@ -403,6 +405,14 @@ impl LocalThreadStore { thread_history::list_items(self, params).await } + /// Lists bounded ordinary and realtime items from the rollout projection. + pub async fn list_timeline( + &self, + params: ListTimelineParams, + ) -> ThreadStoreResult { + thread_history::list_timeline(self, params).await + } + /// Searches projection-backed visible messages within one paginated thread. pub async fn search_thread_occurrences( &self, @@ -596,6 +606,10 @@ impl ThreadStore for LocalThreadStore { Box::pin(LocalThreadStore::list_items(self, params)) } + fn list_timeline(&self, params: ListTimelineParams) -> ThreadStoreFuture<'_, TimelinePage> { + Box::pin(LocalThreadStore::list_timeline(self, params)) + } + fn search_threads( &self, params: SearchThreadsParams, diff --git a/codex-rs/thread-store/src/local/rollout_migration.rs b/codex-rs/thread-store/src/local/rollout_migration.rs index 46d71d5f11..79f7dd1bf1 100644 --- a/codex-rs/thread-store/src/local/rollout_migration.rs +++ b/codex-rs/thread-store/src/local/rollout_migration.rs @@ -1152,17 +1152,27 @@ impl LocalThreadStore { let fallback_created_at_ms = DateTime::parse_from_rfc3339(&line.timestamp) .map_err(migration_error)? .timestamp_millis(); - batch.push(RolloutProjectionStep::Line(ProjectedRolloutLine { - ordinal, - start_byte_offset: offset, - end_byte_offset: next_offset, - fallback_created_at_ms: Some(fallback_created_at_ms), - changes: if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) { - Default::default() - } else { - project_rollout_line(&line) + let is_inherited_subagent_history = + subagent_history_start_ordinal.is_some_and(|start| ordinal < start); + batch.push(RolloutProjectionStep::Line(Box::new( + ProjectedRolloutLine { + ordinal, + start_byte_offset: offset, + end_byte_offset: next_offset, + fallback_created_at_ms: Some(fallback_created_at_ms), + changes: if is_inherited_subagent_history { + Default::default() + } else { + project_rollout_line(&line) + }, + realtime_item: match line.item { + RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => { + Some(item) + } + _ => None, + }, }, - })); + ))); offset = next_offset; if offset.saturating_sub(batch_start) >= PROJECTION_BATCH_BYTES { diff --git a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs index 703d666ff8..9eed043bd0 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/canonicalizer.rs @@ -292,6 +292,7 @@ impl LegacyRolloutCanonicalizer { } item @ (RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::TurnContext(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_)) => { self.write_item(writer, ×tamp, item).await?; diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs index 9f9b0653e0..f8404b6177 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs @@ -219,7 +219,7 @@ impl RollbackPlanner { self.pending_turn_records.push(index); } } - RolloutItem::WorldState(_) => {} + RolloutItem::WorldState(_) | RolloutItem::RealtimeItem(_) => {} RolloutItem::SecurityRiskScore(_) => self.record_boundaries[index] = None, } diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs index 744866eeb8..b8d47f36bf 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs @@ -75,6 +75,7 @@ impl ModelReplayPlanner { | RolloutItem::ResponseItem(_) | RolloutItem::EventMsg(_) | RolloutItem::InterAgentCommunicationMetadata { .. } + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_) => return, }; diff --git a/codex-rs/thread-store/src/local/thread_history.rs b/codex-rs/thread-store/src/local/thread_history.rs index b9edf517ab..8f5ca8c6c3 100644 --- a/codex-rs/thread-store/src/local/thread_history.rs +++ b/codex-rs/thread-store/src/local/thread_history.rs @@ -3,18 +3,21 @@ use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::TurnStatus; use codex_protocol::ThreadId; use codex_protocol::models::MessagePhase; +use codex_protocol::realtime::RealtimeItem; use super::LocalThreadStore; use crate::ThreadStoreError; use crate::ThreadStoreResult; mod read; +mod realtime; mod search; mod segment_paging; mod turn_lookup; pub(super) use read::list_items; pub(super) use read::list_turns; +pub(super) use realtime::list_timeline; pub(super) use search::search_thread_occurrences; pub(super) use turn_lookup::find_source_turn; pub(super) use turn_lookup::find_visible_turn; @@ -28,6 +31,7 @@ pub(super) struct ProjectedRolloutLine { pub end_byte_offset: u64, pub fallback_created_at_ms: Option, pub changes: ThreadHistoryChangeSet, + pub realtime_item: Option, } /// One ordered update to apply while advancing a rollout projection checkpoint. @@ -35,7 +39,7 @@ pub(super) struct ProjectedRolloutLine { /// Skipped ordinal ranges keep the byte and ordinal checkpoints describing the same durable /// prefix even when a complete rollout line cannot be projected. pub(super) enum RolloutProjectionStep { - Line(ProjectedRolloutLine), + Line(Box), SkippedOrdinalRange { start_ordinal: u64, end_ordinal_exclusive: u64, @@ -154,6 +158,35 @@ WHERE thread_id = ? projection.changes, ) .await?; + if let Some(item) = projection.realtime_item { + let item_json = serde_json::to_string(&item).map_err(thread_history_error)?; + sqlx::query( + r#" +INSERT INTO thread_realtime_items ( + thread_id, + item_id, + rollout_ordinal, + created_at_ms, + item_type, + item_json +) VALUES (?, ?, ?, ?, json_extract(?, '$.type'), ?) +ON CONFLICT(thread_id, item_id) DO NOTHING + "#, + ) + .bind(thread_id.as_str()) + .bind(item.id.as_str()) + .bind(ordinal) + .bind(projection.fallback_created_at_ms.ok_or_else(|| { + ThreadStoreError::Internal { + message: "realtime rollout item is missing its timestamp".to_string(), + } + })?) + .bind(item_json.as_str()) + .bind(item_json.as_str()) + .execute(&mut *transaction) + .await + .map_err(thread_history_error)?; + } next_ordinal = next_ordinal .checked_add(1) @@ -231,6 +264,11 @@ pub(super) async fn delete_thread( .execute(&mut *transaction) .await .map_err(thread_history_delete_error)?; + sqlx::query("DELETE FROM thread_realtime_items WHERE thread_id = ?") + .bind(thread_id.as_str()) + .execute(&mut *transaction) + .await + .map_err(thread_history_delete_error)?; sqlx::query("DELETE FROM thread_turns WHERE thread_id = ?") .bind(thread_id.as_str()) .execute(&mut *transaction) diff --git a/codex-rs/thread-store/src/local/thread_history/read_tests.rs b/codex-rs/thread-store/src/local/thread_history/read_tests.rs index fdffc43394..798151489e 100644 --- a/codex-rs/thread-store/src/local/thread_history/read_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history/read_tests.rs @@ -3,6 +3,7 @@ use std::time::Duration; use chrono::Utc; use codex_app_server_protocol::CodexErrorInfo; +use codex_app_server_protocol::ThreadTimelineEntry; use codex_protocol::ThreadId; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::HistoryPosition; @@ -10,6 +11,11 @@ use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::ThreadHistoryMode; +use codex_protocol::realtime::BemItemPresentation; +use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_protocol::realtime::RealtimeSessionOutcome; +use codex_protocol::realtime::RealtimeTranscriptRole; use codex_rollout::RolloutItem; use codex_rollout::RolloutLine; use pretty_assertions::assert_eq; @@ -17,6 +23,7 @@ use tempfile::TempDir; use super::*; use crate::ItemSortKey; +use crate::ListTimelineParams; use crate::SearchThreadOccurrencesParams; use crate::SortDirection; use crate::StoredTurnError; @@ -225,6 +232,385 @@ async fn list_items_pages_whole_thread_and_per_turn_rows() { assert_eq!(item_ids(&next_turn_page), vec!["item-3"]); } +#[tokio::test] +async fn timeline_interleaves_items_and_restores_page_boundary_session_state() { + let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await; + let db = history_db(&store).await; + for (item_id, ordinal) in [ + ("before", 10), + ("middle", 20), + ("after", 30), + ("closed", 40), + ] { + insert_item(db, thread_id, "turn-1", item_id, ordinal).await; + } + for (ordinal, item) in [ + ( + 11, + RealtimeItem { + id: "voice-1:started".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }, + ), + ( + 12, + RealtimeItem { + id: "voice-1:source:0".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "Before the artifact".to_string(), + }, + }, + ), + ( + 21, + RealtimeItem { + id: "artifact".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "middle".to_string(), + presentation: BemItemPresentation::InlineMarkdown, + }, + }, + ), + ( + 31, + RealtimeItem { + id: "voice-1:closed".to_string(), + realtime_session_id: "voice-1".to_string(), + content: RealtimeItemContent::RealtimeSessionClosed { + outcome: RealtimeSessionOutcome::Ended, + }, + }, + ), + ] { + let item_json = serde_json::to_string(&item).expect("serialize realtime item"); + sqlx::query( + r#" +INSERT INTO thread_realtime_items ( + thread_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json +) VALUES (?, ?, ?, ?, json_extract(?, '$.type'), ?) + "#, + ) + .bind(thread_id.to_string()) + .bind(&item.id) + .bind(ordinal) + .bind(ordinal * 1_000) + .bind(&item_json) + .bind(&item_json) + .execute(db) + .await + .expect("insert realtime item"); + } + + let latest = store + .list_timeline(ListTimelineParams { + thread_id, + cursor: None, + page_size: 3, + }) + .await + .expect("list latest timeline page"); + assert_eq!( + latest + .items + .iter() + .map(|item| match item { + ThreadTimelineEntry::Item { position, .. } + | ThreadTimelineEntry::Realtime { position, .. } + | ThreadTimelineEntry::TurnStarted { position, .. } + | ThreadTimelineEntry::TurnCompleted { position, .. } => *position, + }) + .collect::>(), + vec![30, 31, 40] + ); + assert_eq!( + latest.active_realtime_session_at_page_start, + Some("voice-1".to_string()) + ); + let older = store + .list_timeline(ListTimelineParams { + thread_id, + cursor: latest.next_cursor, + page_size: 3, + }) + .await + .expect("list older timeline page"); + assert_eq!( + older + .items + .iter() + .map(|item| match item { + ThreadTimelineEntry::Item { position, .. } + | ThreadTimelineEntry::Realtime { position, .. } + | ThreadTimelineEntry::TurnStarted { position, .. } + | ThreadTimelineEntry::TurnCompleted { position, .. } => *position, + }) + .collect::>(), + vec![12, 20, 21] + ); + assert_eq!( + older.active_realtime_session_at_page_start, + Some("voice-1".to_string()) + ); + let oldest = store + .list_timeline(ListTimelineParams { + thread_id, + cursor: older.next_cursor, + page_size: 3, + }) + .await + .expect("list oldest timeline page"); + assert_eq!(oldest.active_realtime_session_at_page_start, None); + assert!(matches!( + oldest.items.as_slice(), + [ + ThreadTimelineEntry::Item { position: 10, .. }, + ThreadTimelineEntry::Realtime { position: 11, .. } + ] + )); + + let ordinary_items = store + .list_items(item_params( + thread_id, + /*turn_id*/ None, + /*cursor*/ None, + /*page_size*/ 10, + SortDirection::Asc, + )) + .await + .expect("existing item API excludes realtime facts"); + assert_eq!( + item_ids(&ordinary_items), + vec!["before", "middle", "after", "closed"] + ); +} + +#[tokio::test] +async fn timeline_turn_boundaries_page_through_shared_ordinals() { + let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await; + let db = history_db(&store).await; + let error = r#"{"message":"failed","codexErrorInfo":null,"additionalDetails":null}"#; + for (turn_id, start, end, status, error_json) in [ + ("complete", 10, Some(20), "completed", None), + ("interrupt", 20, Some(30), "interrupted", None), + ("failed", 30, Some(40), "failed", Some(error)), + ("running", 40, None, "inProgress", None), + ] { + insert_turn( + db, thread_id, turn_id, start, status, error_json, /*first_user_item_id*/ None, + /*final_agent_item_id*/ None, + ) + .await; + sqlx::query("UPDATE thread_turns SET rollout_end_ordinal = ?, started_at = ?, completed_at = ?, duration_ms = ? WHERE thread_id = ? AND turn_id = ?") + .bind(end) + .bind(start) + .bind(end) + .bind(end.map(|end| (end - start) * 1000)) + .bind(thread_id.to_string()) + .bind(turn_id) + .execute(db) + .await + .expect("set turn boundary"); + } + insert_item( + db, thread_id, "complete", "item", /*rollout_ordinal*/ 20, + ) + .await; + + let all = store + .list_timeline(ListTimelineParams { + thread_id, + cursor: None, + page_size: 20, + }) + .await + .expect("full timeline"); + let mut cursor = None; + let mut paged = Vec::new(); + loop { + let page = store + .list_timeline(ListTimelineParams { + thread_id, + cursor, + page_size: 1, + }) + .await + .expect("single-entry timeline page"); + assert_eq!(page.items.len(), 1); + paged.extend(page.items); + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + } + paged.reverse(); + assert_eq!(paged, all.items); + assert_eq!( + all.items + .iter() + .map(crate::local::thread_history::realtime::entry_key) + .collect::>(), + vec![ + (10, 0, "complete"), + (20, 0, "interrupt"), + (20, 1, "item"), + (20, 3, "complete"), + (30, 0, "failed"), + (30, 3, "interrupt"), + (40, 0, "running"), + (40, 3, "failed") + ] + ); + let completed = all + .items + .into_iter() + .filter(|entry| matches!(entry, ThreadTimelineEntry::TurnCompleted { .. })) + .collect::>(); + let expected = [ + ("complete", 10, 20, "completed", serde_json::Value::Null), + ("interrupt", 20, 30, "interrupted", serde_json::Value::Null), + ( + "failed", + 30, + 40, + "failed", + serde_json::from_str(error).expect("error"), + ), + ] + .into_iter() + .map(|(id, start, end, status, error)| { + serde_json::from_value( + serde_json::json!({"type":"turnCompleted", "position":end, "turnId":id, + "status":status, "error":error, "startedAt":start, "completedAt":end, + "durationMs":10000}), + ) + .expect("expected boundary") + }) + .collect::>(); + assert_eq!(completed, expected); +} + +#[tokio::test] +async fn timeline_page_state_excludes_realtime_events_after_fork_cutoff() { + let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await; + let source_id = ThreadId::default(); + let source_path = rollout_path(home.path(), source_id); + write_rollout_with_end( + source_path.as_path(), + source_id, + /*history_base*/ None, + /*next_ordinal*/ 5, + ); + write_rollout_with_end( + rollout_path(home.path(), child_id).as_path(), + child_id, + Some(history_position( + source_path.as_path(), + source_id, + /*end_ordinal_exclusive*/ 3, + )), + /*next_ordinal*/ 2, + ); + + let db = history_db(&store).await; + insert_turn( + db, + source_id, + "source-turn", + /*rollout_ordinal*/ 1, + "completed", + /*error_json*/ None, + /*first_user_item_id*/ None, + /*final_agent_item_id*/ None, + ) + .await; + sqlx::query("UPDATE thread_turns SET rollout_end_ordinal = 4 WHERE thread_id = ?") + .bind(source_id.to_string()) + .execute(db) + .await + .expect("source turn end"); + insert_item( + db, + child_id, + "child-turn", + "child-item", + /*rollout_ordinal*/ 4, + ) + .await; + for (ordinal, item) in [ + ( + 1, + RealtimeItem { + id: "voice:started".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }, + ), + ( + 3, + RealtimeItem { + id: "voice:closed".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::RealtimeSessionClosed { + outcome: RealtimeSessionOutcome::Ended, + }, + }, + ), + ] { + let item_json = serde_json::to_string(&item).expect("serialize realtime item"); + sqlx::query( + r#" +INSERT INTO thread_realtime_items ( + thread_id, item_id, rollout_ordinal, created_at_ms, item_type, item_json +) VALUES (?, ?, ?, ?, json_extract(?, '$.type'), ?) + "#, + ) + .bind(source_id.to_string()) + .bind(&item.id) + .bind(ordinal) + .bind(ordinal * 1_000) + .bind(&item_json) + .bind(&item_json) + .execute(db) + .await + .expect("insert realtime boundary"); + } + + let page = store + .list_timeline(ListTimelineParams { + thread_id: child_id, + cursor: None, + page_size: 1, + }) + .await + .expect("list forked timeline page"); + + assert_eq!( + page.active_realtime_session_at_page_start, + Some("voice".to_string()) + ); + let inherited = store + .list_timeline(ListTimelineParams { + thread_id: child_id, + cursor: page.next_cursor, + page_size: 10, + }) + .await + .expect("inherited timeline"); + assert_eq!( + inherited + .items + .iter() + .map(crate::local::thread_history::realtime::entry_key) + .collect::>(), + vec![(1, 0, "source-turn"), (1, 2, "voice:started")] + ); +} + #[tokio::test] async fn list_items_filters_exclusive_update_ordinals_across_pages_and_turns() { let (_home, store, thread_id) = store_with_mode(ThreadHistoryMode::Paginated).await; diff --git a/codex-rs/thread-store/src/local/thread_history/realtime.rs b/codex-rs/thread-store/src/local/thread_history/realtime.rs new file mode 100644 index 0000000000..0dd233b776 --- /dev/null +++ b/codex-rs/thread-store/src/local/thread_history/realtime.rs @@ -0,0 +1,246 @@ +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadTimelineEntry; +use codex_protocol::ThreadId; +use codex_protocol::realtime::RealtimeItem; +use serde::Deserialize; +use serde::Serialize; + +use super::super::LocalThreadStore; +use super::read::validate_thread_for_paginated_reads; +use super::segment_paging::validate_page_size; +use super::sqlite_integer; +use super::thread_history_error; +use crate::ListTimelineParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::TimelinePage; + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct TimelineCursor { + thread_id: ThreadId, + position: u64, + kind: u8, + id: String, +} + +// A rollout record can materialize both an item and a turn boundary. Keep the +// boundary order stable, including when the page cuts through one ordinal. +pub(super) fn entry_key(entry: &ThreadTimelineEntry) -> (u64, u8, &str) { + match entry { + ThreadTimelineEntry::TurnStarted { + position, turn_id, .. + } => (*position, 0, turn_id), + ThreadTimelineEntry::Item { position, item, .. } => (*position, 1, item.id()), + ThreadTimelineEntry::Realtime { position, item } => (*position, 2, &item.id), + ThreadTimelineEntry::TurnCompleted { + position, turn_id, .. + } => (*position, 3, turn_id), + } +} + +pub(in crate::local) async fn list_timeline( + store: &LocalThreadStore, + params: ListTimelineParams, +) -> ThreadStoreResult { + validate_thread_for_paginated_reads( + store, + params.thread_id, + /*include_archived*/ false, + "thread/timeline/list", + ) + .await?; + validate_page_size(params.page_size)?; + + let lineage = store.resolve_rollout_lineage(params.thread_id).await?; + let pool = store.thread_history_db().await?; + let cursor = params + .cursor + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(|_| ThreadStoreError::InvalidRequest { + message: "invalid thread timeline cursor".to_string(), + })? + .map(|cursor| { + if cursor.thread_id != params.thread_id { + Err(ThreadStoreError::InvalidRequest { + message: "thread timeline cursor belongs to another thread".to_string(), + }) + } else { + Ok(cursor) + } + }) + .transpose()?; + let cursor_position = cursor.as_ref().map(|cursor| cursor.position); + let cursor_ordinal = cursor_position + .map(|position| sqlite_integer(position, "rollout ordinal")) + .transpose()? + .unwrap_or(i64::MAX); + let cursor_kind = cursor.as_ref().map_or(4, |cursor| cursor.kind); + let cursor_id = cursor.as_ref().map_or("", |cursor| cursor.id.as_str()); + + let mut rows = Vec::new(); + let max_rows = params.page_size + 1; + for segment in lineage.segments().iter().rev() { + let remaining = max_rows.saturating_sub(rows.len()); + if remaining == 0 { + break; + } + if cursor_position.is_some_and(|position| position < segment.start_ordinal()) { + continue; + } + let upper = segment + .end_ordinal() + .map(|value| sqlite_integer(value, "rollout ordinal")) + .transpose()? + .unwrap_or(i64::MAX); + let segment_rows = sqlx::query_as::<_, (i64, i64, String, Option, String)>( + r#" +WITH starts AS ( +SELECT rollout_ordinal, 0 AS kind, turn_id AS id, turn_id, + json_object('type', 'turnStarted', 'position', rollout_ordinal, + 'turnId', turn_id, 'startedAt', started_at) AS item_json +FROM thread_turns +WHERE thread_id = ?1 AND rollout_ordinal >= ?2 + AND rollout_ordinal < ?3 AND rollout_ordinal <= ?4 + AND (rollout_ordinal, 0, turn_id) < (?4, ?5, ?6) +ORDER BY rollout_ordinal DESC, turn_id DESC LIMIT ?7 +), items AS ( +SELECT rollout_ordinal, 1 AS kind, item_id AS id, turn_id, item_json +FROM thread_items +WHERE thread_id = ?1 AND rollout_ordinal >= ?2 + AND rollout_ordinal < ?3 AND rollout_ordinal <= ?4 + AND (rollout_ordinal, 1, item_id) < (?4, ?5, ?6) +ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7 +), realtime AS ( +SELECT rollout_ordinal, 2 AS kind, item_id AS id, NULL AS turn_id, item_json +FROM thread_realtime_items +WHERE thread_id = ?1 AND rollout_ordinal >= ?2 + AND rollout_ordinal < ?3 AND rollout_ordinal <= ?4 + AND (rollout_ordinal, 2, item_id) < (?4, ?5, ?6) +ORDER BY rollout_ordinal DESC, item_id DESC LIMIT ?7 +), ends AS ( +SELECT rollout_end_ordinal AS rollout_ordinal, 3 AS kind, turn_id AS id, turn_id, + json_object('type', 'turnCompleted', 'position', rollout_end_ordinal, + 'turnId', turn_id, 'status', status, 'error', json(error_json), + 'startedAt', started_at, 'completedAt', completed_at, + 'durationMs', duration_ms) AS item_json +FROM thread_turns +WHERE thread_id = ?1 AND rollout_end_ordinal >= ?2 + AND rollout_end_ordinal < ?3 AND rollout_end_ordinal <= ?4 + AND (rollout_end_ordinal, 3, turn_id) < (?4, ?5, ?6) +ORDER BY rollout_end_ordinal DESC, turn_id DESC LIMIT ?7 +) +SELECT * FROM starts +UNION ALL SELECT * FROM items +UNION ALL SELECT * FROM realtime +UNION ALL SELECT * FROM ends +ORDER BY rollout_ordinal DESC, kind DESC, id DESC +LIMIT ?7 + "#, + ) + .bind(segment.rollout_id().to_string()) + .bind(sqlite_integer(segment.start_ordinal(), "rollout ordinal")?) + .bind(upper) + .bind(cursor_ordinal) + .bind(i64::from(cursor_kind)) + .bind(cursor_id) + .bind(i64::try_from(remaining).map_err(thread_history_error)?) + .fetch_all(pool) + .await + .map_err(thread_history_error)?; + + for (position, kind, _id, turn_id, item_json) in segment_rows { + let position = u64::try_from(position).map_err(thread_history_error)?; + rows.push(match kind { + 0 | 3 => serde_json::from_str::(&item_json) + .map_err(thread_history_error)?, + 1 => ThreadTimelineEntry::Item { + position, + turn_id: turn_id.ok_or_else(|| thread_history_error("missing turn ID"))?, + item: Box::new( + serde_json::from_str::(&item_json) + .map_err(thread_history_error)?, + ), + }, + 2 => ThreadTimelineEntry::Realtime { + position, + item: serde_json::from_str::(&item_json) + .map_err(thread_history_error)? + .into(), + }, + _ => return Err(thread_history_error("invalid timeline entry kind")), + }); + } + } + + let has_more = rows.len() > params.page_size; + rows.truncate(params.page_size); + let next_cursor = if has_more { + rows.last() + .map(|entry| { + let (position, kind, id) = entry_key(entry); + serde_json::to_string(&TimelineCursor { + thread_id: params.thread_id, + position, + kind, + id: id.to_string(), + }) + .map_err(thread_history_error) + }) + .transpose()? + } else { + None + }; + + let (page_start, page_start_kind, page_start_id) = + rows.last().map(entry_key).unwrap_or_default(); + let mut active_realtime_session_at_page_start = None; + for segment in lineage.segments().iter().rev() { + let upper = page_start.min( + segment + .end_ordinal() + .map(|ordinal| ordinal.saturating_sub(1)) + .unwrap_or(u64::MAX), + ); + if upper < segment.start_ordinal() { + continue; + } + let boundary = sqlx::query_as::<_, (String, String)>( + r#" +SELECT item_type, json_extract(item_json, '$.realtime_session_id') +FROM thread_realtime_items +WHERE thread_id = ? + AND rollout_ordinal >= ? + AND rollout_ordinal <= ? + AND (rollout_ordinal, 2, item_id) < (?, ?, ?) + AND item_type IN ('realtime_session_started', 'realtime_session_closed') +ORDER BY rollout_ordinal DESC +LIMIT 1 + "#, + ) + .bind(segment.rollout_id().to_string()) + .bind(sqlite_integer(segment.start_ordinal(), "rollout ordinal")?) + .bind(sqlite_integer(upper, "rollout ordinal")?) + .bind(sqlite_integer(page_start, "rollout ordinal")?) + .bind(i64::from(page_start_kind)) + .bind(page_start_id) + .fetch_optional(pool) + .await + .map_err(thread_history_error)?; + if let Some((item_type, session_id)) = boundary { + if item_type == "realtime_session_started" { + active_realtime_session_at_page_start = Some(session_id); + } + break; + } + } + + rows.reverse(); + Ok(TimelinePage { + items: rows, + next_cursor, + active_realtime_session_at_page_start, + }) +} diff --git a/codex-rs/thread-store/src/local/thread_history_materialization.rs b/codex-rs/thread-store/src/local/thread_history_materialization.rs index e110ecc2f4..a63ed7fcb1 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization.rs @@ -5,6 +5,7 @@ use chrono::DateTime; use codex_app_server_protocol::ThreadHistoryChangeSet; use codex_app_server_protocol::project_rollout_line; use codex_protocol::ThreadId; +use codex_rollout::RolloutItem; use tokio::io::AsyncReadExt; use tokio::io::AsyncSeekExt; use tracing::warn; @@ -204,7 +205,9 @@ async fn read_projection_steps( ), }); } - let changes = if subagent_history_start_ordinal.is_some_and(|start| ordinal < start) { + let is_inherited_subagent_history = + subagent_history_start_ordinal.is_some_and(|start| ordinal < start); + let changes = if is_inherited_subagent_history { ThreadHistoryChangeSet::default() } else { project_rollout_line(&line) @@ -213,6 +216,8 @@ async fn read_projection_steps( .changed_items .iter() .any(|item| item.started_at_ms.is_none()) + || (!is_inherited_subagent_history + && matches!(&line.item, RolloutItem::RealtimeItem(_))) { match DateTime::parse_from_rfc3339(line.timestamp.as_str()) { Ok(timestamp) => Some(timestamp.timestamp_millis()), @@ -259,13 +264,19 @@ async fn read_projection_steps( .ok_or_else(|| ThreadStoreError::Internal { message: "rollout ordinal exceeds SQLite integer range".to_string(), })?; - projections.push(RolloutProjectionStep::Line(ProjectedRolloutLine { - ordinal, - start_byte_offset: line_start_offset, - end_byte_offset: line_end_offset, - fallback_created_at_ms, - changes, - })); + projections.push(RolloutProjectionStep::Line(Box::new( + ProjectedRolloutLine { + ordinal, + start_byte_offset: line_start_offset, + end_byte_offset: line_end_offset, + fallback_created_at_ms, + changes, + realtime_item: match line.item { + RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => Some(item), + _ => None, + }, + }, + ))); next_ordinal = next_line_ordinal; next_offset = line_end_offset; line_start_offset = line_end_offset; diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs index 42f1df3533..4354e39244 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs @@ -26,6 +26,11 @@ use codex_protocol::protocol::TokenCountEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::protocol::UserMessageEvent; +use codex_protocol::realtime::BemItemPresentation; +use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_protocol::realtime::RealtimeSessionOutcome; +use codex_protocol::realtime::RealtimeTranscriptRole; use codex_rollout::RolloutConfig; use codex_rollout::RolloutItem; use codex_rollout::RolloutLine; @@ -426,6 +431,216 @@ WHERE thread_id = ? assert_eq!(projection_state, (rollout_len, 5)); } +#[tokio::test] +async fn paginated_realtime_items_materialize_separately_in_rollout_order() { + let home = TempDir::new().expect("temp dir"); + let store = projection_store(home.path()).await; + let thread_id = ThreadId::default(); + create_paginated_thread(&store, thread_id).await; + store + .persist_thread(thread_id, PersistContext::Standard) + .await + .expect("persist thread metadata"); + + store + .append_items(AppendThreadItemsParams { + thread_id, + items: vec![ + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice:started".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }), + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice:transcript:0".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::TranscriptSegment { + role: RealtimeTranscriptRole::Assistant, + text: "Here is the result.".to_string(), + }, + }), + turn_started("turn-1"), + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice:promoted:agent-1".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::BemItemPromoted { + turn_id: "turn-1".to_string(), + item_id: "agent-1".to_string(), + presentation: BemItemPresentation::InlineMarkdown, + }, + }), + completed_item( + thread_id, + "turn-1", + TurnItem::AgentMessage(AgentMessageItem { + id: "agent-1".to_string(), + content: vec![AgentMessageContent::Text { + text: "Result".to_string(), + }], + phase: None, + memory_citation: None, + delivery: None, + }), + ), + RolloutItem::RealtimeItem(RealtimeItem { + id: "voice:closed".to_string(), + realtime_session_id: "voice".to_string(), + content: RealtimeItemContent::RealtimeSessionClosed { + outcome: RealtimeSessionOutcome::Ended, + }, + }), + ], + }) + .await + .expect("append interleaved realtime and ordinary rollout items"); + + let rollout_path = store + .live_rollout_path(thread_id) + .await + .expect("rollout path"); + let expected_rows = fs::read_to_string(rollout_path.as_path()) + .expect("read canonical rollout") + .lines() + .map(|line| serde_json::from_str::(line).expect("parse rollout line")) + .filter_map(|line| match line.item { + RolloutItem::RealtimeItem(item) => Some(( + item.id, + i64::try_from(line.ordinal.expect("paginated rollout ordinal")) + .expect("ordinal fits SQLite integer"), + chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str()) + .expect("valid rollout timestamp") + .timestamp_millis(), + serde_json::to_value(item.content) + .expect("serialize realtime content") + .get("type") + .and_then(serde_json::Value::as_str) + .expect("realtime content discriminator") + .to_string(), + )), + _ => None, + }) + .collect::>(); + assert_eq!( + expected_rows + .iter() + .map(|(id, ordinal, _, item_type)| (id.as_str(), *ordinal, item_type.as_str())) + .collect::>(), + vec![ + ("voice:started", 1, "realtime_session_started"), + ("voice:transcript:0", 2, "transcript_segment"), + ("voice:promoted:agent-1", 4, "bem_item_promoted"), + ("voice:closed", 6, "realtime_session_closed"), + ] + ); + + let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing( + home.path().abs(), + )) + .await + .expect("open thread history db"); + let projected_rows = sqlx::query_as::<_, (String, i64, i64, String)>( + "SELECT item_id, rollout_ordinal, created_at_ms, item_type FROM thread_realtime_items WHERE thread_id = ? ORDER BY rollout_ordinal", + ) + .bind(thread_id.to_string()) + .fetch_all(&pool) + .await + .expect("read separately projected realtime items"); + assert_eq!(projected_rows, expected_rows); + + let ordinary_items = sqlx::query_as::<_, (String, String)>( + "SELECT turn_id, item_id FROM thread_items WHERE thread_id = ? ORDER BY rollout_ordinal", + ) + .bind(thread_id.to_string()) + .fetch_all(&pool) + .await + .expect("read ordinary turn items"); + assert_eq!( + ordinary_items, + vec![("turn-1".to_string(), "agent-1".to_string())] + ); + + sqlx::query("DELETE FROM thread_history_projection_state WHERE thread_id = ?") + .bind(thread_id.to_string()) + .execute(&pool) + .await + .expect("rewind projection checkpoint"); + super::materialize_to_sqlite(&store, thread_id, rollout_path.as_path()) + .await + .expect("replay canonical rollout"); + + let replayed_rows = sqlx::query_as::<_, (String, i64, i64, String)>( + "SELECT item_id, rollout_ordinal, created_at_ms, item_type FROM thread_realtime_items WHERE thread_id = ? ORDER BY rollout_ordinal", + ) + .bind(thread_id.to_string()) + .fetch_all(&pool) + .await + .expect("read replayed realtime items"); + assert_eq!(replayed_rows, expected_rows); + + let legacy_thread_id = ThreadId::new(); + store + .create_thread(CreateThreadParams { + session_id: legacy_thread_id.into(), + thread_id: legacy_thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Exec, + thread_source: None, + originator: "test_originator".to_string(), + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + selected_capability_roots: Vec::new(), + multi_agent_version: None, + history_mode: ThreadHistoryMode::Legacy, + history_base: None, + subagent_history_start_ordinal: None, + initial_window_id: "window-1".to_string(), + metadata: ThreadPersistenceMetadata { + cwd: Some(home.path().to_path_buf()), + model_provider: "test-provider".to_string(), + memory_mode: ThreadMemoryMode::Enabled, + }, + }) + .await + .expect("create legacy thread"); + store + .persist_thread(legacy_thread_id, PersistContext::Standard) + .await + .expect("persist legacy thread metadata"); + store + .append_items(AppendThreadItemsParams { + thread_id: legacy_thread_id, + items: vec![RolloutItem::RealtimeItem(RealtimeItem { + id: "legacy:started".to_string(), + realtime_session_id: "legacy".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + })], + }) + .await + .expect("ignore realtime item for legacy rollout"); + let legacy_rollout_path = store + .live_rollout_path(legacy_thread_id) + .await + .expect("legacy rollout path"); + let (legacy_items, _, _) = RolloutRecorder::load_rollout_items(legacy_rollout_path.as_path()) + .await + .expect("load legacy rollout"); + assert!( + !legacy_items + .iter() + .any(|item| matches!(item, RolloutItem::RealtimeItem(_))) + ); + let legacy_realtime_count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM thread_realtime_items WHERE thread_id = ?", + ) + .bind(legacy_thread_id.to_string()) + .fetch_one(&pool) + .await + .expect("count legacy realtime items"); + assert_eq!(legacy_realtime_count, 0); +} + #[tokio::test] async fn referenced_paginated_rollout_projects_inherited_ordinal_range() { let home = TempDir::new().expect("temp dir"); @@ -1010,7 +1225,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { &store, thread_id, /*history_base*/ None, - /*subagent_history_start_ordinal*/ Some(4), + /*subagent_history_start_ordinal*/ Some(5), ) .await; store @@ -1022,6 +1237,11 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { .append_items(AppendThreadItemsParams { thread_id, items: vec![ + RolloutItem::RealtimeItem(RealtimeItem { + id: "parent:started".to_string(), + realtime_session_id: "parent".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }), turn_started("parent-turn"), completed_item( thread_id, @@ -1034,6 +1254,11 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { ), turn_completed("parent-turn"), turn_started("child-turn"), + RolloutItem::RealtimeItem(RealtimeItem { + id: "child:started".to_string(), + realtime_session_id: "child".to_string(), + content: RealtimeItemContent::RealtimeSessionStarted, + }), completed_item( thread_id, "child-turn", @@ -1054,7 +1279,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { .await .expect("rollout path"); let (child_start_byte_offset, _) = - rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 4); + rollout_line_byte_offsets(rollout_path.as_path(), /*ordinal*/ 5); let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing( home.path().abs(), )) @@ -1069,7 +1294,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { .expect("read projected turns"); assert_eq!( turns, - vec![("child-turn".to_string(), 4, Some(child_start_byte_offset))] + vec![("child-turn".to_string(), 5, Some(child_start_byte_offset))] ); let items = sqlx::query_as::<_, (String, i64)>( "SELECT item_id, rollout_ordinal FROM thread_items WHERE thread_id = ?", @@ -1078,8 +1303,16 @@ async fn subagent_prefix_advances_projection_without_materializing_history() { .fetch_all(&pool) .await .expect("read projected items"); - assert_eq!(items, vec![("child-user".to_string(), 5)]); - assert_eq!(projection_state(&pool, thread_id).await.1, 7); + assert_eq!(items, vec![("child-user".to_string(), 7)]); + let realtime_items = sqlx::query_as::<_, (String, i64)>( + "SELECT item_id, rollout_ordinal FROM thread_realtime_items WHERE thread_id = ?", + ) + .bind(thread_id.to_string()) + .fetch_all(&pool) + .await + .expect("read projected realtime items"); + assert_eq!(realtime_items, vec![("child:started".to_string(), 6)]); + assert_eq!(projection_state(&pool, thread_id).await.1, 9); } #[tokio::test] diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 5cc582c51f..1e7c0637a8 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -363,6 +363,18 @@ pub trait ThreadStore: Any + Send + Sync { }) } + /// Lists bounded ordinary and realtime thread history in rollout order. + fn list_timeline( + &self, + _params: crate::ListTimelineParams, + ) -> ThreadStoreFuture<'_, crate::TimelinePage> { + Box::pin(async { + Err(ThreadStoreError::Unsupported { + operation: "thread/timeline/list", + }) + }) + } + /// Applies a literal metadata patch and returns the updated thread when one was materialized. /// /// `None` means the update succeeded without materializing a thread, for example because the diff --git a/codex-rs/thread-store/src/thread_metadata_sync.rs b/codex-rs/thread-store/src/thread_metadata_sync.rs index 9a4a67916d..76f3f1ce5a 100644 --- a/codex-rs/thread-store/src/thread_metadata_sync.rs +++ b/codex-rs/thread-store/src/thread_metadata_sync.rs @@ -304,6 +304,7 @@ impl ThreadMetadataSync { | RolloutItem::InterAgentCommunication(_) | RolloutItem::InterAgentCommunicationMetadata { .. } | RolloutItem::Compacted(_) + | RolloutItem::RealtimeItem(_) | RolloutItem::SecurityRiskScore(_) | RolloutItem::WorldState(_) => {} } diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 192e6bed6e..496a076478 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; use codex_app_server_protocol::CodexErrorInfo; +use codex_app_server_protocol::ThreadTimelineEntry; use codex_protocol::SessionId; use codex_protocol::ThreadId; use codex_protocol::capabilities::SelectedCapabilityRoot; @@ -510,6 +511,22 @@ pub struct ItemPage { pub backwards_cursor: Option, } +/// Parameters for reading a bounded mixed thread timeline. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ListTimelineParams { + pub thread_id: ThreadId, + pub cursor: Option, + pub page_size: usize, +} + +/// Ordinary items, realtime facts, and the session state preceding their page. +#[derive(Clone, Debug, PartialEq)] +pub struct TimelinePage { + pub items: Vec, + pub next_cursor: Option, + pub active_realtime_session_at_page_start: Option, +} + /// Parameters for searching visible message occurrences within one paginated thread. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SearchThreadOccurrencesParams { diff --git a/codex-rs/tui/src/app/app_server_event_targets.rs b/codex-rs/tui/src/app/app_server_event_targets.rs index 19bb5db25f..f347964596 100644 --- a/codex-rs/tui/src/app/app_server_event_targets.rs +++ b/codex-rs/tui/src/app/app_server_event_targets.rs @@ -148,6 +148,15 @@ pub(super) fn server_notification_thread_target( ServerNotification::ThreadRealtimeItemAdded(notification) => { Some(notification.thread_id.as_str()) } + ServerNotification::ThreadRealtimeItemStarted(notification) => { + Some(notification.thread_id.as_str()) + } + ServerNotification::ThreadRealtimeItemTranscriptDelta(notification) => { + Some(notification.thread_id.as_str()) + } + ServerNotification::ThreadRealtimeItemCompleted(notification) => { + Some(notification.thread_id.as_str()) + } ServerNotification::ThreadRealtimeTranscriptDelta(notification) => { Some(notification.thread_id.as_str()) } diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 8d81edbe0b..d487babe1b 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -238,6 +238,9 @@ impl ChatWidget { | ServerNotification::FuzzyFileSearchSessionCompleted(_) | ServerNotification::ThreadRealtimeStarted(_) | ServerNotification::ThreadRealtimeItemAdded(_) + | ServerNotification::ThreadRealtimeItemStarted(_) + | ServerNotification::ThreadRealtimeItemTranscriptDelta(_) + | ServerNotification::ThreadRealtimeItemCompleted(_) | ServerNotification::ThreadRealtimeOutputAudioDelta(_) | ServerNotification::ThreadRealtimeError(_) | ServerNotification::ThreadRealtimeClosed(_)