github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-25 07:55:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support attaching to existing realtime calls (#39876)

Browse Source 
## Why

Clients that create and negotiate realtime calls themselves need Codex to join
those calls without creating another call or overwriting the client-owned
session configuration.

## What changed

- Add the `existingCall` transport to `thread/realtime/start`, accepting a
  client-provided `callId` and optional `realtimeSessionId`.
- Attach Codex over the call's sideband WebSocket without SDP negotiation or a
  session update, and reconnect that sideband while preserving transcript
  state.
- Default existing calls to realtime v1, support v1 and v3, and reject v2 or
  options that would reconfigure the client-owned session.
- Encode call IDs as a single URL path segment for v3 sideband connections.

## Testing

Add protocol, app-server, core, reconnection, authentication-header, and URL
encoding coverage for existing-call attachments.

GitOrigin-RevId: 2046b70566f98efa55381bb1e461ea3cc68256c5
This commit is contained in:
Zahan Malkani
2026-08-21 06:41:28 +00:00
committed by copyberry
parent 2aaefa32b0
commit 536f86e5cc
18 changed files with 765 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/client.rs
+14
View File
@@ -686,6 +686,20 @@ impl ModelClient {
})
}
pub(crate) async fn realtime_sideband_headers(
&self,
mut extra_headers: ApiHeaderMap,
) -> Result<ApiHeaderMap> {
let client_setup = self.current_client_setup().await?;
if let Some(header_value) = self.generate_attestation_header_for().await {
extra_headers.insert(X_OAI_ATTESTATION_HEADER, header_value);
}
extra_headers.extend(sideband_websocket_auth_headers(
client_setup.api_auth.as_ref(),
));
Ok(extra_headers)
}
/// Builds memory summaries for each provided normalized raw memory.
///
/// This is a unary call (no streaming) to `/v1/memories/trace_summarize`.
codex-rs/core/src/client_tests.rs
+19
View File
@@ -974,6 +974,25 @@ async fn websocket_handshake_includes_attestation_for_chatgpt_codex_responses()
assert_eq!(attestation_calls.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn existing_call_sideband_headers_include_attestation() {
let (model_client, attestation_calls) =
model_client_with_counting_attestation(/*include_attestation*/ true);
let headers = model_client
.realtime_sideband_headers(http::HeaderMap::new())
.await
.expect("existing call sideband headers should build");
assert_eq!(
headers
.get(crate::attestation::X_OAI_ATTESTATION_HEADER)
.and_then(|value| value.to_str().ok()),
Some("v1.header-1"),
);
assert_eq!(attestation_calls.load(Ordering::Relaxed), 1);
}
#[tokio::test]
async fn non_chatgpt_codex_endpoints_omit_attestation_generation() {
let (model_client, attestation_calls) =
codex-rs/core/src/realtime_conversation.rs
+70 -9
View File
@@ -22,6 +22,7 @@ use codex_api::RealtimeEventParser;
use codex_api::RealtimeSessionConfig;
use codex_api::RealtimeSessionMode;
use codex_api::RealtimeWebsocketClient;
use codex_api::RealtimeWebsocketConnection;
use codex_api::RealtimeWebsocketEvents;
use codex_api::RealtimeWebsocketWriter;
use codex_api::build_session_headers;
@@ -80,6 +81,7 @@ use tracing::info;
use tracing::warn;
mod bem;
mod existing_call;
mod sideband;
use self::bem::ChannelParser as BemChannelParser;
@@ -478,6 +480,7 @@ struct RealtimeStart {
session_config: RealtimeSessionConfig,
model_client: ModelClient,
sdp: Option<String>,
existing_call_id: Option<String>,
}
struct RealtimeStartOutput {
@@ -550,6 +553,7 @@ impl RealtimeConversationManager {
session_config,
model_client,
sdp,
existing_call_id,
} = start;
let event_parser = session_config.event_parser;
let session_kind = match event_parser {
@@ -612,6 +616,7 @@ impl RealtimeConversationManager {
session_config,
call_id: call.call_id,
sideband_headers: call.sideband_headers,
session_initialization: RealtimeSidebandSessionInitialization::CoreCreated,
input_channels,
events_tx,
handoff_state: handoff.clone(),
@@ -622,6 +627,24 @@ impl RealtimeConversationManager {
stop_token: stop_token.clone(),
});
(task, Some(call.sdp))
} else if let Some(call_id) = existing_call_id {
let task = existing_call::attach(existing_call::ExistingCallAttachment {
client,
model_client,
session_config,
call_id,
extra_headers: extra_headers.unwrap_or_default(),
input_channels,
events_tx,
handoff_state: handoff.clone(),
session_kind,
event_parser,
realtime_active: Arc::clone(&realtime_active),
transcript_tail_flush,
stop_token: stop_token.clone(),
})
.await?;
(task, None)
} else {
let connection = client
.connect(
@@ -1173,19 +1196,46 @@ async fn prepare_realtime_start(
};
let version = params.version.unwrap_or(match &transport {
ConversationStartTransport::Websocket => config.realtime.version,
ConversationStartTransport::Webrtc { .. } => RealtimeWsVersion::V1,
ConversationStartTransport::Webrtc { .. }
| ConversationStartTransport::ExistingCall { .. } => RealtimeWsVersion::V1,
});
if matches!(transport, ConversationStartTransport::Webrtc { .. }) {
validate_avas_webrtc_start(version, config.realtime.session_type)?;
match &transport {
ConversationStartTransport::Webrtc { .. } => {
validate_avas_webrtc_start(version, config.realtime.session_type)?;
}
ConversationStartTransport::ExistingCall { .. } => {
if version == RealtimeWsVersion::V2 {
return Err(CodexErr::InvalidRequest(
"AVAS realtime calls require realtime v1 or v3".to_string(),
));
}
if params.include_startup_context
|| params.prompt.is_some()
|| !params.initial_items.is_empty()
|| params.model.is_some()
|| params.voice.is_some()
|| params.delegation_ack_filler.is_some()
{
return Err(CodexErr::InvalidRequest(
"existing realtime calls do not support session configuration options"
.to_string(),
));
}
}
ConversationStartTransport::Websocket => {}
}
let configured_voice = match (&transport, params.version) {
(ConversationStartTransport::Webrtc { .. }, None) => ConfiguredRealtimeVoice::Ignore,
(ConversationStartTransport::ExistingCall { .. }, _)
| (ConversationStartTransport::Webrtc { .. }, None) => ConfiguredRealtimeVoice::Ignore,
(ConversationStartTransport::Webrtc { .. } | ConversationStartTransport::Websocket, _) => {
ConfiguredRealtimeVoice::Use
}
};
let session_config =
let mut session_config =
build_realtime_session_config(sess, &params, version, configured_voice).await?;
if matches!(&transport, ConversationStartTransport::ExistingCall { .. }) {
session_config.session_id = params.realtime_session_id.clone();
}
let requested_realtime_session_id = session_config.session_id.clone();
let event_parser = session_config.event_parser;
let originator = sess.originator().await;
@@ -1199,7 +1249,8 @@ async fn prepare_realtime_start(
originator.as_str(),
)?
}
ConversationStartTransport::Webrtc { .. } => {
ConversationStartTransport::Webrtc { .. }
| ConversationStartTransport::ExistingCall { .. } => {
realtime_request_headers(
requested_realtime_session_id.as_deref(),
/*api_key*/ None,
@@ -1454,9 +1505,10 @@ async fn handle_start_inner(
transport,
} = prepared_start;
info!("starting realtime conversation");
let sdp = match transport {
ConversationStartTransport::Websocket => None,
ConversationStartTransport::Webrtc { sdp } => Some(sdp),
let (sdp, existing_call_id) = match transport {
ConversationStartTransport::Websocket => (None, None),
ConversationStartTransport::Webrtc { sdp } => (Some(sdp), None),
ConversationStartTransport::ExistingCall { call_id } => (None, Some(call_id)),
};
let mode_instructions = RealtimeModeInstructions {
start: realtime_start_instructions,
@@ -1476,6 +1528,7 @@ async fn handle_start_inner(
session_config,
model_client: sess.services.model_client.clone(),
sdp,
existing_call_id,
};
let start_output = sess.conversation.start(start, mode_instructions).await?;
@@ -1757,6 +1810,7 @@ struct RealtimeWebrtcSidebandInputTask {
session_config: RealtimeSessionConfig,
call_id: String,
sideband_headers: HeaderMap,
session_initialization: RealtimeSidebandSessionInitialization,
input_channels: RealtimeInputChannels,
events_tx: Sender<RealtimeEvent>,
handoff_state: RealtimeHandoffState,
@@ -1767,6 +1821,13 @@ struct RealtimeWebrtcSidebandInputTask {
stop_token: CancellationToken,
}
enum RealtimeSidebandSessionInitialization {
CoreCreated,
ExistingCall {
initial_connection: Option<RealtimeWebsocketConnection>,
},
}
async fn run_realtime_input_task(
input: RealtimeInputTask,
pending_outbound: Option<Box<RealtimePendingOutbound>>,
codex-rs/core/src/realtime_conversation/existing_call.rs
+90
View File
@@ -0,0 +1,90 @@
use super::RealtimeHandoffState;
use super::RealtimeInputChannels;
use super::RealtimeSessionKind;
use super::RealtimeSidebandSessionInitialization;
use super::RealtimeTranscriptTailFlush;
use super::RealtimeWebrtcSidebandInputTask;
use super::spawn_webrtc_sideband_input_task;
use crate::client::ModelClient;
use async_channel::Sender;
use codex_api::RealtimeEvent;
use codex_api::RealtimeEventParser;
use codex_api::RealtimeSessionConfig;
use codex_api::RealtimeTranscriptState;
use codex_api::RealtimeWebsocketClient;
use codex_api::map_api_error;
use codex_login::default_client::default_headers;
use codex_protocol::error::Result as CodexResult;
use http::HeaderMap;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
pub(super) struct ExistingCallAttachment {
pub(super) client: RealtimeWebsocketClient,
pub(super) model_client: ModelClient,
pub(super) session_config: RealtimeSessionConfig,
pub(super) call_id: String,
pub(super) extra_headers: HeaderMap,
pub(super) input_channels: RealtimeInputChannels,
pub(super) events_tx: Sender<RealtimeEvent>,
pub(super) handoff_state: RealtimeHandoffState,
pub(super) session_kind: RealtimeSessionKind,
pub(super) event_parser: RealtimeEventParser,
pub(super) realtime_active: Arc<AtomicBool>,
pub(super) transcript_tail_flush: RealtimeTranscriptTailFlush,
pub(super) stop_token: CancellationToken,
}
pub(super) async fn attach(attachment: ExistingCallAttachment) -> CodexResult<JoinHandle<()>> {
let ExistingCallAttachment {
client,
model_client,
session_config,
call_id,
extra_headers,
input_channels,
events_tx,
handoff_state,
session_kind,
event_parser,
realtime_active,
transcript_tail_flush,
stop_token,
} = attachment;
let sideband_headers = model_client
.realtime_sideband_headers(extra_headers)
.await?;
let transcript_state = RealtimeTranscriptState::default();
let connection = client
.connect_existing_call_sideband(
session_config.clone(),
&call_id,
sideband_headers.clone(),
default_headers(),
transcript_state,
)
.await
.map_err(map_api_error)?;
Ok(spawn_webrtc_sideband_input_task(
RealtimeWebrtcSidebandInputTask {
client,
session_config,
call_id,
sideband_headers,
session_initialization: RealtimeSidebandSessionInitialization::ExistingCall {
initial_connection: Some(connection),
},
input_channels,
events_tx,
handoff_state,
session_kind,
event_parser,
realtime_active,
transcript_tail_flush,
stop_token,
},
))
}
codex-rs/core/src/realtime_conversation/sideband.rs
+36 -8
View File
@@ -1,5 +1,6 @@
use super::RealtimeInputTask;
use super::RealtimeInputTaskExit;
use super::RealtimeSidebandSessionInitialization;
use super::RealtimeWebrtcSidebandInputTask;
use super::flush_realtime_transcript_tail;
use super::report_realtime_transport_loss;
@@ -31,6 +32,7 @@ pub(super) fn spawn_webrtc_sideband_input_task(
session_config,
call_id,
sideband_headers,
mut session_initialization,
input_channels,
events_tx,
handoff_state,
@@ -45,18 +47,44 @@ pub(super) fn spawn_webrtc_sideband_input_task(
let mut reconnecting = false;
let mut rapid_disconnects = 0_u32;
let mut pending_outbound = None;
let transcript_state = RealtimeTranscriptState::default();
let transcript_state = match &session_initialization {
RealtimeSidebandSessionInitialization::ExistingCall {
initial_connection: Some(connection),
} => connection.events().transcript_state(),
RealtimeSidebandSessionInitialization::CoreCreated
| RealtimeSidebandSessionInitialization::ExistingCall {
initial_connection: None,
} => RealtimeTranscriptState::default(),
};
while realtime_active.load(Ordering::Relaxed) {
let connection = match tokio::select! {
biased;
_ = stop_token.cancelled() => break,
connection = client.connect_webrtc_sideband(
session_config.clone(),
&call_id,
sideband_headers.clone(),
default_headers(),
transcript_state.clone(),
) => connection,
connection = async {
match &mut session_initialization {
RealtimeSidebandSessionInitialization::CoreCreated => {
client.connect_webrtc_sideband(
session_config.clone(),
&call_id,
sideband_headers.clone(),
default_headers(),
transcript_state.clone(),
).await
}
RealtimeSidebandSessionInitialization::ExistingCall { initial_connection } => {
match initial_connection.take() {
Some(connection) => Ok(connection),
None => client.connect_existing_call_sideband(
session_config.clone(),
&call_id,
sideband_headers.clone(),
default_headers(),
transcript_state.clone(),
).await,
}
}
}
} => connection,
} {
Ok(connection) => connection,
Err(err) => {