Add realtime voice selection (#17176)

- Add realtime voice selection for realtime/start.
- Expose the supported v1/v2 voice lists and cover explicit, configured,
default, and invalid voice paths.
This commit is contained in:
Ahmed Ibrahim authored and GitHub committed 2026-04-08 20:19:15 -07:00
1 parent 4c2a1ae31b
commit 2f9090be62
36 files changed
+860 -33

No files matched your search

+19
View File
@@ -4626,6 +4626,10 @@ async fn submission_loop(sess: Arc<Session>, config: Arc<Config>, rx_sub: Receiv
handle_realtime_conversation_close(&sess, sub.id.clone()).await;
false
}
Op::RealtimeConversationListVoices => {
handlers::realtime_conversation_list_voices(&sess, sub.id.clone()).await;
false
}
Op::OverrideTurnContext {
cwd,
approval_policy,
@@ -4860,6 +4864,8 @@ mod handlers {
use codex_protocol::protocol::ListSkillsResponseEvent;
use codex_protocol::protocol::McpServerRefreshConfig;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_protocol::protocol::ReviewDecision;
use codex_protocol::protocol::ReviewRequest;
use codex_protocol::protocol::RolloutItem;
@@ -4894,6 +4900,18 @@ mod handlers {
sess.close_unified_exec_processes().await;
}
pub async fn realtime_conversation_list_voices(sess: &Session, sub_id: String) {
sess.send_event_raw(Event {
id: sub_id,
msg: EventMsg::RealtimeConversationListVoicesResponse(
RealtimeConversationListVoicesResponseEvent {
voices: RealtimeVoicesList::builtin(),
},
),
})
.await;
}
pub async fn override_turn_context(
sess: &Session,
sub_id: String,
@@ -7189,6 +7207,7 @@ fn realtime_text_for_event(msg: &EventMsg) -> Option<String> {
| EventMsg::GetHistoryEntryResponse(_)
| EventMsg::McpListToolsResponse(_)
| EventMsg::ListSkillsResponse(_)
| EventMsg::RealtimeConversationListVoicesResponse(_)
| EventMsg::SkillsUpdateAvailable
| EventMsg::PlanUpdate(_)
| EventMsg::TurnAborted(_)
+48
View File
@@ -75,6 +75,9 @@ use codex_protocol::protocol::NetworkApprovalProtocol;
use codex_protocol::protocol::RateLimitSnapshot;
use codex_protocol::protocol::RateLimitWindow;
use codex_protocol::protocol::RealtimeAudioFrame;
use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_protocol::protocol::ResumedHistory;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::Submission;
@@ -4588,6 +4591,51 @@ async fn run_user_shell_command_does_not_set_reference_context_item() {
);
}
#[tokio::test]
async fn realtime_conversation_list_voices_emits_builtin_list() {
let (session, _turn_context, rx) = make_session_and_context_with_rx().await;
handlers::realtime_conversation_list_voices(&session, "sub-id".to_string()).await;
let event = rx.recv().await.expect("event");
let voices = match event.msg {
EventMsg::RealtimeConversationListVoicesResponse(
RealtimeConversationListVoicesResponseEvent { voices },
) => voices,
msg => panic!("expected list voices response, got {msg:?}"),
};
assert_eq!(
voices,
RealtimeVoicesList {
v1: vec![
RealtimeVoice::Juniper,
RealtimeVoice::Maple,
RealtimeVoice::Spruce,
RealtimeVoice::Ember,
RealtimeVoice::Vale,
RealtimeVoice::Breeze,
RealtimeVoice::Arbor,
RealtimeVoice::Sol,
RealtimeVoice::Cove,
],
v2: vec![
RealtimeVoice::Alloy,
RealtimeVoice::Ash,
RealtimeVoice::Ballad,
RealtimeVoice::Coral,
RealtimeVoice::Echo,
RealtimeVoice::Sage,
RealtimeVoice::Shimmer,
RealtimeVoice::Verse,
RealtimeVoice::Marin,
RealtimeVoice::Cedar,
],
default_v1: RealtimeVoice::Cove,
default_v2: RealtimeVoice::Marin,
},
);
}
#[derive(Clone, Copy)]
struct NeverEndingTask {
kind: TaskKind,
+4
View File
@@ -53,6 +53,7 @@ use codex_protocol::permissions::FileSystemSandboxPolicy;
use codex_protocol::permissions::FileSystemSpecialPath;
use codex_protocol::permissions::NetworkSandboxPolicy;
use codex_protocol::protocol::ReadOnlyAccess;
use codex_protocol::protocol::RealtimeVoice;
use serde::Deserialize;
use tempfile::tempdir;
@@ -6489,6 +6490,7 @@ fn realtime_loads_from_config_toml() -> std::io::Result<()> {
version = "v2"
type = "transcription"
transport = "webrtc"
voice = "cedar"
"#,
)
.expect("TOML deserialization should succeed");
@@ -6499,6 +6501,7 @@ transport = "webrtc"
version: Some(RealtimeWsVersion::V2),
session_type: Some(RealtimeWsMode::Transcription),
transport: Some(RealtimeTransport::WebRtc),
voice: Some(RealtimeVoice::Cedar),
})
);
@@ -6515,6 +6518,7 @@ transport = "webrtc"
version: RealtimeWsVersion::V2,
session_type: RealtimeWsMode::Transcription,
transport: RealtimeTransport::WebRtc,
voice: Some(RealtimeVoice::Cedar),
}
);
Ok(())
+12
View File
@@ -1019,6 +1019,18 @@ impl ConfigEditsBuilder {
self
}
pub fn set_realtime_voice(mut self, voice: Option<&str>) -> Self {
let segments = vec!["realtime".to_string(), "voice".to_string()];
match voice {
Some(voice) => self.edits.push(ConfigEdit::SetPath {
segments,
value: value(voice),
}),
None => self.edits.push(ConfigEdit::ClearPath { segments }),
}
self
}
pub fn clear_legacy_windows_sandbox_keys(mut self) -> Self {
for key in [
"experimental_windows_sandbox",
+35
View File
@@ -1066,6 +1066,41 @@ fn blocking_builder_set_realtime_audio_persists_and_clears() {
);
}
#[test]
fn blocking_builder_set_realtime_voice_persists_and_clears() {
let tmp = tempdir().expect("tmpdir");
let codex_home = tmp.path();
ConfigEditsBuilder::new(codex_home)
.set_realtime_voice(Some("cedar"))
.apply_blocking()
.expect("persist realtime voice");
let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config");
let config: TomlValue = toml::from_str(&raw).expect("parse config");
let realtime = config
.get("realtime")
.and_then(TomlValue::as_table)
.expect("realtime table should exist");
assert_eq!(
realtime.get("voice").and_then(TomlValue::as_str),
Some("cedar")
);
ConfigEditsBuilder::new(codex_home)
.set_realtime_voice(/*voice*/ None)
.apply_blocking()
.expect("clear realtime voice");
let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config");
let config: TomlValue = toml::from_str(&raw).expect("parse config");
let realtime = config
.get("realtime")
.and_then(TomlValue::as_table)
.expect("realtime table should exist");
assert_eq!(realtime.get("voice"), None);
}
#[test]
fn replace_mcp_servers_blocking_clears_table_when_empty() {
let tmp = tempdir().expect("tmpdir");
+1
View File
@@ -2098,6 +2098,7 @@ impl Config {
version: realtime.version.unwrap_or_default(),
session_type: realtime.session_type.unwrap_or_default(),
transport: realtime.transport.unwrap_or_default(),
voice: realtime.voice,
}),
experimental_realtime_ws_backend_prompt: cfg.experimental_realtime_ws_backend_prompt,
experimental_realtime_ws_startup_context: cfg.experimental_realtime_ws_startup_context,
+42 -1
View File
@@ -39,6 +39,8 @@ use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationSdpEvent;
use codex_protocol::protocol::RealtimeConversationStartedEvent;
use codex_protocol::protocol::RealtimeHandoffRequested;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use http::HeaderMap;
use http::HeaderValue;
use http::header::AUTHORIZATION;
@@ -521,7 +523,7 @@ async fn prepare_realtime_start(
}
let version = config.realtime.version;
let session_config =
build_realtime_session_config(sess, params.prompt, params.session_id).await?;
build_realtime_session_config(sess, params.prompt, params.session_id, params.voice).await?;
let requested_session_id = session_config.session_id.clone();
let extra_headers = match transport {
ConversationStartTransport::Websocket => {
@@ -549,6 +551,7 @@ pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
prompt: Option<Option<String>>,
session_id: Option<String>,
voice: Option<RealtimeVoice>,
) -> CodexResult<RealtimeSessionConfig> {
let config = sess.get_config().await;
let prompt = prepare_realtime_backend_prompt(
@@ -578,15 +581,53 @@ pub(crate) async fn build_realtime_session_config(
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
};
let voice = voice
.or(config.realtime.voice)
.unwrap_or_else(|| default_realtime_voice(config.realtime.version));
validate_realtime_voice(config.realtime.version, voice)?;
Ok(RealtimeSessionConfig {
instructions: prompt,
model,
session_id: Some(session_id.unwrap_or_else(|| sess.conversation_id.to_string())),
event_parser,
session_mode,
voice,
})
}
fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice {
let voices = RealtimeVoicesList::builtin();
match version {
RealtimeWsVersion::V1 => voices.default_v1,
RealtimeWsVersion::V2 => voices.default_v2,
}
}
fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> {
let voices = RealtimeVoicesList::builtin();
let allowed = match version {
RealtimeWsVersion::V1 => &voices.v1,
RealtimeWsVersion::V2 => &voices.v2,
};
if allowed.contains(&voice) {
return Ok(());
}
let version = match version {
RealtimeWsVersion::V1 => "v1",
RealtimeWsVersion::V2 => "v2",
};
let allowed = allowed
.iter()
.map(|voice| voice.wire_name())
.collect::<Vec<_>>()
.join(", ");
Err(CodexErr::InvalidRequest(format!(
"realtime voice `{}` is not supported for {version}; supported voices: {allowed}",
voice.wire_name()
)))
}
async fn handle_start_inner(
sess: &Arc<Session>,
sub_id: &str,