mirror of
https://github.com/openai/codex.git
synced 2026-11-05 16:29:31 +00:00
Add realtime voice selection (#17176)
- Add realtime voice selection for realtime/start. - Expose the supported v1/v2 voice lists and cover explicit, configured, default, and invalid voice paths.
This commit is contained in:
1 parent
4c2a1ae31b
commit
2f9090be62
36 files changed
+860
-33
No files matched your search
@@ -4626,6 +4626,10 @@ async fn submission_loop(sess: Arc<Session>, config: Arc<Config>, rx_sub: Receiv
|
||||
handle_realtime_conversation_close(&sess, sub.id.clone()).await;
|
||||
false
|
||||
}
|
||||
Op::RealtimeConversationListVoices => {
|
||||
handlers::realtime_conversation_list_voices(&sess, sub.id.clone()).await;
|
||||
false
|
||||
}
|
||||
Op::OverrideTurnContext {
|
||||
cwd,
|
||||
approval_policy,
|
||||
@@ -4860,6 +4864,8 @@ mod handlers {
|
||||
use codex_protocol::protocol::ListSkillsResponseEvent;
|
||||
use codex_protocol::protocol::McpServerRefreshConfig;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent;
|
||||
use codex_protocol::protocol::RealtimeVoicesList;
|
||||
use codex_protocol::protocol::ReviewDecision;
|
||||
use codex_protocol::protocol::ReviewRequest;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
@@ -4894,6 +4900,18 @@ mod handlers {
|
||||
sess.close_unified_exec_processes().await;
|
||||
}
|
||||
|
||||
pub async fn realtime_conversation_list_voices(sess: &Session, sub_id: String) {
|
||||
sess.send_event_raw(Event {
|
||||
id: sub_id,
|
||||
msg: EventMsg::RealtimeConversationListVoicesResponse(
|
||||
RealtimeConversationListVoicesResponseEvent {
|
||||
voices: RealtimeVoicesList::builtin(),
|
||||
},
|
||||
),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn override_turn_context(
|
||||
sess: &Session,
|
||||
sub_id: String,
|
||||
@@ -7189,6 +7207,7 @@ fn realtime_text_for_event(msg: &EventMsg) -> Option<String> {
|
||||
| EventMsg::GetHistoryEntryResponse(_)
|
||||
| EventMsg::McpListToolsResponse(_)
|
||||
| EventMsg::ListSkillsResponse(_)
|
||||
| EventMsg::RealtimeConversationListVoicesResponse(_)
|
||||
| EventMsg::SkillsUpdateAvailable
|
||||
| EventMsg::PlanUpdate(_)
|
||||
| EventMsg::TurnAborted(_)
|
||||
|
||||
@@ -75,6 +75,9 @@ use codex_protocol::protocol::NetworkApprovalProtocol;
|
||||
use codex_protocol::protocol::RateLimitSnapshot;
|
||||
use codex_protocol::protocol::RateLimitWindow;
|
||||
use codex_protocol::protocol::RealtimeAudioFrame;
|
||||
use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::RealtimeVoicesList;
|
||||
use codex_protocol::protocol::ResumedHistory;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::Submission;
|
||||
@@ -4588,6 +4591,51 @@ async fn run_user_shell_command_does_not_set_reference_context_item() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn realtime_conversation_list_voices_emits_builtin_list() {
|
||||
let (session, _turn_context, rx) = make_session_and_context_with_rx().await;
|
||||
|
||||
handlers::realtime_conversation_list_voices(&session, "sub-id".to_string()).await;
|
||||
|
||||
let event = rx.recv().await.expect("event");
|
||||
let voices = match event.msg {
|
||||
EventMsg::RealtimeConversationListVoicesResponse(
|
||||
RealtimeConversationListVoicesResponseEvent { voices },
|
||||
) => voices,
|
||||
msg => panic!("expected list voices response, got {msg:?}"),
|
||||
};
|
||||
assert_eq!(
|
||||
voices,
|
||||
RealtimeVoicesList {
|
||||
v1: vec![
|
||||
RealtimeVoice::Juniper,
|
||||
RealtimeVoice::Maple,
|
||||
RealtimeVoice::Spruce,
|
||||
RealtimeVoice::Ember,
|
||||
RealtimeVoice::Vale,
|
||||
RealtimeVoice::Breeze,
|
||||
RealtimeVoice::Arbor,
|
||||
RealtimeVoice::Sol,
|
||||
RealtimeVoice::Cove,
|
||||
],
|
||||
v2: vec![
|
||||
RealtimeVoice::Alloy,
|
||||
RealtimeVoice::Ash,
|
||||
RealtimeVoice::Ballad,
|
||||
RealtimeVoice::Coral,
|
||||
RealtimeVoice::Echo,
|
||||
RealtimeVoice::Sage,
|
||||
RealtimeVoice::Shimmer,
|
||||
RealtimeVoice::Verse,
|
||||
RealtimeVoice::Marin,
|
||||
RealtimeVoice::Cedar,
|
||||
],
|
||||
default_v1: RealtimeVoice::Cove,
|
||||
default_v2: RealtimeVoice::Marin,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct NeverEndingTask {
|
||||
kind: TaskKind,
|
||||
|
||||
@@ -53,6 +53,7 @@ use codex_protocol::permissions::FileSystemSandboxPolicy;
|
||||
use codex_protocol::permissions::FileSystemSpecialPath;
|
||||
use codex_protocol::permissions::NetworkSandboxPolicy;
|
||||
use codex_protocol::protocol::ReadOnlyAccess;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use serde::Deserialize;
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -6489,6 +6490,7 @@ fn realtime_loads_from_config_toml() -> std::io::Result<()> {
|
||||
version = "v2"
|
||||
type = "transcription"
|
||||
transport = "webrtc"
|
||||
voice = "cedar"
|
||||
"#,
|
||||
)
|
||||
.expect("TOML deserialization should succeed");
|
||||
@@ -6499,6 +6501,7 @@ transport = "webrtc"
|
||||
version: Some(RealtimeWsVersion::V2),
|
||||
session_type: Some(RealtimeWsMode::Transcription),
|
||||
transport: Some(RealtimeTransport::WebRtc),
|
||||
voice: Some(RealtimeVoice::Cedar),
|
||||
})
|
||||
);
|
||||
|
||||
@@ -6515,6 +6518,7 @@ transport = "webrtc"
|
||||
version: RealtimeWsVersion::V2,
|
||||
session_type: RealtimeWsMode::Transcription,
|
||||
transport: RealtimeTransport::WebRtc,
|
||||
voice: Some(RealtimeVoice::Cedar),
|
||||
}
|
||||
);
|
||||
Ok(())
|
||||
|
||||
@@ -1019,6 +1019,18 @@ impl ConfigEditsBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn set_realtime_voice(mut self, voice: Option<&str>) -> Self {
|
||||
let segments = vec!["realtime".to_string(), "voice".to_string()];
|
||||
match voice {
|
||||
Some(voice) => self.edits.push(ConfigEdit::SetPath {
|
||||
segments,
|
||||
value: value(voice),
|
||||
}),
|
||||
None => self.edits.push(ConfigEdit::ClearPath { segments }),
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn clear_legacy_windows_sandbox_keys(mut self) -> Self {
|
||||
for key in [
|
||||
"experimental_windows_sandbox",
|
||||
|
||||
@@ -1066,6 +1066,41 @@ fn blocking_builder_set_realtime_audio_persists_and_clears() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_builder_set_realtime_voice_persists_and_clears() {
|
||||
let tmp = tempdir().expect("tmpdir");
|
||||
let codex_home = tmp.path();
|
||||
|
||||
ConfigEditsBuilder::new(codex_home)
|
||||
.set_realtime_voice(Some("cedar"))
|
||||
.apply_blocking()
|
||||
.expect("persist realtime voice");
|
||||
|
||||
let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config");
|
||||
let config: TomlValue = toml::from_str(&raw).expect("parse config");
|
||||
let realtime = config
|
||||
.get("realtime")
|
||||
.and_then(TomlValue::as_table)
|
||||
.expect("realtime table should exist");
|
||||
assert_eq!(
|
||||
realtime.get("voice").and_then(TomlValue::as_str),
|
||||
Some("cedar")
|
||||
);
|
||||
|
||||
ConfigEditsBuilder::new(codex_home)
|
||||
.set_realtime_voice(/*voice*/ None)
|
||||
.apply_blocking()
|
||||
.expect("clear realtime voice");
|
||||
|
||||
let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config");
|
||||
let config: TomlValue = toml::from_str(&raw).expect("parse config");
|
||||
let realtime = config
|
||||
.get("realtime")
|
||||
.and_then(TomlValue::as_table)
|
||||
.expect("realtime table should exist");
|
||||
assert_eq!(realtime.get("voice"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_mcp_servers_blocking_clears_table_when_empty() {
|
||||
let tmp = tempdir().expect("tmpdir");
|
||||
|
||||
@@ -2098,6 +2098,7 @@ impl Config {
|
||||
version: realtime.version.unwrap_or_default(),
|
||||
session_type: realtime.session_type.unwrap_or_default(),
|
||||
transport: realtime.transport.unwrap_or_default(),
|
||||
voice: realtime.voice,
|
||||
}),
|
||||
experimental_realtime_ws_backend_prompt: cfg.experimental_realtime_ws_backend_prompt,
|
||||
experimental_realtime_ws_startup_context: cfg.experimental_realtime_ws_startup_context,
|
||||
|
||||
@@ -39,6 +39,8 @@ use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
|
||||
use codex_protocol::protocol::RealtimeConversationSdpEvent;
|
||||
use codex_protocol::protocol::RealtimeConversationStartedEvent;
|
||||
use codex_protocol::protocol::RealtimeHandoffRequested;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::RealtimeVoicesList;
|
||||
use http::HeaderMap;
|
||||
use http::HeaderValue;
|
||||
use http::header::AUTHORIZATION;
|
||||
@@ -521,7 +523,7 @@ async fn prepare_realtime_start(
|
||||
}
|
||||
let version = config.realtime.version;
|
||||
let session_config =
|
||||
build_realtime_session_config(sess, params.prompt, params.session_id).await?;
|
||||
build_realtime_session_config(sess, params.prompt, params.session_id, params.voice).await?;
|
||||
let requested_session_id = session_config.session_id.clone();
|
||||
let extra_headers = match transport {
|
||||
ConversationStartTransport::Websocket => {
|
||||
@@ -549,6 +551,7 @@ pub(crate) async fn build_realtime_session_config(
|
||||
sess: &Arc<Session>,
|
||||
prompt: Option<Option<String>>,
|
||||
session_id: Option<String>,
|
||||
voice: Option<RealtimeVoice>,
|
||||
) -> CodexResult<RealtimeSessionConfig> {
|
||||
let config = sess.get_config().await;
|
||||
let prompt = prepare_realtime_backend_prompt(
|
||||
@@ -578,15 +581,53 @@ pub(crate) async fn build_realtime_session_config(
|
||||
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
|
||||
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
|
||||
};
|
||||
let voice = voice
|
||||
.or(config.realtime.voice)
|
||||
.unwrap_or_else(|| default_realtime_voice(config.realtime.version));
|
||||
validate_realtime_voice(config.realtime.version, voice)?;
|
||||
Ok(RealtimeSessionConfig {
|
||||
instructions: prompt,
|
||||
model,
|
||||
session_id: Some(session_id.unwrap_or_else(|| sess.conversation_id.to_string())),
|
||||
event_parser,
|
||||
session_mode,
|
||||
voice,
|
||||
})
|
||||
}
|
||||
|
||||
fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice {
|
||||
let voices = RealtimeVoicesList::builtin();
|
||||
match version {
|
||||
RealtimeWsVersion::V1 => voices.default_v1,
|
||||
RealtimeWsVersion::V2 => voices.default_v2,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> {
|
||||
let voices = RealtimeVoicesList::builtin();
|
||||
let allowed = match version {
|
||||
RealtimeWsVersion::V1 => &voices.v1,
|
||||
RealtimeWsVersion::V2 => &voices.v2,
|
||||
};
|
||||
if allowed.contains(&voice) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let version = match version {
|
||||
RealtimeWsVersion::V1 => "v1",
|
||||
RealtimeWsVersion::V2 => "v2",
|
||||
};
|
||||
let allowed = allowed
|
||||
.iter()
|
||||
.map(|voice| voice.wire_name())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
Err(CodexErr::InvalidRequest(format!(
|
||||
"realtime voice `{}` is not supported for {version}; supported voices: {allowed}",
|
||||
voice.wire_name()
|
||||
)))
|
||||
}
|
||||
|
||||
async fn handle_start_inner(
|
||||
sess: &Arc<Session>,
|
||||
sub_id: &str,
|
||||
|
||||
Reference in new issue
Block a user