github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-21 05:22:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow microphone channel selection for voice conversations (#49836)

Browse Source 
## Why

Mixing every input on a multichannel audio device can include playback channels in microphone capture. Selecting only microphone inputs keeps those playback channels out of the captured audio.

## What changed

- Add `audio.microphone_channel` with one-based channel numbers, accepting a single value or an array such as `[1, 2]`. Leaving it unset preserves mixing of all inputs.
- Add an input-channel picker for the selected microphone when it has at least three channels. Require a nonempty selection and save it locally for the next voice conversation.
- Average only the selected channels, deduplicate selections, and reject empty or unavailable selections without falling back to other inputs.
- Clear saved channel selections when saving an input device and report configuration overrides that prevent the reset from taking effect.

## Testing

Add regression coverage for excluding playback channels, sample conversion, subset mixing, invalid selections, configuration loading, local persistence and overrides, and picker selection and cancellation. Update UI snapshots and verify channel selections are passed to the voice helper.

GitOrigin-RevId: 99960dcfa7b10cf38270768a90bb84a4be95a406
This commit is contained in:
Benjamin Carlsson
2026-10-01 02:59:26 +00:00
committed by copyberry
parent 08e2b58b07
commit e53e932dc8
24 changed files with 661 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/config/src/config_toml.rs
+21
View File
@@ -614,8 +614,27 @@ impl ProjectConfig {
}
}
/// Selected microphone inputs. Scalars preserve existing single-channel configuration.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
#[serde(untagged)]
pub enum MicrophoneChannels {
Single(std::num::NonZeroU16),
Multiple(Vec<std::num::NonZeroU16>),
}
impl MicrophoneChannels {
pub fn as_slice(&self) -> &[std::num::NonZeroU16] {
match self {
Self::Single(channel) => std::slice::from_ref(channel),
Self::Multiple(channels) => channels,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RealtimeAudioConfig {
/// One-based microphone channels to mix; unset mixes all input channels.
pub microphone_channel: Option<MicrophoneChannels>,
pub microphone: Option<String>,
pub speaker: Option<String>,
}
@@ -663,6 +682,8 @@ pub struct RealtimeToml {
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct RealtimeAudioToml {
/// One-based microphone channels to mix; unset mixes all input channels.
pub microphone_channel: Option<MicrophoneChannels>,
pub microphone: Option<String>,
pub speaker: Option<String>,
}
codex-rs/core/config.schema.json
+26
View File
@@ -2586,6 +2586,24 @@
],
"type": "string"
},
"MicrophoneChannels": {
"anyOf": [
{
"format": "uint16",
"minimum": 1.0,
"type": "integer"
},
{
"items": {
"format": "uint16",
"minimum": 1.0,
"type": "integer"
},
"type": "array"
}
],
"description": "Selected microphone inputs. Scalars preserve existing single-channel configuration."
},
"ModelAvailabilityNuxConfig": {
"additionalProperties": {
"format": "uint32",
@@ -3792,6 +3810,14 @@
"microphone": {
"type": "string"
},
"microphone_channel": {
"allOf": [
{
"$ref": "#/definitions/MicrophoneChannels"
}
],
"description": "One-based microphone channels to mix; unset mixes all input channels."
},
"speaker": {
"type": "string"
}
codex-rs/core/src/config/config_tests.rs
+27 -29
View File
@@ -13215,35 +13215,33 @@ voice = "cedar"
#[tokio::test]
async fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> {
let cfg: ConfigToml = toml::from_str(
r#"
[audio]
microphone = "USB Mic"
speaker = "Desk Speakers"
"#,
)
.expect("TOML deserialization should succeed");
let realtime_audio = cfg
.audio
.as_ref()
.expect("realtime audio config should be present");
assert_eq!(realtime_audio.microphone.as_deref(), Some("USB Mic"));
assert_eq!(realtime_audio.speaker.as_deref(), Some("Desk Speakers"));
let codex_home = TempDir::new()?;
let config = Config::load_from_base_config_with_overrides(
cfg,
ConfigOverrides::default(),
codex_home.abs(),
)
.await?;
assert_eq!(config.realtime_audio.microphone.as_deref(), Some("USB Mic"));
assert_eq!(
config.realtime_audio.speaker.as_deref(),
Some("Desk Speakers")
);
for selection in ["1", "[1, 2]"] {
let cfg: ConfigToml = toml::from_str(&format!(
"[audio]\nmicrophone = \"USB Mic\"\nmicrophone_channel = {selection}\nspeaker = \"Desk Speakers\"\n"
)).expect("TOML deserialization should succeed");
let expected_audio = cfg.audio.as_ref().unwrap().clone();
let codex_home = TempDir::new()?;
let config = Config::load_from_base_config_with_overrides(
cfg,
ConfigOverrides::default(),
codex_home.abs(),
)
.await?;
assert_eq!(
config.realtime_audio,
codex_config::config_toml::RealtimeAudioConfig {
microphone: Some("USB Mic".into()),
speaker: Some("Desk Speakers".into()),
microphone_channel: expected_audio.microphone_channel,
}
);
}
for invalid in ["0", "[1, 0]"] {
assert!(
toml::from_str::<ConfigToml>(&format!("[audio]\nmicrophone_channel = {invalid}"))
.is_err()
);
}
Ok(())
}
codex-rs/core/src/config/mod.rs
+1
View File
@@ -4434,6 +4434,7 @@ impl Config {
.audio
.map_or_else(RealtimeAudioConfig::default, |audio| RealtimeAudioConfig {
microphone: audio.microphone,
microphone_channel: audio.microphone_channel,
speaker: audio.speaker,
}),
experimental_realtime_ws_base_url: cfg.experimental_realtime_ws_base_url,
codex-rs/realtime-webrtc/src/protocol.rs
+1
View File
@@ -37,6 +37,7 @@ pub enum AudioDeviceKind {
pub struct AudioDeviceSelection {
pub microphone: Option<String>,
pub speaker: Option<String>,
pub channel: Option<Vec<std::num::NonZeroU16>>,
}
/// Local device metadata; enumeration never opens streams.
codex-rs/realtime-webrtc/tests/session_actor.rs
+5 -1
View File
@@ -31,6 +31,10 @@ fn startup_controls_meters_and_helper_loss() -> Result<()> {
AudioDeviceSelection {
microphone: Some("Zen Go Synergy Core Playback".into()),
speaker: Some("Headphones".into()),
channel: Some(vec![
std::num::NonZeroU16::new(/*n*/ 1).unwrap(),
std::num::NonZeroU16::new(/*n*/ 2).unwrap(),
]),
},
)?;
assert_eq!(started.offer_sdp, "synthetic-offer");
@@ -44,7 +48,7 @@ fn startup_controls_meters_and_helper_loss() -> Result<()> {
answer.join().expect("answer thread")?;
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&fs::read(root.join("input-selection"))?)?,
serde_json::json!({"microphone": "Zen Go Synergy Core Playback", "speaker": "Headphones"})
serde_json::json!({"microphone": "Zen Go Synergy Core Playback", "speaker": "Headphones", "channel": [1, 2]})
);
let initial: Vec<AudioControls> = serde_json::from_slice(&fs::read(root.join("controls"))?)?;
assert_eq!(
codex-rs/tui/src/app/event_dispatch.rs
+4
View File
@@ -2490,6 +2490,7 @@ impl App {
AppEvent::OpenRealtimeSoundDevices => self.chat_widget.open_realtime_sound_devices(),
AppEvent::OpenRealtimeVoices => self.open_realtime_voices(app_server).await,
AppEvent::OpenRealtimeDevicePicker { kind } => self.list_realtime_devices(kind),
AppEvent::OpenRealtimeInputChannels { device } => self.chat_widget.open_realtime_input_channels(device),
AppEvent::RealtimeDevicesListed { origin, kind, result } => {
if origin == self.active_thread_id {
match result {
@@ -2501,6 +2502,9 @@ impl App {
AppEvent::PersistRealtimeDevice { kind, name } => {
self.persist_realtime_device(kind, name).await;
}
AppEvent::PersistRealtimeInputChannel { channel } => {
self.persist_realtime_input_channel(channel).await;
}
AppEvent::PersistRealtimeVoiceSelection { voice } => {
self.persist_realtime_voice(app_server, voice).await;
}
codex-rs/tui/src/app/realtime_settings.rs
+46 -3
View File
@@ -6,6 +6,7 @@ use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ThreadRealtimeListVoicesParams;
use codex_app_server_protocol::ThreadRealtimeListVoicesResponse;
use codex_config::config_toml::MicrophoneChannels;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_realtime_webrtc::AudioDeviceKind;
@@ -31,6 +32,25 @@ fn configured_voice(
}
impl App {
pub(super) async fn persist_realtime_input_channel(
&mut self,
channel: Option<MicrophoneChannels>,
) {
self.persist_realtime_audio(
"microphone_channel",
channel.map(|channel| match channel {
MicrophoneChannels::Single(channel) => i64::from(channel.get()).into(),
MicrophoneChannels::Multiple(channels) => toml::Value::Array(
channels
.into_iter()
.map(|channel| i64::from(channel.get()).into())
.collect(),
),
}),
)
.await;
}
pub(super) async fn persist_realtime_device(
&mut self,
kind: AudioDeviceKind,
@@ -47,14 +67,34 @@ impl App {
async fn persist_realtime_audio(&mut self, key: &str, value: Option<toml::Value>) {
// Audio runs on the TUI's machine, even when the app server is remote.
let segments = vec!["audio".to_string(), key.to_string()];
let edits = vec![match &value {
let mut edits = vec![match &value {
Some(toml::Value::String(value)) => ConfigEdit::SetPath {
segments,
value: value.clone().into(),
},
Some(_) => unreachable!("audio device settings are names"),
Some(toml::Value::Integer(value)) => ConfigEdit::SetPath {
segments,
value: (*value).into(),
},
Some(toml::Value::Array(channels)) => ConfigEdit::SetPath {
segments,
value: channels
.iter()
.map(|channel| match channel {
toml::Value::Integer(channel) => *channel,
_ => unreachable!("audio channel arrays contain only integers"),
})
.collect::<toml_edit::Array>()
.into(),
},
Some(_) => unreachable!("audio settings are device names or channel numbers"),
None => ConfigEdit::ClearPath { segments },
}];
if key == "microphone" {
edits.push(ConfigEdit::ClearPath {
segments: vec!["audio".into(), "microphone_channel".into()],
});
}
if let Err(error) =
ConfigEditsBuilder::for_config_path(self.local_settings.user_config_path.as_path())
.with_edits(edits)
@@ -86,9 +126,12 @@ impl App {
let effective = toml::Value::try_from(&audio)
.ok()
.and_then(|audio| audio.get(key).cloned());
let channel_overridden = key == "microphone" && audio.microphone_channel.is_some();
self.local_settings.audio = Ok(audio);
self.chat_widget.local_settings.audio = self.local_settings.audio.clone();
if effective == value {
if channel_overridden {
self.chat_widget.add_error_message("Input device saved, but the input channel is overridden by another configuration layer. Update that override before starting voice.".into());
} else if effective == value {
self.chat_widget.add_info_message(
"Audio setting saved. Applies to your next voice conversation.".to_string(),
/*hint*/ None,
codex-rs/tui/src/app/tests/realtime_requests.rs
+95 -11
View File
@@ -2081,9 +2081,74 @@ async fn overridden_voice_save_keeps_effective_voice() -> Result<()> {
}
#[tokio::test]
async fn audio_devices_are_persisted() -> Result<()> {
use codex_realtime_webrtc::AudioDeviceKind;
async fn microphone_channel_is_persisted_locally_and_reloaded() -> Result<()> {
let mut app = make_test_app().await;
use codex_config::config_toml::MicrophoneChannels;
for (saved, overridden, expected) in [
(Some("1"), None, Some("1")),
(Some("[1, 2]"), None, Some("[1, 2]")),
(Some("[1, 2]"), Some("3"), Some("3")),
(Some("1"), Some("[2, 3]"), Some("[2, 3]")),
(None, None, None),
] {
let parse = |value: &str| -> MicrophoneChannels {
toml::from_str::<codex_config::config_toml::RealtimeAudioToml>(&format!(
"microphone_channel = {value}"
))
.unwrap()
.microphone_channel
.unwrap()
};
app.cli_kv_overrides = overridden
.into_iter()
.map(|value| {
(
"audio.microphone_channel".to_string(),
toml::Value::try_from(parse(value)).unwrap(),
)
})
.collect();
let mut expected_settings = app.local_settings.clone();
app.persist_realtime_input_channel(saved.map(parse)).await;
expected_settings.audio = app.local_settings.audio.clone();
assert_eq!(app.local_settings, expected_settings);
let actual = [&app.local_settings, &app.chat_widget.local_settings]
.map(|settings| settings.audio.as_ref().unwrap().microphone_channel.clone());
assert_eq!(
actual,
std::array::from_fn::<_, 2, _>(|_| expected.map(parse))
);
}
Ok(())
}
#[tokio::test]
async fn audio_devices_are_persisted_and_input_changes_clear_channels() -> Result<()> {
use codex_realtime_webrtc::AudioDeviceKind;
let (mut app, mut events, _ops) = make_test_app_with_channels().await;
app.cli_kv_overrides = vec![("audio.microphone_channel".into(), 3.into())];
app.persist_realtime_device(AudioDeviceKind::Input, Some("Mono mic".into()))
.await;
let messages = std::iter::from_fn(|| events.try_recv().ok())
.filter_map(|event| match event {
AppEvent::InsertHistoryCell(cell) => Some(
cell.display_lines(/*width*/ 120)
.iter()
.map(ToString::to_string)
.collect::<String>(),
),
_ => None,
})
.collect::<String>();
assert!(messages.contains("input channel is overridden"));
app.cli_kv_overrides.clear();
app.persist_realtime_input_channel(Some(
codex_config::config_toml::MicrophoneChannels::Multiple(vec![
std::num::NonZeroU16::new(/*n*/ 1).unwrap(),
std::num::NonZeroU16::new(/*n*/ 3).unwrap(),
]),
))
.await;
app.persist_realtime_device(AudioDeviceKind::Output, Some("Headphones".into()))
.await;
app.persist_realtime_device(AudioDeviceKind::Input, Some("Interface".into()))
@@ -2091,11 +2156,29 @@ async fn audio_devices_are_persisted() -> Result<()> {
let expected = codex_config::config_toml::RealtimeAudioToml {
microphone: Some("Interface".into()),
speaker: Some("Headphones".into()),
microphone_channel: None,
};
assert_eq!(app.local_settings.audio.as_ref().unwrap(), &expected);
assert_eq!(LocalSettings::from(&app.config).audio, Ok(expected));
app.persist_realtime_device(AudioDeviceKind::Input, /*name*/ None)
.await;
app.persist_realtime_input_channel(Some(
codex_config::config_toml::MicrophoneChannels::Multiple(vec![
std::num::NonZeroU16::new(/*n*/ 1).unwrap(),
std::num::NonZeroU16::new(/*n*/ 3).unwrap(),
]),
))
.await;
app.persist_realtime_device(AudioDeviceKind::Input, /*name*/ None)
.await;
assert_eq!(
app.local_settings
.audio
.as_ref()
.unwrap()
.microphone_channel,
None
);
app.persist_realtime_device(AudioDeviceKind::Output, /*name*/ None)
.await;
assert_eq!(app.local_settings.audio, Ok(Default::default()));
@@ -2107,14 +2190,14 @@ async fn audio_devices_are_persisted() -> Result<()> {
}
#[tokio::test]
async fn audio_device_is_machine_local_across_project_and_remote_transitions() -> Result<()> {
async fn microphone_channel_is_machine_local_across_project_and_remote_transitions() -> Result<()> {
let mut app = make_test_app().await;
let project = tempfile::tempdir()?;
let remote = tempfile::tempdir()?;
std::fs::create_dir_all(project.path().join(".codex"))?;
std::fs::write(
project.path().join(".codex/config.toml"),
"default_permissions = \"work\"\n[audio]\nmicrophone = \"Project mic\"\n[permissions.work]\nextends = \":workspace\"\n",
"default_permissions = \"work\"\n[audio]\nmicrophone_channel = 2\n[permissions.work]\nextends = \":workspace\"\n",
)?;
crate::legacy_core::config::set_project_trust_level(
&app.config.codex_home,
@@ -2132,9 +2215,9 @@ async fn audio_device_is_machine_local_across_project_and_remote_transitions() -
ThreadId::new(),
remote.path().to_path_buf(),
));
app.persist_realtime_device(
codex_realtime_webrtc::AudioDeviceKind::Input,
Some("Machine mic".into()),
app.persist_realtime_input_channel(
std::num::NonZeroU16::new(/*n*/ 1)
.map(codex_config::config_toml::MicrophoneChannels::Single),
)
.await;
app.cli_kv_overrides.clear();
@@ -2147,15 +2230,16 @@ async fn audio_device_is_machine_local_across_project_and_remote_transitions() -
let reloaded = app.local_settings.reloaded(&config);
assert_eq!(reloaded.audio, app.chat_widget.local_settings.audio);
assert_eq!(
reloaded.audio.as_ref().unwrap().microphone.as_deref(),
Some("Machine mic")
reloaded.audio.as_ref().unwrap().microphone_channel,
std::num::NonZeroU16::new(/*n*/ 1)
.map(codex_config::config_toml::MicrophoneChannels::Single)
);
}
// A project value must not hide invalid machine audio.
// A project value must not hide invalid machine audio and silently enable mixed capture.
std::fs::write(
app.local_settings.user_config_path.as_path(),
std::fs::read_to_string(app.local_settings.user_config_path.as_path())?
.replace("microphone = \"Machine mic\"", "microphone = 0"),
.replace("microphone_channel = 1", "microphone_channel = 0"),
)?;
let config = app
.rebuild_config_for_cwd(project.path().to_path_buf())
codex-rs/tui/src/app_event.rs
+6
View File
@@ -1238,6 +1238,9 @@ pub(crate) enum AppEvent {
OpenRealtimeDevicePicker {
kind: codex_realtime_webrtc::AudioDeviceKind,
},
OpenRealtimeInputChannels {
device: codex_realtime_webrtc::AudioDevice,
},
RealtimeDevicesListed {
origin: Option<ThreadId>,
kind: codex_realtime_webrtc::AudioDeviceKind,
@@ -1247,6 +1250,9 @@ pub(crate) enum AppEvent {
kind: codex_realtime_webrtc::AudioDeviceKind,
name: Option<String>,
},
PersistRealtimeInputChannel {
channel: Option<codex_config::config_toml::MicrophoneChannels>,
},
/// Save the voice for subsequent conversations through the app server.
PersistRealtimeVoiceSelection {
codex-rs/tui/src/bottom_pane/list_selection_view.rs
+10 -3
View File
@@ -1,6 +1,7 @@
//! Render selectable lists with shared picker controls and full-width focus rows.
//! Selection and scrolling use filtered row indices; actions map back to source items.
//! Controls and overflow indicators keep their own rows outside the result viewport.
//! Letter-labeled children are indented and do not consume numeric shortcuts.
use crossterm::event::KeyCode;
use crossterm::event::KeyEvent;
@@ -157,6 +158,8 @@ pub(crate) type OnCancelCallback = Option<Box<dyn Fn(&AppEventSender) + Send + S
#[derive(Default)]
pub(crate) struct SelectionItem {
pub name: String,
/// Indent beneath the preceding parent, using this letter instead of a numeric shortcut.
pub child_label: Option<char>,
pub name_prefix_spans: Vec<Span<'static>>,
pub toggle: Option<SelectionToggle>,
pub toggle_placeholder: Option<&'static str>,
@@ -643,7 +646,7 @@ impl ListSelectionView {
.filter(|actual_idx| {
self.active_items()
.get(**actual_idx)
.is_some_and(Self::item_is_enabled)
.is_some_and(|item| Self::item_is_enabled(item) && item.child_label.is_none())
})
.count()
.max(1)
@@ -681,6 +684,8 @@ impl ListSelectionView {
} else {
format!("{prefix} {}", " ".repeat(enabled_row_number_width + 2))
}
} else if let Some(label) = item.child_label {
format!("{prefix} {label}. ")
} else {
enabled_row_number += 1;
let n = enabled_row_number;
@@ -776,7 +781,7 @@ impl ListSelectionView {
self.active_items()
.iter()
.enumerate()
.filter(|(_, item)| Self::item_is_enabled(item))
.filter(|(_, item)| Self::item_is_enabled(item) && item.child_label.is_none())
.nth(number - 1)
.map(|(idx, _)| idx)
}
@@ -1184,8 +1189,10 @@ impl BottomPaneView for ListSelectionView {
&& !modifiers.contains(KeyModifiers::ALT) =>
{
if let Some(idx) = self.items.iter().position(|item| {
item.display_shortcut
(item
.display_shortcut
.is_some_and(|shortcut| shortcut.is_press(key_event))
|| item.child_label == Some(c))
&& Self::item_is_enabled(item)
}) {
self.select_shortcut(idx);
codex-rs/tui/src/bottom_pane/mod.rs
+2
View File
@@ -77,6 +77,8 @@ mod empty_state_policy;
mod hook_status;
mod mcp_server_elicitation;
mod multi_select_picker;
pub(crate) use multi_select_picker::MultiSelectItem;
pub(crate) use multi_select_picker::MultiSelectPicker;
#[cfg(test)]
#[path = "questions_tests.rs"]
mod question_tests;
codex-rs/tui/src/bottom_pane/multi_select_picker.rs
+13 -1
View File
@@ -5,6 +5,7 @@
//!
//! - **Fuzzy search**: Type to filter items by name
//! - **Toggle selection**: Space to toggle items on/off
//! - **Required selection**: Optional nonempty confirmation; cancellation always remains available
//! - **Reordering**: Optional left/right arrow support to reorder items
//! - **Live preview**: Optional callback to show a preview of current selections
//! - **Callbacks**: Hooks for change, confirm, and cancel events
@@ -191,6 +192,7 @@ pub(crate) struct MultiSelectPicker {
/// Whether left/right arrow reordering is enabled.
ordering_enabled: bool,
require_selection: bool,
/// Shared list keybindings for navigation and completion.
keymap: ListKeymap,
@@ -417,7 +419,8 @@ impl MultiSelectPicker {
/// Collects the IDs of all enabled items and passes them to the
/// `on_confirm` callback. Does nothing if already complete.
fn confirm_selection(&mut self) {
if self.complete {
if self.complete || (self.require_selection && !self.items.iter().any(|item| item.enabled))
{
return;
}
self.complete = true;
@@ -707,6 +710,7 @@ pub(crate) struct MultiSelectPickerBuilder {
instructions: Vec<Span<'static>>,
items: Vec<MultiSelectItem>,
ordering_enabled: bool,
require_selection: bool,
app_event_tx: AppEventSender,
keymap: ListKeymap,
preview_builder: Option<PreviewCallback>,
@@ -724,6 +728,7 @@ impl MultiSelectPickerBuilder {
instructions: Vec::new(),
items: Vec::new(),
ordering_enabled: false,
require_selection: false,
app_event_tx,
keymap: RuntimeKeymap::defaults().list,
preview_builder: None,
@@ -777,6 +782,12 @@ impl MultiSelectPickerBuilder {
self
}
/// Requires a nonempty selection to confirm without restricting cancellation.
pub fn require_selection(mut self) -> Self {
self.require_selection = true;
self
}
/// Sets a callback invoked when the user confirms their selection (Enter).
///
/// The callback receives a list of IDs for all enabled items.
@@ -844,6 +855,7 @@ impl MultiSelectPickerBuilder {
header: Box::new(header),
footer_hint: Line::from(instructions),
ordering_enabled: self.ordering_enabled,
require_selection: self.require_selection,
keymap: self.keymap,
search_query: String::new(),
filtered_indices: Vec::new(),
codex-rs/tui/src/chatwidget/realtime.rs
+3
View File
@@ -328,6 +328,9 @@ impl ChatWidget {
let selection = codex_realtime_webrtc::AudioDeviceSelection {
microphone: audio.microphone,
speaker: audio.speaker,
channel: audio
.microphone_channel
.map(|channels| channels.as_slice().to_vec()),
};
std::thread::spawn(move || {
let result = RealtimeWebrtcSession::start(abort_registration, selection)
codex-rs/tui/src/chatwidget/realtime_settings.rs
+101
View File
@@ -1,6 +1,9 @@
//! Voice settings hierarchy with parent navigation; selections apply to subsequent conversations.
use super::*;
use crate::bottom_pane::MultiSelectItem;
use crate::bottom_pane::MultiSelectPicker;
use codex_config::config_toml::MicrophoneChannels;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use codex_realtime_webrtc::AudioDevice;
@@ -122,6 +125,14 @@ impl ChatWidget {
AudioDeviceKind::Input => ("Input device", audio.microphone),
AudioDeviceKind::Output => ("Output device", audio.speaker),
};
let selected = devices
.iter()
.find(|device| {
current
.as_ref()
.map_or(device.is_default, |name| *name == device.name)
})
.cloned();
let ambiguous: std::collections::HashSet<_> = devices
.iter()
.filter(|device| {
@@ -157,6 +168,39 @@ impl ChatWidget {
}
})
.collect();
if kind == AudioDeviceKind::Input
&& let Some(device) = selected.filter(|device| device.channels >= 3)
{
let parent = items.iter().position(|item| item.is_current).unwrap_or(0);
items.insert(
parent + 1,
SelectionItem {
name: "Input channels".into(),
child_label: Some('a'),
description: Some(audio.microphone_channel.as_ref().map_or_else(
|| "All channels (mixed)".into(),
|channels| {
format!(
"Inputs {}",
channels
.as_slice()
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", ")
)
},
)),
actions: vec![Box::new(move |tx| {
tx.send(AppEvent::OpenRealtimeInputChannels {
device: device.clone(),
})
})],
dismiss_on_select: true,
..Default::default()
},
);
}
items.push(back_item(|| AppEvent::OpenRealtimeSoundDevices));
self.bottom_pane.show_selection_view(SelectionViewParams {
title: Some(title.into()),
@@ -168,6 +212,63 @@ impl ChatWidget {
});
}
pub(crate) fn open_realtime_input_channels(&mut self, microphone: AudioDevice) {
let Some(audio) = self.realtime_audio_settings() else {
return;
};
let current = audio.microphone_channel;
let count = microphone.channels;
let picker = MultiSelectPicker::builder(
format!("Microphone: {}", microphone.name),
Some("Select the input channels to mix. Select at least one.".into()),
self.app_event_tx.clone(),
)
.list_keymap(self.bottom_pane.list_keymap())
.items(
(1..=count)
.map(|number| MultiSelectItem {
id: number.to_string(),
name: format!("Input {number}"),
enabled: current.as_ref().is_none_or(|channels| {
channels
.as_slice()
.iter()
.any(|channel| channel.get() == number)
}),
..Default::default()
})
.collect(),
)
.require_selection()
.on_preview(|items| {
items
.iter()
.all(|item| !item.enabled)
.then(|| "Select at least one input channel.".red().into())
})
.on_confirm(move |ids, tx| {
let selected: Vec<_> = (1..=count)
.filter(|number| ids.contains(&number.to_string()))
.filter_map(std::num::NonZeroU16::new)
.collect();
let channel = if selected.len() == usize::from(count) {
None
} else if selected.len() == 1 {
selected.first().copied().map(MicrophoneChannels::Single)
} else {
Some(MicrophoneChannels::Multiple(selected))
};
tx.send(AppEvent::PersistRealtimeInputChannel { channel });
})
.on_cancel(|tx| {
tx.send(AppEvent::OpenRealtimeDevicePicker {
kind: AudioDeviceKind::Input,
})
})
.build();
self.bottom_pane.show_view(Box::new(picker));
}
pub(crate) fn set_realtime_voice(&mut self, voice: Option<RealtimeVoice>) {
self.config.realtime.voice = voice;
}
codex-rs/tui/src/chatwidget/realtime_settings_tests.rs
+124 -8
View File
@@ -81,10 +81,16 @@ async fn voice_settings_uses_server_catalog() {
}
#[tokio::test]
async fn sound_devices_picker_routes_selection_without_starting_audio() {
async fn microphone_channel_picker_routes_selection_without_starting_audio() {
let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await;
chat.local_settings.audio.as_mut().unwrap().microphone =
Some("Zen Go Synergy Core Playback".to_string());
chat.local_settings
.audio
.as_mut()
.unwrap()
.microphone_channel = std::num::NonZeroU16::new(/*n*/ 1)
.map(codex_config::config_toml::MicrophoneChannels::Single);
chat.open_realtime_settings();
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
@@ -109,6 +115,81 @@ async fn sound_devices_picker_routes_selection_without_starting_audio() {
}
))
);
chat.open_realtime_input_channels(AudioDevice {
name: "Zen Go Synergy Core Playback".into(),
channels: 4,
is_default: true,
});
insta::assert_snapshot!(
"voice_input_channels",
render_bottom_popup(&chat, /*width*/ 90)
);
// Space toggles without persisting; Enter saves both selected inputs.
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE));
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Char(' '), KeyModifiers::NONE));
assert!(events.try_recv().is_err());
insta::assert_snapshot!(
"voice_input_channel_subset",
render_bottom_popup(&chat, /*width*/ 90)
);
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
let selected = std::iter::from_fn(|| events.try_recv().ok()).find_map(|event| match event {
AppEvent::PersistRealtimeInputChannel { channel } => Some(channel),
_ => None,
});
assert_eq!(
selected,
Some(Some(
codex_config::config_toml::MicrophoneChannels::Multiple(vec![
std::num::NonZeroU16::new(/*n*/ 1).unwrap(),
std::num::NonZeroU16::new(/*n*/ 2).unwrap()
])
))
);
// Empty confirmation stays in the picker; Escape discards the edit.
chat.open_realtime_input_channels(AudioDevice {
name: "Mic".into(),
channels: 4,
is_default: true,
});
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Char(' '), KeyModifiers::NONE));
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
assert!(chat.has_active_view());
assert!(events.try_recv().is_err());
insta::assert_snapshot!(
"voice_input_channels_empty",
render_bottom_popup(&chat, /*width*/ 90)
);
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE));
assert!(matches!(
events.try_recv().unwrap(),
AppEvent::OpenRealtimeDevicePicker {
kind: AudioDeviceKind::Input
}
));
chat.local_settings
.audio
.as_mut()
.unwrap()
.microphone_channel = None;
chat.open_realtime_input_channels(AudioDevice {
name: "Mic".into(),
channels: 4,
is_default: true,
});
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
assert!(matches!(
events.try_recv().unwrap(),
AppEvent::PersistRealtimeInputChannel { channel: None }
));
assert!(ops.try_recv().is_err());
assert!(!chat.realtime_conversation_is_running());
}
@@ -117,12 +198,28 @@ async fn sound_devices_picker_routes_selection_without_starting_audio() {
async fn submenus_return_to_their_parent_with_back_or_escape() {
let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await;
type OpenMenu = fn(&mut ChatWidget);
let menus: [OpenMenu; 2] = [
|chat| chat.open_realtime_voices(/*current*/ None, RealtimeVoicesList::builtin()),
ChatWidget::open_realtime_sound_devices,
let menus: [(OpenMenu, bool); 3] = [
(
|chat| chat.open_realtime_voices(/*current*/ None, RealtimeVoicesList::builtin()),
false,
),
(ChatWidget::open_realtime_sound_devices, false),
(
|chat| {
chat.open_realtime_input_channels(AudioDevice {
name: "Mic".into(),
channels: 4,
is_default: true,
})
},
true,
),
];
for open in menus {
for (open, sound_parent) in menus {
for key in [KeyCode::Esc, KeyCode::Enter] {
if sound_parent && key == KeyCode::Enter {
continue;
}
while events.try_recv().is_ok() {}
open(&mut chat);
if key == KeyCode::Enter {
@@ -132,8 +229,15 @@ async fn submenus_return_to_their_parent_with_back_or_escape() {
chat.bottom_pane
.handle_key_event(KeyEvent::new(key, KeyModifiers::NONE));
assert!(
std::iter::from_fn(|| events.try_recv().ok())
.any(|event| matches!(event, AppEvent::OpenRealtimeSettings))
std::iter::from_fn(|| events.try_recv().ok()).any(|event| matches!(
(sound_parent, event),
(
true,
AppEvent::OpenRealtimeDevicePicker {
kind: AudioDeviceKind::Input
}
) | (false, AppEvent::OpenRealtimeSettings)
))
);
}
}
@@ -142,7 +246,7 @@ async fn submenus_return_to_their_parent_with_back_or_escape() {
}
#[tokio::test]
async fn device_pickers_route_selection() {
async fn device_pickers_route_selection_and_only_offer_channels_for_multichannel_inputs() {
let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await;
for (kind, channels, microphone, snapshot) in [
(
@@ -189,6 +293,18 @@ async fn device_pickers_route_selection() {
_ => None,
});
assert_eq!(selected, Some((kind, Some(device.name.clone()))));
chat.open_realtime_device_picker(kind, devices.clone());
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Char('a'), KeyModifiers::NONE));
let channel_device =
std::iter::from_fn(|| events.try_recv().ok()).find_map(|event| match event {
AppEvent::OpenRealtimeInputChannels { device } => Some(device),
_ => None,
});
assert_eq!(
channel_device,
(kind == AudioDeviceKind::Input && channels >= 3).then_some(device)
);
chat.open_realtime_device_picker(kind, devices);
chat.bottom_pane
.handle_key_event(KeyEvent::new(KeyCode::Char('4'), KeyModifiers::NONE));
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_input_channel_subset.snap
+15
View File
@@ -0,0 +1,15 @@
---
source: tui/src/chatwidget/realtime_settings_tests.rs
expression: "render_bottom_popup(&chat, 90)"
---
Microphone: Zen Go Synergy Core Playback
Select the input channels to mix. Select at least one.
Type to search
[x] Input 1
› [x] Input 2
[ ] Input 3
[ ] Input 4
space toggle · enter save · esc cancel
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_input_channels.snap
+15
View File
@@ -0,0 +1,15 @@
---
source: tui/src/chatwidget/realtime_settings_tests.rs
expression: "render_bottom_popup(&chat, 90)"
---
Microphone: Zen Go Synergy Core Playback
Select the input channels to mix. Select at least one.
Type to search
› [x] Input 1
[ ] Input 2
[ ] Input 3
[ ] Input 4
space toggle · enter save · esc cancel
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_input_channels_empty.snap
+16
View File
@@ -0,0 +1,16 @@
---
source: tui/src/chatwidget/realtime_settings_tests.rs
expression: "render_bottom_popup(&chat, 90)"
---
Microphone: Mic
Select the input channels to mix. Select at least one.
Type to search
› [ ] Input 1
[ ] Input 2
[ ] Input 3
[ ] Input 4
Select at least one input channel.
space toggle · enter save · esc cancel
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_input_devices.snap
+1
View File
@@ -7,6 +7,7 @@ expression: "render_bottom_popup(&chat, 80)"
› 1. System default (current)
a. Input channels All channels (mixed)
2. Audio interface
3. Other device
4. Back
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_selected_input_devices.snap
+1
View File
@@ -8,6 +8,7 @@ expression: "render_bottom_popup(&chat, 80)"
1. System default
› 2. Audio interface (current)
a. Input channels All channels (mixed)
3. Other device
4. Back
codex-rs/voice-host/src/devices.rs
+7 -5
View File
@@ -3,6 +3,9 @@
//! Capture and actual rendered output carry device timing.
//! References start with worker service; unmute rejects earlier device capture buffers.
#[path = "input_channel.rs"]
mod input_channel;
#[path = "audio_sink.rs"]
mod audio_sink;
#[path = "device_buffers.rs"]
@@ -163,6 +166,7 @@ impl Devices {
.default_output_config()
.map_err(|_| io::Error::other("speaker configuration unavailable"))?;
let input_stream_config = bounded_stream_config(&input_config)?;
let source = input_channel::InputChannel::new(selection.channel, input_config.channels())?;
let output_stream_config = bounded_stream_config(&output_config)?;
let buffers = Arc::new(Buffers::new(
input_config.sample_rate(),
@@ -186,6 +190,7 @@ impl Devices {
build_input,
&input,
&input_stream_config,
source,
buffers.clone()
)
.map_err(|_| io::Error::other("failed to open microphone"))?;
@@ -367,6 +372,7 @@ fn handle_stream_error(buffers: &Buffers, error: cpal::Error) {
fn build_input<T>(
device: &cpal::Device,
config: &cpal::StreamConfig,
source: input_channel::InputChannel,
buffers: Arc<Buffers>,
) -> Result<cpal::Stream, cpal::Error>
where
@@ -417,11 +423,7 @@ where
generation,
};
for (output, input) in frame.samples.iter_mut().zip(chunk.chunks_exact(channels)) {
*output = input
.iter()
.map(|sample| f32::from_sample(*sample))
.sum::<f32>()
/ channels as f32;
*output = source.sample(input);
if !output.is_finite() {
buffers.failed.store(true, Ordering::Release);
return;
codex-rs/voice-host/src/input_channel.rs
+45
View File
@@ -0,0 +1,45 @@
//! Mixes validated microphone channels before metering or processing captured audio.
//! Unset selections retain the legacy mix; explicit selections never fall back to other channels.
use std::io;
use std::num::NonZeroU16;
use cpal::FromSample;
use cpal::Sample;
pub(super) struct InputChannel(Vec<usize>);
impl InputChannel {
pub(super) fn new(channel: Option<Vec<NonZeroU16>>, channels: u16) -> io::Result<Self> {
let mut indices: Vec<_> = channel.map_or_else(
|| (0..usize::from(channels)).collect(),
|selected| {
selected
.into_iter()
.map(|channel| usize::from(channel.get() - 1))
.collect()
},
);
if indices.is_empty() || indices.iter().any(|index| *index >= usize::from(channels)) {
return Err(io::Error::other("selected microphone channels unavailable"));
}
indices.sort_unstable();
indices.dedup();
Ok(Self(indices))
}
pub(super) fn sample<T: Copy>(&self, frame: &[T]) -> f32
where
f32: FromSample<T>,
{
self.0
.iter()
.map(|index| f32::from_sample(frame[*index]))
.sum::<f32>()
/ self.0.len() as f32
}
}
#[cfg(test)]
#[path = "input_channel_tests.rs"]
mod tests;
codex-rs/voice-host/src/input_channel_tests.rs
+76
View File
@@ -0,0 +1,76 @@
//! Multichannel capture regression coverage independent of physical hardware.
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn microphone_selection_excludes_playback_channels() {
let source = InputChannel::new(
Some(vec![NonZeroU16::new(/*n*/ 1).unwrap()]),
/*channels*/ 16,
)
.unwrap();
let mut plugged_in = [0.0_f32; 16];
plugged_in[0] = 0.25;
plugged_in[2] = 0.8;
plugged_in[3] = 0.7;
let mut unplugged = plugged_in;
unplugged[0] = 0.0;
let interleaved = [plugged_in, unplugged].concat();
let captured: Vec<_> = interleaved
.chunks_exact(/*chunk_size*/ 16)
.map(|frame| source.sample(frame))
.collect();
assert_eq!(captured, vec![0.25, 0.0]);
}
#[test]
fn selected_channel_converts_integer_samples_without_attenuation() {
let source = InputChannel::new(
Some(vec![NonZeroU16::new(/*n*/ 2).unwrap()]),
/*channels*/ 2,
)
.unwrap();
assert_eq!(source.sample(&[0_i16, 16384]), 0.5);
}
#[test]
fn unset_channel_preserves_mixing() {
let source = InputChannel::new(/*channel*/ None, /*channels*/ 2).unwrap();
assert_eq!(source.sample(&[0.25_f32, 0.75]), 0.5);
}
#[test]
fn unavailable_channel_is_rejected() {
assert!(
InputChannel::new(
Some(vec![NonZeroU16::new(/*n*/ 17).unwrap()]),
/*channels*/ 16
)
.is_err()
);
assert!(InputChannel::new(/*channel*/ None, /*channels*/ 0).is_err());
}
#[test]
fn subset_mixes_only_selected_inputs_and_deduplicates() {
let selection = Some(vec![
NonZeroU16::new(/*n*/ 2).unwrap(),
NonZeroU16::new(/*n*/ 1).unwrap(),
NonZeroU16::new(/*n*/ 2).unwrap(),
]);
let source = InputChannel::new(selection, /*channels*/ 4).unwrap();
assert_eq!(source.sample(&[0.2_f32, 0.6, 0.9, 0.9]), 0.4);
assert_eq!(source.sample(&[0.0_f32, 0.0, 0.9, 0.9]), 0.0);
assert!(InputChannel::new(Some(vec![]), /*channels*/ 4).is_err());
assert!(
InputChannel::new(
Some(vec![
NonZeroU16::new(/*n*/ 1).unwrap(),
NonZeroU16::new(/*n*/ 5).unwrap()
]),
/*channels*/ 4
)
.is_err()
);
}