From 7dc7c7a7566a970f6d4d09e1384f854aebaf39e0 Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Sat, 5 Sep 2026 19:52:16 +0000 Subject: [PATCH] Send processed microphone audio over RTP in voice-host (#43090) ## Why `voice-host` previously drained captured audio locally without sending it to the peer. Connect capture to the outgoing media track while preserving mute boundaries and limiting stale audio. ## What changed - Resample capture and render references to 48 kHz with Rubato, apply Sonora echo cancellation, noise suppression, and gain control, and encode capture as 20 ms mono Opus packets. - Attach an Opus track to the WebRTC peer and advance RTP timestamps across mute gaps without introducing sequence-number gaps. - Prioritize queued controls and shutdown between sends. Clear pending audio and processing history on microphone mute transitions, reject delayed pre-unmute buffers, and reset partial history after capture gaps. - Size bounded capture and render queues for device rates, validate callback timing, enforce a 500 ms processing deadline, and time out stalled sends after 100 ms. Receive/decode and TUI integration remain unimplemented. The bundled Opus encoder build requires CMake and a C compiler. ## Testing Add synthetic-audio tests for Opus decoding at 44.1 and 48 kHz input rates, echo reduction, capture gaps, stale audio, queue capacity, and control priority. A loopback WebRTC test verifies RTP delivery, decoded signal, mute isolation, and timestamp and sequence continuity across mute gaps. GitOrigin-RevId: 38b8438577cfadb8124f8aeb426dffbab4845964 --- MODULE.bazel.lock | 16 + codex-rs/Cargo.lock | 162 ++++++++++ codex-rs/voice-host/Cargo.toml | 4 + codex-rs/voice-host/README.md | 10 +- codex-rs/voice-host/src/audio_track.rs | 122 +++++++ codex-rs/voice-host/src/audio_track_tests.rs | 17 + codex-rs/voice-host/src/capture_worker.rs | 79 +++++ .../voice-host/src/capture_worker_tests.rs | 300 ++++++++++++++++++ codex-rs/voice-host/src/device_buffers.rs | 32 +- .../voice-host/src/device_buffers_tests.rs | 48 ++- codex-rs/voice-host/src/devices.rs | 67 ++-- codex-rs/voice-host/src/devices_tests.rs | 93 ++++-- .../voice-host/src/devices_unavailable.rs | 7 +- codex-rs/voice-host/src/main.rs | 42 ++- codex-rs/voice-host/src/main_tests.rs | 44 ++- codex-rs/voice-host/src/processing.rs | 251 +++++++++++++++ codex-rs/voice-host/src/processing_tests.rs | 285 +++++++++++++++++ codex-rs/voice-host/src/transport.rs | 8 + codex-rs/voice-host/src/transport_tests.rs | 2 + codex-rs/voice-host/tests/lifecycle.rs | 4 + 20 files changed, 1526 insertions(+), 67 deletions(-) create mode 100644 codex-rs/voice-host/src/audio_track.rs create mode 100644 codex-rs/voice-host/src/audio_track_tests.rs create mode 100644 codex-rs/voice-host/src/capture_worker.rs create mode 100644 codex-rs/voice-host/src/capture_worker_tests.rs create mode 100644 codex-rs/voice-host/src/processing.rs create mode 100644 codex-rs/voice-host/src/processing_tests.rs diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f785217ce1..8a91f2cbee 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -731,6 +731,10 @@ "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", "atomic_0.5.3": "{\"dependencies\":[],\"features\":{\"default\":[\"fallback\"],\"fallback\":[],\"nightly\":[],\"std\":[]}}", "atomic_0.6.1": "{\"dependencies\":[{\"name\":\"bytemuck\",\"req\":\"^1.13.1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.13.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.219\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.219\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"default\":[\"fallback\"],\"fallback\":[],\"nightly\":[],\"serde\":[\"dep:serde\"],\"std\":[]}}", + "audio-codec-algorithms_0.8.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8.2\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"internal-no-panic\":[\"dep:no-panic\"]}}", + "audioadapter-buffers_5.2.0": "{\"dependencies\":[{\"name\":\"audioadapter\",\"req\":\"^5.0.0\"},{\"features\":[\"test-utils\"],\"kind\":\"dev\",\"name\":\"audioadapter\",\"req\":\"^5.0.0\"},{\"default_features\":false,\"name\":\"audioadapter-sample\",\"req\":\"^5.2.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"audioadapter-sample/std\",\"num-traits/std\",\"alloc\"]}}", + "audioadapter-sample_5.2.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"audio-codec-algorithms\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"num-traits/std\"]}}", + "audioadapter_5.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"default_features\":false,\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.15\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"num-traits\",\"req\":\"^0.2.15\"}],\"features\":{\"default\":[],\"test-utils\":[\"dep:num-traits\"]}}", "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", "aws-config_1.8.12": "{\"dependencies\":[{\"features\":[\"test-util\"],\"name\":\"aws-credential-types\",\"req\":\"^1.2.11\"},{\"name\":\"aws-runtime\",\"req\":\"^1.5.17\"},{\"default_features\":false,\"name\":\"aws-sdk-signin\",\"optional\":true,\"req\":\"^1.2.0\"},{\"default_features\":false,\"name\":\"aws-sdk-sso\",\"optional\":true,\"req\":\"^1.91.0\"},{\"default_features\":false,\"name\":\"aws-sdk-ssooidc\",\"optional\":true,\"req\":\"^1.93.0\"},{\"default_features\":false,\"name\":\"aws-sdk-sts\",\"req\":\"^1.95.0\"},{\"name\":\"aws-smithy-async\",\"req\":\"^1.2.7\"},{\"features\":[\"rt-tokio\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-async\",\"req\":\"^1.2.7\"},{\"name\":\"aws-smithy-http\",\"req\":\"^0.62.6\"},{\"features\":[\"default-client\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-http-client\",\"req\":\"^1.1.5\"},{\"name\":\"aws-smithy-json\",\"req\":\"^0.61.8\"},{\"features\":[\"client\"],\"name\":\"aws-smithy-runtime\",\"req\":\"^1.9.5\"},{\"features\":[\"client\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime\",\"req\":\"^1.9.5\"},{\"features\":[\"client\"],\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.9.3\"},{\"features\":[\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.9.3\"},{\"name\":\"aws-smithy-types\",\"req\":\"^1.3.5\"},{\"name\":\"aws-types\",\"req\":\"^1.3.11\"},{\"name\":\"base64-simd\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bytes\",\"req\":\"^1.1.0\"},{\"name\":\"fastrand\",\"req\":\"^2.3.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.29\"},{\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"p256\",\"optional\":true,\"req\":\"^0.13.2\"},{\"default_features\":false,\"features\":[\"std\",\"std_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8.5\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10.9\"},{\"features\":[\"parsing\"],\"name\":\"time\",\"req\":\"^0.3.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.13.1\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.23.1\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"fmt\",\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"tracing-test\",\"req\":\"^0.2.4\"},{\"name\":\"url\",\"req\":\"^2.5.4\"},{\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.18.1\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"allow-compilation\":[],\"behavior-version-latest\":[],\"client-hyper\":[\"aws-smithy-runtime/default-https-client\"],\"credentials-login\":[\"dep:aws-sdk-signin\",\"dep:sha2\",\"dep:zeroize\",\"dep:hex\",\"dep:base64-simd\",\"dep:uuid\",\"uuid?/v4\",\"dep:p256\",\"p256?/arithmetic\",\"p256?/pem\",\"dep:rand\"],\"credentials-process\":[\"tokio/process\"],\"default\":[\"default-https-client\",\"rt-tokio\",\"credentials-process\",\"sso\"],\"default-https-client\":[\"aws-smithy-runtime/default-https-client\"],\"rt-tokio\":[\"aws-smithy-async/rt-tokio\",\"aws-smithy-runtime/rt-tokio\",\"tokio/rt\"],\"rustls\":[\"client-hyper\"],\"sso\":[\"dep:aws-sdk-sso\",\"dep:aws-sdk-ssooidc\",\"dep:ring\",\"dep:hex\",\"dep:zeroize\",\"aws-smithy-runtime-api/http-auth\"],\"test-util\":[\"aws-runtime/test-util\"]}}", "aws-credential-types_1.2.11": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.74\"},{\"name\":\"aws-smithy-async\",\"req\":\"^1.2.7\"},{\"features\":[\"client\",\"http-auth\"],\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.9.3\"},{\"features\":[\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.9.3\"},{\"name\":\"aws-smithy-types\",\"req\":\"^1.3.5\"},{\"features\":[\"full\",\"test-util\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.23.1\"},{\"name\":\"zeroize\",\"req\":\"^1.7.0\"}],\"features\":{\"hardcoded-credentials\":[],\"test-util\":[\"aws-smithy-runtime-api/test-util\"]}}", @@ -1429,6 +1433,8 @@ "opentelemetry_0.31.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"js-sys\",\"req\":\"^0.3.63\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"os_rng\",\"thread_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\">=0.1.40\"}],\"features\":{\"default\":[\"trace\",\"metrics\",\"logs\",\"internal-logs\",\"futures\"],\"futures\":[\"futures-core\",\"futures-sink\",\"pin-project-lite\"],\"internal-logs\":[\"tracing\"],\"logs\":[],\"metrics\":[],\"spec_unstable_logs_enabled\":[\"logs\"],\"testing\":[\"trace\"],\"trace\":[\"futures\",\"thiserror\"]}}", "opentelemetry_sdk_0.31.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-executor\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\",\"sink\",\"async-await-macro\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"http\",\"optional\":true,\"req\":\"^1.1\"},{\"default_features\":false,\"name\":\"opentelemetry\",\"req\":\"^0.31\"},{\"default_features\":false,\"name\":\"opentelemetry-http\",\"optional\":true,\"req\":\"^0.31\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"flamegraph\",\"criterion\"],\"kind\":\"dev\",\"name\":\"pprof\",\"req\":\"^0.14\",\"target\":\"cfg(not(target_os = \\\"windows\\\"))\"},{\"default_features\":false,\"features\":[\"std\",\"std_rng\",\"small_rng\",\"os_rng\",\"thread_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.23.0\"},{\"default_features\":false,\"features\":[\"derive\",\"rc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"temp-env\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"url\",\"optional\":true,\"req\":\"^2.5\"}],\"features\":{\"default\":[\"trace\",\"metrics\",\"logs\",\"internal-logs\"],\"experimental_async_runtime\":[],\"experimental_logs_batch_log_processor_with_async_runtime\":[\"logs\",\"experimental_async_runtime\"],\"experimental_logs_concurrent_log_processor\":[\"logs\"],\"experimental_metrics_custom_reader\":[\"metrics\"],\"experimental_metrics_disable_name_validation\":[\"metrics\"],\"experimental_metrics_periodicreader_with_async_runtime\":[\"metrics\",\"experimental_async_runtime\"],\"experimental_trace_batch_span_processor_with_async_runtime\":[\"tokio/sync\",\"trace\",\"experimental_async_runtime\"],\"internal-logs\":[\"opentelemetry/internal-logs\"],\"jaeger_remote_sampler\":[\"trace\",\"opentelemetry-http\",\"http\",\"serde\",\"serde_json\",\"url\",\"experimental_async_runtime\"],\"logs\":[\"opentelemetry/logs\"],\"metrics\":[\"opentelemetry/metrics\"],\"rt-tokio\":[\"tokio/rt\",\"tokio/time\",\"tokio-stream\",\"experimental_async_runtime\"],\"rt-tokio-current-thread\":[\"tokio/rt\",\"tokio/time\",\"tokio-stream\",\"experimental_async_runtime\"],\"spec_unstable_logs_enabled\":[\"logs\",\"opentelemetry/spec_unstable_logs_enabled\"],\"spec_unstable_metrics_views\":[\"metrics\"],\"testing\":[\"opentelemetry/testing\",\"trace\",\"metrics\",\"logs\",\"rt-tokio\",\"rt-tokio-current-thread\",\"tokio/macros\",\"tokio/rt-multi-thread\"],\"trace\":[\"opentelemetry/trace\",\"rand\",\"percent-encoding\"]}}", "option-ext_0.2.0": "{\"dependencies\":[],\"features\":{}}", + "opus_0.4.0": "{\"dependencies\":[{\"name\":\"opusic-sys\",\"req\":\"^0.7.3\"}],\"features\":{}}", + "opusic-sys_0.7.5": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.72\"},{\"kind\":\"build\",\"name\":\"cmake\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{\"build-bindgen\":[\"bindgen\"],\"bundled\":[\"dep:cmake\"],\"default\":[\"bundled\"],\"dred\":[],\"fixed-point\":[],\"no-fortify-source\":[],\"no-hardening\":[],\"no-runtime-feature-detection\":[],\"no-simd\":[],\"no-stack-protector\":[],\"osce\":[]}}", "ordered-float_4.6.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2.0\"},{\"default_features\":false,\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12.2\"},{\"name\":\"derive-visitor\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"num-cmp\",\"optional\":true,\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.9\"},{\"name\":\"proptest\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8.3\"},{\"default_features\":false,\"features\":[\"rend\"],\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.7.41\"},{\"name\":\"schemars\",\"optional\":true,\"req\":\"^0.8.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"speedy\",\"optional\":true,\"req\":\"^0.8.3\"}],\"features\":{\"default\":[\"std\"],\"libm\":[\"num-traits/libm\"],\"randtest\":[\"rand/std\",\"rand/std_rng\"],\"rkyv\":[\"rkyv_32\"],\"rkyv_16\":[\"dep:rkyv\",\"rkyv?/size_16\"],\"rkyv_32\":[\"dep:rkyv\",\"rkyv?/size_32\"],\"rkyv_64\":[\"dep:rkyv\",\"rkyv?/size_64\"],\"rkyv_ck\":[\"rkyv?/validation\"],\"serde\":[\"dep:serde\",\"rand?/serde1\"],\"std\":[\"num-traits/std\"]}}", "ordered-stream_0.2.0": "{\"dependencies\":[{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"futures-executor\",\"req\":\"^0.3.25\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.25\"}],\"features\":{}}", "os_info_3.14.0": "{\"dependencies\":[{\"name\":\"android_system_properties\",\"req\":\"^0.1\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"features\":[\"feature\"],\"name\":\"nix\",\"req\":\"^0.30\",\"target\":\"cfg(any(target_os = \\\"aix\\\", target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"illumos\\\", target_os = \\\"linux\\\", target_os = \\\"macos\\\", target_os = \\\"netbsd\\\", target_os = \\\"openbsd\\\", target_os = \\\"cygwin\\\"))\"},{\"name\":\"objc2\",\"req\":\"^0.6\",\"target\":\"cfg(target_os = \\\"ios\\\")\"},{\"features\":[\"NSString\"],\"name\":\"objc2-foundation\",\"req\":\"^0.3\",\"target\":\"cfg(target_os = \\\"ios\\\")\"},{\"features\":[\"NSData\",\"NSError\",\"NSEnumerator\",\"NSString\"],\"name\":\"objc2-foundation\",\"req\":\"^0.3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"name\":\"objc2-ui-kit\",\"req\":\"^0.3\",\"target\":\"cfg(target_os = \\\"ios\\\")\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1\"},{\"name\":\"schemars\",\"optional\":true,\"req\":\"^1.0.3\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_LibraryLoader\",\"Win32_System_Registry\",\"Win32_System_SystemInformation\",\"Win32_System_SystemServices\",\"Win32_System_Threading\",\"Win32_UI_WindowsAndMessaging\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"serde\"]}}", @@ -1622,6 +1628,7 @@ "rtc-stun_0.20.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"aws-lc-sys\"],\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.17.3\"},{\"name\":\"base64\",\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.12.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.6.1\"},{\"name\":\"crc\",\"req\":\"^3.0.1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8.2\"},{\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"md-5\",\"req\":\"^0.10\"},{\"name\":\"rand\",\"req\":\"^0.10.1\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17.14\"},{\"name\":\"sansio\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"shared\",\"package\":\"rtc-shared\",\"req\":\"^0.20.3\"},{\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"name\":\"url\",\"req\":\"^2.5.0\"}],\"features\":{\"aws-lc-rs\":[\"dep:aws-lc-rs\"],\"bench\":[],\"default\":[\"ring\"],\"ring\":[\"dep:ring\"]}}", "rtc-turn_0.20.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1.12.0\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.45\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.6.1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8.2\"},{\"kind\":\"dev\",\"name\":\"crossbeam-channel\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"ctrlc\",\"req\":\"^3.5.2\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11.11\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"log\",\"req\":\"^0.4.33\"},{\"name\":\"sansio\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"shared\",\"package\":\"rtc-shared\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"name\":\"stun\",\"package\":\"rtc-stun\",\"req\":\"^0.20.3\"}],\"features\":{\"aws-lc-rs\":[\"stun/aws-lc-rs\"],\"default\":[\"ring\"],\"metrics\":[],\"ring\":[\"stun/ring\"]}}", "rtc_0.20.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"aws-lc-sys\"],\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.17.3\"},{\"name\":\"bytes\",\"req\":\"^1.12.0\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1.12.0\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.45\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.6.1\"},{\"kind\":\"dev\",\"name\":\"ctrlc\",\"req\":\"^3.5.2\"},{\"name\":\"datachannel\",\"package\":\"rtc-datachannel\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"name\":\"dtls\",\"package\":\"rtc-dtls\",\"req\":\"^0.20.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11.11\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"hex\",\"req\":\"^0.4\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14.32\"},{\"default_features\":false,\"name\":\"ice\",\"package\":\"rtc-ice\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ice\",\"package\":\"rtc-ice\",\"req\":\"^0.20.3\"},{\"name\":\"interceptor\",\"package\":\"rtc-interceptor\",\"req\":\"^0.20.3\"},{\"name\":\"log\",\"req\":\"^0.4.33\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.33\"},{\"name\":\"mdns\",\"package\":\"rtc-mdns\",\"req\":\"^0.20.3\"},{\"name\":\"media\",\"package\":\"rtc-media\",\"req\":\"^0.20.3\"},{\"name\":\"pem\",\"optional\":true,\"req\":\"^3\"},{\"name\":\"rand\",\"req\":\"^0.10.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10.1\"},{\"default_features\":false,\"features\":[\"pem\"],\"name\":\"rcgen\",\"req\":\"^0.14.8\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17.14\"},{\"name\":\"rtcp\",\"package\":\"rtc-rtcp\",\"req\":\"^0.20.3\"},{\"name\":\"rtp\",\"package\":\"rtc-rtp\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"rustls\",\"req\":\"^0.23.35\"},{\"name\":\"sansio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"sansio\",\"req\":\"^1\"},{\"name\":\"sctp\",\"package\":\"rtc-sctp\",\"req\":\"^0.20.3\"},{\"name\":\"sdp\",\"package\":\"rtc-sdp\",\"req\":\"^0.20.3\"},{\"name\":\"serde\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.228\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.145\"},{\"name\":\"sha2\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"crypto\",\"marshal\",\"replay\"],\"name\":\"shared\",\"package\":\"rtc-shared\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"shared\",\"package\":\"rtc-shared\",\"req\":\"^0.20.3\"},{\"kind\":\"dev\",\"name\":\"signal\",\"package\":\"rtc-signal\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"name\":\"srtp\",\"package\":\"rtc-srtp\",\"req\":\"^0.20.3\"},{\"default_features\":false,\"name\":\"stun\",\"package\":\"rtc-stun\",\"req\":\"^0.20.3\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.52.3\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.28.0\"},{\"features\":[\"codec\"],\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.17\"},{\"default_features\":false,\"name\":\"turn\",\"package\":\"rtc-turn\",\"req\":\"^0.20.3\"},{\"name\":\"unicase\",\"req\":\"^2.8\"},{\"name\":\"url\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"webrtc\",\"req\":\"^0.14.0\"}],\"features\":{\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"dtls/aws-lc-rs\",\"rustls/aws-lc-rs\",\"rcgen/aws_lc_rs\",\"ice/aws-lc-rs\",\"stun/aws-lc-rs\",\"srtp/aws-lc-rs\",\"turn/aws-lc-rs\"],\"default\":[\"ring\"],\"openssl\":[\"srtp/openssl\"],\"pem\":[\"dep:pem\",\"dtls/pem\"],\"ring\":[\"dep:ring\",\"dtls/ring\",\"rustls/ring\",\"rcgen/ring\",\"ice/ring\",\"stun/ring\",\"srtp/ring\",\"turn/ring\"],\"vendored-openssl\":[\"srtp/vendored-openssl\"]}}", + "rubato_5.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"approx\",\"req\":\"^0.5.1\"},{\"name\":\"audioadapter\",\"req\":\"^5.0\"},{\"name\":\"audioadapter-buffers\",\"req\":\"^5.1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.6\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.18\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.18\"},{\"name\":\"num-complex\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"num-integer\",\"req\":\"^0.1.45\"},{\"name\":\"num-traits\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-traits\",\"req\":\"^0.2.15\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10.0\"},{\"name\":\"realfft\",\"optional\":true,\"req\":\"^3.5.0\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-log\",\"req\":\"^0.2.16\"},{\"name\":\"visibility\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"waveadapter\",\"req\":\"^0.2\"},{\"name\":\"windowfunctions\",\"req\":\"^0.1.1\"}],\"features\":{\"bench_asyncro\":[],\"default\":[\"fft_resampler\"],\"fft_resampler\":[\"realfft\",\"num-complex\"],\"log\":[\"dep:log\"]}}", "rust-embed-impl_8.11.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"rust-embed-utils\",\"req\":\"^8.11.0\"},{\"name\":\"shellexpand\",\"optional\":true,\"req\":\"^3\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"proc-macro\",\"printing\"],\"name\":\"syn\",\"req\":\"^2\"},{\"name\":\"walkdir\",\"req\":\"^2.3.1\"}],\"features\":{\"compression\":[],\"debug-embed\":[],\"deterministic-timestamps\":[],\"include-exclude\":[\"rust-embed-utils/include-exclude\"],\"interpolate-folder-path\":[\"shellexpand\"],\"mime-guess\":[\"rust-embed-utils/mime-guess\"]}}", "rust-embed-utils_8.11.0": "{\"dependencies\":[{\"name\":\"globset\",\"optional\":true,\"req\":\"^0.4.8\"},{\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0.4\"},{\"name\":\"sha2\",\"req\":\"^0.10.5\"},{\"name\":\"walkdir\",\"req\":\"^2.3.1\"}],\"features\":{\"debug-embed\":[],\"include-exclude\":[\"globset\"],\"mime-guess\":[\"mime_guess\"]}}", "rust-embed_8.11.0": "{\"dependencies\":[{\"name\":\"actix-web\",\"optional\":true,\"req\":\"^4\"},{\"default_features\":false,\"features\":[\"http1\",\"tokio\"],\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"name\":\"include-flate\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0.5\"},{\"default_features\":false,\"features\":[\"server\"],\"name\":\"poem\",\"optional\":true,\"req\":\"^1.3.30\"},{\"default_features\":false,\"name\":\"rocket\",\"optional\":true,\"req\":\"^0.5.0-rc.2\"},{\"name\":\"rust-embed-impl\",\"req\":\"^8.9.0\"},{\"name\":\"rust-embed-utils\",\"req\":\"^8.9.0\"},{\"default_features\":false,\"name\":\"salvo\",\"optional\":true,\"req\":\"^0.16\"},{\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"default_features\":false,\"name\":\"warp\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"actix\":[\"actix-web\",\"mime_guess\"],\"axum-ex\":[\"axum\",\"tokio\",\"mime_guess\"],\"compression\":[\"rust-embed-impl/compression\",\"include-flate\"],\"debug-embed\":[\"rust-embed-impl/debug-embed\",\"rust-embed-utils/debug-embed\"],\"deterministic-timestamps\":[\"rust-embed-impl/deterministic-timestamps\"],\"include-exclude\":[\"rust-embed-impl/include-exclude\",\"rust-embed-utils/include-exclude\"],\"interpolate-folder-path\":[\"rust-embed-impl/interpolate-folder-path\"],\"mime-guess\":[\"rust-embed-impl/mime-guess\",\"rust-embed-utils/mime-guess\"],\"poem-ex\":[\"poem\",\"tokio\",\"mime_guess\",\"hex\"],\"salvo-ex\":[\"salvo\",\"tokio\",\"mime_guess\",\"hex\"],\"warp-ex\":[\"warp\",\"tokio\",\"mime_guess\"]}}", @@ -1732,6 +1739,13 @@ "smol_str_0.3.5": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.4.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\"],\"std\":[\"serde_core?/std\",\"borsh?/std\"]}}", "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "sonora-aec3_0.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11\"},{\"name\":\"sonora-common-audio\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-fft\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-simd\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.4\"}],\"features\":{}}", + "sonora-agc2_0.2.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"bytemuck\",\"req\":\"^1\"},{\"features\":[\"debug\"],\"name\":\"derive_more\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11\"},{\"name\":\"sonora-common-audio\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-fft\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-simd\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.4\"},{\"name\":\"tracing\",\"req\":\"^0.1\"}],\"features\":{}}", + "sonora-common-audio_0.2.0": "{\"dependencies\":[{\"features\":[\"debug\"],\"name\":\"derive_more\",\"req\":\"^2\"},{\"name\":\"sonora-simd\",\"req\":\"^0.2.0\"}],\"features\":{}}", + "sonora-fft_0.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.4\"}],\"features\":{}}", + "sonora-ns_0.2.0": "{\"dependencies\":[{\"name\":\"sonora-fft\",\"req\":\"^0.2.0\"}],\"features\":{}}", + "sonora-simd_0.2.0": "{\"dependencies\":[{\"name\":\"cpufeatures\",\"req\":\"^0.3\",\"target\":\"cfg(any(target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"}],\"features\":{}}", + "sonora_0.2.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4\"},{\"name\":\"cpal\",\"optional\":true,\"req\":\"^0.18\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\"},{\"features\":[\"termination\"],\"name\":\"ctrlc\",\"optional\":true,\"req\":\"^3\"},{\"name\":\"hound\",\"optional\":true,\"req\":\"^3\"},{\"name\":\"ringbuf\",\"optional\":true,\"req\":\"^0.5\"},{\"name\":\"sonora-aec3\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-agc2\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-common-audio\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"sonora-fft\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-ns\",\"req\":\"^0.2.0\"},{\"name\":\"sonora-simd\",\"req\":\"^0.2.0\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"examples\":[\"dep:anyhow\",\"dep:clap\",\"dep:cpal\",\"dep:ctrlc\",\"dep:hound\",\"dep:ringbuf\"]}}", "sorted_vector_map_0.2.1": "{\"dependencies\":[{\"name\":\"itertools\",\"req\":\"^0.14.0\"},{\"name\":\"quickcheck\",\"req\":\"^1.0\"}],\"features\":{}}", "spin_0.9.8": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\"},{\"name\":\"lock_api_crate\",\"optional\":true,\"package\":\"lock_api\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"barrier\":[\"mutex\"],\"default\":[\"lock_api\",\"mutex\",\"spin_mutex\",\"rwlock\",\"once\",\"lazy\",\"barrier\"],\"fair_mutex\":[\"mutex\"],\"lazy\":[\"once\"],\"lock_api\":[\"lock_api_crate\"],\"mutex\":[],\"once\":[],\"portable_atomic\":[\"portable-atomic\"],\"rwlock\":[],\"spin_mutex\":[\"mutex\"],\"std\":[],\"ticket_mutex\":[\"mutex\"],\"use_ticket_mutex\":[\"mutex\",\"ticket_mutex\"]}}", "spki_0.7.3": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"base64ct\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"oid\"],\"name\":\"der\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"alloc\":[\"base64ct?/alloc\",\"der/alloc\"],\"arbitrary\":[\"std\",\"dep:arbitrary\",\"der/arbitrary\"],\"base64\":[\"dep:base64ct\"],\"fingerprint\":[\"sha2\"],\"pem\":[\"alloc\",\"der/pem\"],\"std\":[\"der/std\",\"alloc\"]}}", @@ -1925,6 +1939,7 @@ "valuable_0.1.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"name\":\"valuable-derive\",\"optional\":true,\"req\":\"=0.1.1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"valuable-derive\"],\"std\":[\"alloc\"]}}", "vcpkg_0.2.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tempdir\",\"req\":\"^0.3.7\"}],\"features\":{}}", "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "visibility_0.1.1": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.26\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.0\"}],\"features\":{\"nightly\":[]}}", "vsimd_0.8.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"const-str\",\"req\":\"^0.5.3\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2.8\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.33\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"alloc\":[],\"detect\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", "vt100_0.16.2": "{\"dependencies\":[{\"name\":\"itoa\",\"req\":\"^1.0.15\"},{\"features\":[\"term\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.30.1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.219\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"},{\"kind\":\"dev\",\"name\":\"terminal_size\",\"req\":\"^0.4.2\"},{\"name\":\"unicode-width\",\"req\":\"^0.2.1\"},{\"name\":\"vte\",\"req\":\"^0.15.0\"}],\"features\":{}}", "vte_0.15.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arrayvec\",\"req\":\"^0.7.2\"},{\"default_features\":false,\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"default_features\":false,\"name\":\"cursor-icon\",\"optional\":true,\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.7.4\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.160\"}],\"features\":{\"ansi\":[\"log\",\"cursor-icon\",\"bitflags\"],\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[\"memchr/std\"]}}", @@ -1994,6 +2009,7 @@ "winapi-util_0.1.11": "{\"dependencies\":[{\"features\":[\"Win32_Foundation\",\"Win32_Storage_FileSystem\",\"Win32_System_Console\",\"Win32_System_SystemInformation\"],\"name\":\"windows-sys\",\"req\":\">=0.48.0, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{}}", "winapi-x86_64-pc-windows-gnu_0.4.0": "{\"dependencies\":[],\"features\":{}}", "winapi_0.3.9": "{\"dependencies\":[{\"name\":\"winapi-i686-pc-windows-gnu\",\"req\":\"^0.4\",\"target\":\"i686-pc-windows-gnu\"},{\"name\":\"winapi-x86_64-pc-windows-gnu\",\"req\":\"^0.4\",\"target\":\"x86_64-pc-windows-gnu\"}],\"features\":{\"accctrl\":[],\"aclapi\":[],\"activation\":[],\"adhoc\":[],\"appmgmt\":[],\"audioclient\":[],\"audiosessiontypes\":[],\"avrt\":[],\"basetsd\":[],\"bcrypt\":[],\"bits\":[],\"bits10_1\":[],\"bits1_5\":[],\"bits2_0\":[],\"bits2_5\":[],\"bits3_0\":[],\"bits4_0\":[],\"bits5_0\":[],\"bitscfg\":[],\"bitsmsg\":[],\"bluetoothapis\":[],\"bluetoothleapis\":[],\"bthdef\":[],\"bthioctl\":[],\"bthledef\":[],\"bthsdpdef\":[],\"bugcodes\":[],\"cderr\":[],\"cfg\":[],\"cfgmgr32\":[],\"cguid\":[],\"combaseapi\":[],\"coml2api\":[],\"commapi\":[],\"commctrl\":[],\"commdlg\":[],\"commoncontrols\":[],\"consoleapi\":[],\"corecrt\":[],\"corsym\":[],\"d2d1\":[],\"d2d1_1\":[],\"d2d1_2\":[],\"d2d1_3\":[],\"d2d1effectauthor\":[],\"d2d1effects\":[],\"d2d1effects_1\":[],\"d2d1effects_2\":[],\"d2d1svg\":[],\"d2dbasetypes\":[],\"d3d\":[],\"d3d10\":[],\"d3d10_1\":[],\"d3d10_1shader\":[],\"d3d10effect\":[],\"d3d10misc\":[],\"d3d10sdklayers\":[],\"d3d10shader\":[],\"d3d11\":[],\"d3d11_1\":[],\"d3d11_2\":[],\"d3d11_3\":[],\"d3d11_4\":[],\"d3d11on12\":[],\"d3d11sdklayers\":[],\"d3d11shader\":[],\"d3d11tokenizedprogramformat\":[],\"d3d12\":[],\"d3d12sdklayers\":[],\"d3d12shader\":[],\"d3d9\":[],\"d3d9caps\":[],\"d3d9types\":[],\"d3dcommon\":[],\"d3dcompiler\":[],\"d3dcsx\":[],\"d3dkmdt\":[],\"d3dkmthk\":[],\"d3dukmdt\":[],\"d3dx10core\":[],\"d3dx10math\":[],\"d3dx10mesh\":[],\"datetimeapi\":[],\"davclnt\":[],\"dbghelp\":[],\"dbt\":[],\"dcommon\":[],\"dcomp\":[],\"dcompanimation\":[],\"dcomptypes\":[],\"dde\":[],\"ddraw\":[],\"ddrawi\":[],\"ddrawint\":[],\"debug\":[\"impl-debug\"],\"debugapi\":[],\"devguid\":[],\"devicetopology\":[],\"devpkey\":[],\"devpropdef\":[],\"dinput\":[],\"dinputd\":[],\"dispex\":[],\"dmksctl\":[],\"dmusicc\":[],\"docobj\":[],\"documenttarget\":[],\"dot1x\":[],\"dpa_dsa\":[],\"dpapi\":[],\"dsgetdc\":[],\"dsound\":[],\"dsrole\":[],\"dvp\":[],\"dwmapi\":[],\"dwrite\":[],\"dwrite_1\":[],\"dwrite_2\":[],\"dwrite_3\":[],\"dxdiag\":[],\"dxfile\":[],\"dxgi\":[],\"dxgi1_2\":[],\"dxgi1_3\":[],\"dxgi1_4\":[],\"dxgi1_5\":[],\"dxgi1_6\":[],\"dxgidebug\":[],\"dxgiformat\":[],\"dxgitype\":[],\"dxva2api\":[],\"dxvahd\":[],\"eaptypes\":[],\"enclaveapi\":[],\"endpointvolume\":[],\"errhandlingapi\":[],\"everything\":[],\"evntcons\":[],\"evntprov\":[],\"evntrace\":[],\"excpt\":[],\"exdisp\":[],\"fibersapi\":[],\"fileapi\":[],\"functiondiscoverykeys_devpkey\":[],\"gl-gl\":[],\"guiddef\":[],\"handleapi\":[],\"heapapi\":[],\"hidclass\":[],\"hidpi\":[],\"hidsdi\":[],\"hidusage\":[],\"highlevelmonitorconfigurationapi\":[],\"hstring\":[],\"http\":[],\"ifdef\":[],\"ifmib\":[],\"imm\":[],\"impl-debug\":[],\"impl-default\":[],\"in6addr\":[],\"inaddr\":[],\"inspectable\":[],\"interlockedapi\":[],\"intsafe\":[],\"ioapiset\":[],\"ipexport\":[],\"iphlpapi\":[],\"ipifcons\":[],\"ipmib\":[],\"iprtrmib\":[],\"iptypes\":[],\"jobapi\":[],\"jobapi2\":[],\"knownfolders\":[],\"ks\":[],\"ksmedia\":[],\"ktmtypes\":[],\"ktmw32\":[],\"l2cmn\":[],\"libloaderapi\":[],\"limits\":[],\"lmaccess\":[],\"lmalert\":[],\"lmapibuf\":[],\"lmat\":[],\"lmcons\":[],\"lmdfs\":[],\"lmerrlog\":[],\"lmjoin\":[],\"lmmsg\":[],\"lmremutl\":[],\"lmrepl\":[],\"lmserver\":[],\"lmshare\":[],\"lmstats\":[],\"lmsvc\":[],\"lmuse\":[],\"lmwksta\":[],\"lowlevelmonitorconfigurationapi\":[],\"lsalookup\":[],\"memoryapi\":[],\"minschannel\":[],\"minwinbase\":[],\"minwindef\":[],\"mmdeviceapi\":[],\"mmeapi\":[],\"mmreg\":[],\"mmsystem\":[],\"mprapidef\":[],\"msaatext\":[],\"mscat\":[],\"mschapp\":[],\"mssip\":[],\"mstcpip\":[],\"mswsock\":[],\"mswsockdef\":[],\"namedpipeapi\":[],\"namespaceapi\":[],\"nb30\":[],\"ncrypt\":[],\"netioapi\":[],\"nldef\":[],\"ntddndis\":[],\"ntddscsi\":[],\"ntddser\":[],\"ntdef\":[],\"ntlsa\":[],\"ntsecapi\":[],\"ntstatus\":[],\"oaidl\":[],\"objbase\":[],\"objidl\":[],\"objidlbase\":[],\"ocidl\":[],\"ole2\":[],\"oleauto\":[],\"olectl\":[],\"oleidl\":[],\"opmapi\":[],\"pdh\":[],\"perflib\":[],\"physicalmonitorenumerationapi\":[],\"playsoundapi\":[],\"portabledevice\":[],\"portabledeviceapi\":[],\"portabledevicetypes\":[],\"powerbase\":[],\"powersetting\":[],\"powrprof\":[],\"processenv\":[],\"processsnapshot\":[],\"processthreadsapi\":[],\"processtopologyapi\":[],\"profileapi\":[],\"propidl\":[],\"propkey\":[],\"propkeydef\":[],\"propsys\":[],\"prsht\":[],\"psapi\":[],\"qos\":[],\"realtimeapiset\":[],\"reason\":[],\"restartmanager\":[],\"restrictederrorinfo\":[],\"rmxfguid\":[],\"roapi\":[],\"robuffer\":[],\"roerrorapi\":[],\"rpc\":[],\"rpcdce\":[],\"rpcndr\":[],\"rtinfo\":[],\"sapi\":[],\"sapi51\":[],\"sapi53\":[],\"sapiddk\":[],\"sapiddk51\":[],\"schannel\":[],\"sddl\":[],\"securityappcontainer\":[],\"securitybaseapi\":[],\"servprov\":[],\"setupapi\":[],\"shellapi\":[],\"shellscalingapi\":[],\"shlobj\":[],\"shobjidl\":[],\"shobjidl_core\":[],\"shtypes\":[],\"softpub\":[],\"spapidef\":[],\"spellcheck\":[],\"sporder\":[],\"sql\":[],\"sqlext\":[],\"sqltypes\":[],\"sqlucode\":[],\"sspi\":[],\"std\":[],\"stralign\":[],\"stringapiset\":[],\"strmif\":[],\"subauth\":[],\"synchapi\":[],\"sysinfoapi\":[],\"systemtopologyapi\":[],\"taskschd\":[],\"tcpestats\":[],\"tcpmib\":[],\"textstor\":[],\"threadpoolapiset\":[],\"threadpoollegacyapiset\":[],\"timeapi\":[],\"timezoneapi\":[],\"tlhelp32\":[],\"transportsettingcommon\":[],\"tvout\":[],\"udpmib\":[],\"unknwnbase\":[],\"urlhist\":[],\"urlmon\":[],\"usb\":[],\"usbioctl\":[],\"usbiodef\":[],\"usbscan\":[],\"usbspec\":[],\"userenv\":[],\"usp10\":[],\"utilapiset\":[],\"uxtheme\":[],\"vadefs\":[],\"vcruntime\":[],\"vsbackup\":[],\"vss\":[],\"vsserror\":[],\"vswriter\":[],\"wbemads\":[],\"wbemcli\":[],\"wbemdisp\":[],\"wbemprov\":[],\"wbemtran\":[],\"wct\":[],\"werapi\":[],\"winbase\":[],\"wincodec\":[],\"wincodecsdk\":[],\"wincon\":[],\"wincontypes\":[],\"wincred\":[],\"wincrypt\":[],\"windef\":[],\"windot11\":[],\"windowsceip\":[],\"windowsx\":[],\"winefs\":[],\"winerror\":[],\"winevt\":[],\"wingdi\":[],\"winhttp\":[],\"wininet\":[],\"winineti\":[],\"winioctl\":[],\"winnetwk\":[],\"winnls\":[],\"winnt\":[],\"winreg\":[],\"winsafer\":[],\"winscard\":[],\"winsmcrd\":[],\"winsock2\":[],\"winspool\":[],\"winstring\":[],\"winsvc\":[],\"wintrust\":[],\"winusb\":[],\"winusbio\":[],\"winuser\":[],\"winver\":[],\"wlanapi\":[],\"wlanihv\":[],\"wlanihvtypes\":[],\"wlantypes\":[],\"wlclient\":[],\"wmistr\":[],\"wnnc\":[],\"wow64apiset\":[],\"wpdmtpextensions\":[],\"ws2bth\":[],\"ws2def\":[],\"ws2ipdef\":[],\"ws2spi\":[],\"ws2tcpip\":[],\"wtsapi32\":[],\"wtypes\":[],\"wtypesbase\":[],\"xinput\":[]}}", + "windowfunctions_0.1.1": "{\"dependencies\":[{\"name\":\"num-traits\",\"req\":\"^0.2\"}],\"features\":{}}", "windows-collections_0.3.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-core\",\"req\":\"^0.62.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"windows-strings\",\"req\":\"^0.5.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"windows-core/std\"]}}", "windows-core_0.58.0": "{\"dependencies\":[{\"name\":\"windows-implement\",\"req\":\"^0.58.0\"},{\"name\":\"windows-interface\",\"req\":\"^0.58.0\"},{\"name\":\"windows-result\",\"req\":\"^0.2.0\"},{\"name\":\"windows-strings\",\"req\":\"^0.1.0\"},{\"name\":\"windows-targets\",\"req\":\"^0.52.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "windows-core_0.62.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-implement\",\"req\":\"^0.60.2\"},{\"default_features\":false,\"name\":\"windows-interface\",\"req\":\"^0.59.3\"},{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"windows-result\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"windows-strings\",\"req\":\"^0.5.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"windows-result/std\",\"windows-strings/std\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3c921f6465..6adf98e9b2 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -844,6 +844,39 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "audio-codec-algorithms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107" + +[[package]] +name = "audioadapter" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4" + +[[package]] +name = "audioadapter-buffers" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d" +dependencies = [ + "audio-codec-algorithms", + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -4996,8 +5029,12 @@ dependencies = [ "crossbeam-queue", "futures", "libloading", + "opus", "pretty_assertions", + "rand 0.9.3", "rtc", + "rubato", + "sonora", "tempfile", "tokio", "webrtc", @@ -10720,6 +10757,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33718946cc77d4032911d4efe03a66dbcbfbd2bb16c3da06aaeadcc637c32216" +dependencies = [ + "opusic-sys", +] + +[[package]] +name = "opusic-sys" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d1ecdf206421bc74343ab3bb2f30ad2abbfee41fa341f7181fecbaf957769a" +dependencies = [ + "cmake", +] + [[package]] name = "ordered-float" version = "4.6.0" @@ -12873,6 +12928,20 @@ dependencies = [ "sansio", ] +[[package]] +name = "rubato" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-integer", + "num-traits", + "visibility", + "windowfunctions", +] + [[package]] name = "runfiles" version = "0.1.0" @@ -13931,6 +14000,79 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonora" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1ce442ab5cb392c4067b8f2cced20be4cf12e479f6dfbf44acbe9b14ac9250" +dependencies = [ + "sonora-aec3", + "sonora-agc2", + "sonora-common-audio", + "sonora-ns", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-aec3" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50280e31256d09f7838aff6715ed4b7640e514dc1092a1e14013c01e552380db" +dependencies = [ + "sonora-common-audio", + "sonora-fft", + "sonora-simd", +] + +[[package]] +name = "sonora-agc2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a93a9fa8373a33e61a7fb441d8a9e725ac6024079d2c77a845cf6abe1790b9f" +dependencies = [ + "bytemuck", + "derive_more 2.1.1", + "sonora-common-audio", + "sonora-fft", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-common-audio" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6635bc61b24807ceac93ad0807b1292b6b0620caa6cd360d3be89fc616b833cc" +dependencies = [ + "derive_more 2.1.1", + "sonora-simd", +] + +[[package]] +name = "sonora-fft" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27e272c70f34363afb5382dd1818c05f8f63ef6785635736a14e9ccbbb5ba012" + +[[package]] +name = "sonora-ns" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d511feef19302c716d245c4555087d161ba91c178735ea11107dcce0d0b7a86f" +dependencies = [ + "sonora-fft", +] + +[[package]] +name = "sonora-simd" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cf0804e4752a5562262e143014f53043080ead4cd5a98f9278d3ceaf32d868" +dependencies = [ + "cpufeatures 0.3.0", +] + [[package]] name = "sorted_vector_map" version = "0.2.1" @@ -15956,6 +16098,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vsimd" version = "0.8.0" @@ -16481,6 +16634,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.58.0" diff --git a/codex-rs/voice-host/Cargo.toml b/codex-rs/voice-host/Cargo.toml index 03328ef698..e612f17c02 100644 --- a/codex-rs/voice-host/Cargo.toml +++ b/codex-rs/voice-host/Cargo.toml @@ -16,6 +16,7 @@ codex-process-hardening = { workspace = true } codex-realtime-webrtc = { workspace = true } futures = { workspace = true } libloading = "=0.8.9" +rand = { workspace = true } rtc = "=0.20.3" tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } webrtc = "=0.20.3" @@ -23,6 +24,9 @@ webrtc = "=0.20.3" [target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"), all(windows, target_env = "msvc")))'.dependencies] cpal = "=0.18.2" crossbeam-queue = "0.3.12" +opus = "=0.4.0" +rubato = { version = "=5.0.0", default-features = false } +sonora = { version = "=0.2.0", default-features = false } [dev-dependencies] anyhow = { workspace = true } diff --git a/codex-rs/voice-host/README.md b/codex-rs/voice-host/README.md index dfe1d753b1..9c1a80dbb9 100644 --- a/codex-rs/voice-host/README.md +++ b/codex-rs/voice-host/README.md @@ -44,8 +44,14 @@ stale at one second. The queue's sample capacity must span more than that servic interval. Processing lag can still overflow a queue. Before devices open, the worker blocks on commands instead of polling. -`devicesOpened` confirms device opening only: capture/reference are drained locally -until the following media stage connects encoding, decoding and the peer. +`devicesOpened` confirms device opening only. Capture now uses Rubato resampling, +Sonora echo/noise/gain processing and 20 ms Opus encoding before sending RTP. +Mute resets retained capture history and rejects delayed pre-unmute buffers. +The receive/decode pipeline and TUI connection remain subsequent stages. + +The capture encoder uses `opus 0.4.0` and its bundled `opusic-sys` build, which +requires CMake and a C compiler. This is separate from the runtime's decoder Opus +copy; final symbol binding and packaging validation must cover both copies. Cargo Linux builds require ALSA development inputs discoverable by pkg-config (for example `libasound2-dev` on Debian/Ubuntu). Bazel uses the declared `alsa_lib` diff --git a/codex-rs/voice-host/src/audio_track.rs b/codex-rs/voice-host/src/audio_track.rs new file mode 100644 index 0000000000..2dde7d7599 --- /dev/null +++ b/codex-rs/voice-host/src/audio_track.rs @@ -0,0 +1,122 @@ +//! A single Opus RTP track. Elapsed mute time advances its clock without inventing packet loss. + +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use rtc::media::Sample; +use rtc::rtp_transceiver::rtp_sender::RTCRtpCodec; +use rtc::rtp_transceiver::rtp_sender::RTCRtpCodecParameters; +use rtc::rtp_transceiver::rtp_sender::RTCRtpCodingParameters; +use rtc::rtp_transceiver::rtp_sender::RTCRtpEncodingParameters; +use rtc::rtp_transceiver::rtp_sender::RtpCodecKind; +use webrtc::media_stream::MediaStreamTrack; +use webrtc::media_stream::track_local::static_sample::TrackLocalStaticSample; +use webrtc::peer_connection::MediaEngine; + +pub(crate) const SEND_TIMEOUT: Duration = Duration::from_millis(/*millis*/ 100); + +pub(crate) struct EncodedAudio { + pub(crate) data: Vec, + pub(crate) at: Instant, +} + +pub(crate) struct AudioTrack { + pub(crate) track: Arc, + ssrc: u32, + end: Option, +} + +impl AudioTrack { + pub(crate) fn new() -> Result<(MediaEngine, Self), &'static str> { + let ssrc = rand::random(); + let codec = RTCRtpCodec { + mime_type: "audio/opus".into(), + clock_rate: 48_000, + channels: 2, + sdp_fmtp_line: "minptime=10;useinbandfec=1".into(), + rtcp_feedback: vec![], + }; + let mut media = MediaEngine::default(); + media + .register_codec( + RTCRtpCodecParameters { + rtp_codec: codec.clone(), + payload_type: 111, + }, + RtpCodecKind::Audio, + ) + .map_err(|_| "failed to register voice codec")?; + let track = TrackLocalStaticSample::new(MediaStreamTrack::new( + "realtime".into(), + format!("audio-{ssrc}"), + "microphone".into(), + RtpCodecKind::Audio, + vec![RTCRtpEncodingParameters { + rtp_coding_parameters: RTCRtpCodingParameters { + ssrc: Some(ssrc), + ..Default::default() + }, + codec, + active: true, + ..Default::default() + }], + )) + .map_err(|_| "failed to create voice track")?; + Ok(( + media, + Self { + track: Arc::new(track), + ssrc, + end: None, + }, + )) + } + + pub(crate) async fn send(&mut self, frame: EncodedAudio) -> Result<(), &'static str> { + tokio::time::timeout(SEND_TIMEOUT, async { + let mut gap = self + .end + .map(|end| frame.at.saturating_duration_since(end)) + .unwrap_or_default(); + while !gap.is_zero() { + let duration = gap.min(Duration::from_secs(/*secs*/ 3600)); + self.track + .write_sample( + self.ssrc, + /*payload_type*/ 111, + &Sample { + duration, + ..Default::default() + }, + &[], + ) + .await + .map_err(|_| "failed to advance voice clock")?; + gap -= duration; + } + let duration = Duration::from_millis(/*millis*/ 20); + self.track + .write_sample( + self.ssrc, + /*payload_type*/ 111, + &Sample { + data: frame.data.into(), + duration, + ..Default::default() + }, + &[], + ) + .await + .map_err(|_| "failed to send voice audio")?; + self.end = Some(self.end.unwrap_or(frame.at).max(frame.at) + duration); + Ok(()) + }) + .await + .map_err(|_| "voice audio sender stalled")? + } +} + +#[cfg(test)] +#[path = "audio_track_tests.rs"] +mod tests; diff --git a/codex-rs/voice-host/src/audio_track_tests.rs b/codex-rs/voice-host/src/audio_track_tests.rs new file mode 100644 index 0000000000..1be43deac2 --- /dev/null +++ b/codex-rs/voice-host/src/audio_track_tests.rs @@ -0,0 +1,17 @@ +use super::*; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn timestamp_jitter_does_not_accumulate_but_mute_advances_the_clock() { + let (_, mut track) = AudioTrack::new().unwrap(); + let start = Instant::now(); + // Empty samples exercise the real packetizer without requiring a bound network peer. + for tick in 0..500 { + let at = start + Duration::from_millis(tick * 20 + tick % 2); + track.send(EncodedAudio { data: vec![], at }).await.unwrap(); + } + assert_eq!(track.end, Some(start + Duration::from_millis(10_001))); + let at = start + Duration::from_secs(30); + track.send(EncodedAudio { data: vec![], at }).await.unwrap(); + assert_eq!(track.end, Some(at + Duration::from_millis(20))); +} diff --git a/codex-rs/voice-host/src/capture_worker.rs b/codex-rs/voice-host/src/capture_worker.rs new file mode 100644 index 0000000000..afef3891e9 --- /dev/null +++ b/codex-rs/voice-host/src/capture_worker.rs @@ -0,0 +1,79 @@ +//! Pumps bounded device queues through capture processing and the outgoing audio track. +//! Devices retain real stream ownership; all queued and partial capture obey mute generations. +//! Retain at most one encoded batch and return between packets so controls can take priority. + +use std::collections::VecDeque; +use std::io; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Instant; + +use super::MAX_CAPTURE_AGE; +use super::buffers::Buffers; +use super::processing; + +pub(super) struct CaptureWorker { + pub(super) buffers: Arc, + pub(super) processor: processing::Processor, + pub(super) pending: VecDeque, +} + +impl CaptureWorker { + pub(super) fn set_controls( + &mut self, + controls: codex_realtime_webrtc::AudioControls, + ) -> io::Result<()> { + let previous = self.buffers.microphone.load(Ordering::Acquire); + Buffers::set_disabled(&self.buffers.microphone, controls.microphone_muted)?; + if previous != self.buffers.microphone.load(Ordering::Acquire) { + self.pending.clear(); + self.processor.reset().map_err(io::Error::other)?; + } + Buffers::set_disabled(&self.buffers.speaker, controls.speaker_suppressed) + } + + pub(super) async fn service( + &mut self, + audio: &mut crate::audio_track::AudioTrack, + now: impl Fn() -> Instant, + ) -> io::Result { + // Some backends start callbacks during stream construction, before open returns. + self.buffers.serviced.store(true, Ordering::Release); + let generation = self.buffers.microphone.load(Ordering::Acquire); + for _ in 0..self.buffers.rendered.capacity() { + let Some(frame) = self.buffers.rendered.pop() else { + break; + }; + if frame.at.elapsed() > MAX_CAPTURE_AGE { + return Err(io::Error::other("audio rendering fell behind")); + } + self.processor.render(&frame).map_err(io::Error::other)?; + } + if self.pending.is_empty() { + for _ in 0..self.buffers.capture.capacity() { + let Some(frame) = self.buffers.capture.pop() else { + break; + }; + if generation.is_multiple_of(2) && frame.generation == generation { + let frames = self.processor.capture(&frame, &now); + self.pending.extend(frames.map_err(io::Error::other)?); + } + } + } + if self.buffers.failed.load(Ordering::Acquire) { + return Err(io::Error::other("audio device failed")); + } + let Some(frame) = self.pending.pop_front() else { + return Ok(0); + }; + if now().saturating_duration_since(frame.at) > processing::MAX_PROCESSING_DELAY { + return Err(io::Error::other("voice processing fell behind")); + } + audio.send(frame).await.map_err(io::Error::other)?; + Ok(1) + } +} + +#[cfg(test)] +#[path = "capture_worker_tests.rs"] +mod tests; diff --git a/codex-rs/voice-host/src/capture_worker_tests.rs b/codex-rs/voice-host/src/capture_worker_tests.rs new file mode 100644 index 0000000000..61b55882ac --- /dev/null +++ b/codex-rs/voice-host/src/capture_worker_tests.rs @@ -0,0 +1,300 @@ +use super::super::buffers::BLOCK; +use super::super::buffers::Frame; +use super::*; +use crate::audio_track::AudioTrack; +use crate::transport::Transport; +use codex_realtime_webrtc::AudioControls; +use futures::future::BoxFuture; +use pretty_assertions::assert_eq; +use rtc::interceptor::Interceptor; +use rtc::interceptor::NoopInterceptor; +use rtc::interceptor::Packet; +use rtc::interceptor::StreamInfo; +use rtc::interceptor::TaggedPacket; +use rtc::interceptor::interceptor; +use rtc::sansio; +use rtc::shared::error::Error; +use std::time::Duration; +use tokio::sync::Notify; +use tokio::sync::mpsc; +use tokio::time::timeout; +use webrtc::peer_connection::PeerConnection; +use webrtc::peer_connection::PeerConnectionBuilder; +use webrtc::peer_connection::PeerConnectionEventHandler; +use webrtc::peer_connection::RTCIceGatheringState; +use webrtc::peer_connection::RTCSessionDescription; + +// Consume remote RTP before the upstream track queue: receipt proves the packet +// traversed negotiation, encryption, the loopback socket, and remote decryption. +#[derive(Interceptor)] +struct RemoteRtp { + #[next] + next: NoopInterceptor, + sender: mpsc::Sender, +} + +#[interceptor] +impl RemoteRtp { + #[overrides] + fn handle_read(&mut self, message: TaggedPacket) -> Result<(), Self::Error> { + if let Packet::Rtp(packet) = message.message { + self.sender.try_send(packet).unwrap(); + } + Ok(()) + } +} + +struct Gathered(Arc); + +impl PeerConnectionEventHandler for Gathered { + fn on_ice_gathering_state_change<'a, 'async_trait>( + &'a self, + state: RTCIceGatheringState, + ) -> BoxFuture<'async_trait, ()> + where + 'a: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + if state == RTCIceGatheringState::Complete { + self.0.notify_one(); + } + }) + } +} + +// Synthetic callback output enters the same bounded queue as the real cpal +// callback. No hardware availability or physical microphone behavior is claimed. +fn enqueue(worker: &CaptureWorker, samples: usize, generation: u64, at: Instant) { + for offset in (0..samples).step_by(BLOCK) { + let mut frame = Frame { + samples: [0.0; BLOCK], + len: BLOCK.min(samples - offset), + at: at + Duration::from_secs_f64(offset as f64 / 48_000.0), + generation, + }; + for (index, sample) in frame.samples.iter_mut().enumerate() { + *sample = + (std::f32::consts::TAU * 440.0 * (offset + index) as f32 / 48_000.0).sin() * 0.25; + } + assert!(worker.buffers.capture.push(frame).is_ok()); + } +} + +async fn receive_capture( + worker: &mut CaptureWorker, + local: &mut Transport, + received: &mut mpsc::Receiver, + now: Instant, +) -> Vec { + let mut count = 0; + loop { + let sent = worker.service(&mut local.audio, || now).await.unwrap(); + if sent == 0 { + break; + } + assert_eq!(sent, 1); + count += sent; + } + assert!(count > 0, "fresh capture must produce audio"); + let mut packets = Vec::new(); + let mut decoder = opus::Decoder::new(/*sample_rate*/ 48_000, opus::Channels::Mono).unwrap(); + let mut energy = 0.0_f32; + for _ in 0..count { + let packet = received.recv().await.unwrap(); + assert_eq!(packet.header.payload_type, 111); + let mut decoded = [0.0; 960]; + assert_eq!( + decoder + .decode_float(&packet.payload, &mut decoded, /*fec*/ false) + .unwrap(), + 960 + ); + assert!(decoded.iter().all(|sample| sample.is_finite())); + energy += decoded.iter().map(|sample| sample * sample).sum::(); + packets.push(packet); + } + assert!( + energy > 0.01, + "decoded microphone signal must not be silence" + ); + packets +} + +#[tokio::test] +async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio() { + timeout(Duration::from_secs(/*secs*/ 30), async { + let (sender, mut received) = mpsc::channel(/*buffer*/ 32); + let gathered = Arc::new(Notify::new()); + let (media, _) = AudioTrack::new().unwrap(); + let mut settings = webrtc::peer_connection::SettingEngine::default(); + settings.set_lite(/*lite*/ true); + let remote = PeerConnectionBuilder::new() + .with_media_engine(media) + .with_setting_engine(settings) + .with_handler(Arc::new(Gathered(gathered.clone()))) + .with_interceptor_registry(rtc::interceptor::Registry::from(RemoteRtp { + next: NoopInterceptor::new(), + sender, + })) + .with_udp_addrs(vec!["0.0.0.0:0"]) + .build() + .await + .unwrap(); + let mut local = Transport::new().await.unwrap(); + remote + .set_remote_description( + RTCSessionDescription::offer(local.offer().await.unwrap()).unwrap(), + ) + .await + .unwrap(); + let answer = remote.create_answer(/*options*/ None).await.unwrap(); + remote.set_local_description(answer).await.unwrap(); + gathered.notified().await; + local + .apply_answer(remote.local_description().await.unwrap().sdp) + .await + .unwrap(); + + let mut worker = CaptureWorker { + buffers: Arc::new(Buffers::new( + /*input_rate*/ 48_000, /*output_rate*/ 48_000, + )), + pending: Default::default(), + processor: processing::Processor::new( + /*input_rate*/ 48_000, /*output_rate*/ 44_100, + ) + .unwrap(), + }; + let unmuted = || AudioControls { + microphone_muted: false, + speaker_suppressed: false, + }; + let muted = || AudioControls { + microphone_muted: true, + speaker_suppressed: true, + }; + worker.set_controls(unmuted()).unwrap(); + let generation = worker.buffers.microphone.load(Ordering::Acquire); + // Freeze processing time so this transport test does not benchmark the CI worker. + // Processor tests separately exercise the real backlog cutoff. + let start = Instant::now(); + let mut now = start; + // Feed a distinct speaker tone through the real render resampler and APM + // before capture; this exercises a different output rate without hardware. + for offset in (0..2_646).step_by(BLOCK) { + let mut frame = Frame { + samples: [0.0; BLOCK], + len: BLOCK.min(2_646 - offset), + at: start + Duration::from_secs_f64(offset as f64 / 44_100.0), + generation: 0, + }; + for (index, sample) in frame.samples.iter_mut().enumerate() { + *sample = (std::f32::consts::TAU * 220.0 * (offset + index) as f32 / 44_100.0) + .sin() + * 0.1; + } + assert!(worker.buffers.rendered.push(frame).is_ok()); + } + enqueue(&worker, /*samples*/ 2_880, generation, now); + let before = receive_capture(&mut worker, &mut local, &mut received, now).await; + assert!(worker.buffers.rendered.is_empty()); + + // Start with a clean processor, then retain less than one Opus packet. + // If either resampler or partial encoder history survives the next mute, + // the equally short fresh segment below would emit audio too early. + worker.set_controls(muted()).unwrap(); + worker.set_controls(unmuted()).unwrap(); + let old_generation = worker.buffers.microphone.load(Ordering::Acquire); + now = Instant::now(); + enqueue(&worker, /*samples*/ 720, old_generation, now); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + enqueue(&worker, /*samples*/ 2_880, old_generation, now); + worker.set_controls(muted()).unwrap(); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + + // A callback that began before mute may enqueue its old generation only + // after unmute; a newly started callback may still carry an old device + // timestamp. Exercise both independently through the actual service loop. + let pre_unmute = Instant::now() - Duration::from_millis(/*millis*/ 100); + worker.set_controls(unmuted()).unwrap(); + let generation = worker.buffers.microphone.load(Ordering::Acquire); + let resumed = Instant::now() + Duration::from_secs(/*secs*/ 1); + now = resumed; + enqueue(&worker, /*samples*/ 2_880, old_generation, now); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + enqueue(&worker, /*samples*/ 2_880, generation, pre_unmute); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + enqueue(&worker, /*samples*/ 720, generation, now); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + + now += Duration::from_millis(/*millis*/ 15); + enqueue(&worker, /*samples*/ 2_880, generation, now); + let after = receive_capture(&mut worker, &mut local, &mut received, now).await; + let clock_gap = after[0] + .header + .timestamp + .wrapping_sub(before[0].header.timestamp); + let expected_gap = (resumed.duration_since(start).as_secs_f64() * 48_000.0) as u32; + // Allow 1 ms for sample rounding without imposing a wall-clock throughput requirement. + assert!( + clock_gap.abs_diff(expected_gap) <= 48, + "received RTP clock gap: {clock_gap}, expected {expected_gap}" + ); + assert_eq!( + after[0].header.sequence_number, + before + .last() + .unwrap() + .header + .sequence_number + .wrapping_add(1) + ); + assert!(received.try_recv().is_err()); + + // A mute received during a send must discard the rest of that encoded + // batch before the next service call, including after a quick unmute. + worker.set_controls(muted()).unwrap(); + worker.set_controls(unmuted()).unwrap(); + let generation = worker.buffers.microphone.load(Ordering::Acquire); + now = Instant::now() + Duration::from_secs(/*secs*/ 1); + enqueue(&worker, /*samples*/ 2_880, generation, now); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 1); + received.recv().await.unwrap(); + assert!(!worker.pending.is_empty()); + let pending = worker.pending.len(); + worker + .set_controls(AudioControls { + microphone_muted: false, + speaker_suppressed: true, + }) + .unwrap(); + assert_eq!(worker.pending.len(), pending); + worker.set_controls(muted()).unwrap(); + worker.set_controls(unmuted()).unwrap(); + assert!(worker.pending.is_empty()); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0); + assert!(received.try_recv().is_err()); + + // Waiting behind a slow send cannot extend the capture freshness limit. + let generation = worker.buffers.microphone.load(Ordering::Acquire); + now = Instant::now() + Duration::from_secs(/*secs*/ 1); + enqueue(&worker, /*samples*/ 2_880, generation, now); + assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 1); + received.recv().await.unwrap(); + let stale = now + Duration::from_secs(/*secs*/ 1); + assert_eq!( + worker + .service(&mut local.audio, || stale) + .await + .unwrap_err() + .to_string(), + "voice processing fell behind" + ); + assert!(received.try_recv().is_err()); + local.close().await.unwrap(); + remote.close().await.unwrap(); + }) + .await + .unwrap(); +} diff --git a/codex-rs/voice-host/src/device_buffers.rs b/codex-rs/voice-host/src/device_buffers.rs index 6ff6e3ace5..97b15198c8 100644 --- a/codex-rs/voice-host/src/device_buffers.rs +++ b/codex-rs/voice-host/src/device_buffers.rs @@ -1,5 +1,6 @@ //! Preallocated callback buffers pack small callbacks into full queue slots without locks. -//! Generation changes and rejected capture buffers discard incomplete frames. +//! Generation changes, rejected capture buffers and capture gaps discard incomplete frames. +//! Capture/render capacity covers the worker deadline; playback and callback limits stay fixed. use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; @@ -11,6 +12,9 @@ use crossbeam_queue::ArrayQueue; pub(super) const BLOCK: usize = 256; pub(super) const QUEUE_CAPACITY: usize = 32; +pub(super) const MAX_CALLBACK_FRAMES: usize = BLOCK * QUEUE_CAPACITY; +// Allow ordinary timestamp jitter consistently at both capture buffering boundaries. +pub(super) const CAPTURE_GAP_TOLERANCE: Duration = Duration::from_millis(/*millis*/ 20); pub(super) struct Frame { pub(super) samples: [f32; BLOCK], @@ -31,6 +35,17 @@ impl FramePacker { self.frame = None; } + // Capture checks continuity before packing can hide a gap inside an old partial block. + // Render callbacks retain their separate existing packing behavior. + pub(super) fn discard_capture_gap(&mut self, start: Instant, rate: f64) { + if self.frame.as_ref().is_some_and(|partial| { + let end = partial.at + Duration::from_secs_f64(partial.len as f64 / rate); + start.saturating_duration_since(end) > CAPTURE_GAP_TOLERANCE + }) { + self.reset(); + } + } + pub(super) fn push(&mut self, frame: Frame, rate: f64, queue: &ArrayQueue) -> bool { if self .frame @@ -80,10 +95,19 @@ pub(super) struct Buffers { } impl Buffers { - pub(super) fn new() -> Self { + pub(super) fn new(input_rate: u32, output_rate: u32) -> Self { + // Rates are validated before opening devices. A pending batch can defer + // capture consumption until its freshness deadline plus one in-flight send. + let budget = super::processing::MAX_PROCESSING_DELAY + + crate::audio_track::SEND_TIMEOUT + + 2 * crate::DEVICE_SERVICE_INTERVAL; + let capacity = |rate: u32| { + let samples = (u128::from(rate) * budget.as_nanos()).div_ceil(1_000_000_000); + (samples as usize + MAX_CALLBACK_FRAMES + BLOCK).div_ceil(BLOCK) + }; Self { - capture: ArrayQueue::new(QUEUE_CAPACITY), - rendered: ArrayQueue::new(QUEUE_CAPACITY), + capture: ArrayQueue::new(capacity(input_rate)), + rendered: ArrayQueue::new(capacity(output_rate)), playback: ArrayQueue::new(QUEUE_CAPACITY), microphone: AtomicU64::new(/*v*/ 1), speaker: AtomicU64::new(/*v*/ 1), diff --git a/codex-rs/voice-host/src/device_buffers_tests.rs b/codex-rs/voice-host/src/device_buffers_tests.rs index b81e38385d..c1244d7d45 100644 --- a/codex-rs/voice-host/src/device_buffers_tests.rs +++ b/codex-rs/voice-host/src/device_buffers_tests.rs @@ -41,7 +41,7 @@ fn unmute_rejects_backlog_until_capture_reaches_the_following_callback() { #[test] fn suppression_discards_partial_and_queued_previous_generations() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000); let mut playback = Playback::default(); Buffers::set_disabled(&buffers.speaker, /*disabled*/ false).unwrap(); let generation = buffers.speaker.load(Ordering::Acquire); @@ -68,7 +68,7 @@ fn suppression_discards_partial_and_queued_previous_generations() { #[test] fn output_is_finite_and_bounded_and_underflow_is_silence() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000); Buffers::set_disabled(&buffers.speaker, /*disabled*/ false).unwrap(); let generation = buffers.speaker.load(Ordering::Acquire); let mut samples = [0.0; BLOCK]; @@ -91,7 +91,7 @@ fn output_is_finite_and_bounded_and_underflow_is_silence() { #[test] fn repeated_controls_preserve_epoch_but_transitions_invalidate_capture() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000); Buffers::set_disabled(&buffers.microphone, /*disabled*/ false).unwrap(); let first = buffers.microphone.load(Ordering::Acquire); Buffers::set_disabled(&buffers.microphone, /*disabled*/ false).unwrap(); @@ -103,7 +103,7 @@ fn repeated_controls_preserve_epoch_but_transitions_invalidate_capture() { #[test] fn tiny_callbacks_share_slots_and_preserve_the_oldest_timestamp() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000); let start = Instant::now(); for queue in [&buffers.capture, &buffers.rendered] { let mut packer = FramePacker::default(); @@ -210,3 +210,43 @@ fn packing_preserves_a_remainder_timestamp_across_uneven_callbacks() { ); assert!(queue.is_empty()); } + +#[test] +fn capture_queue_retains_high_rate_audio_during_bounded_service_pause() { + let buffers = Buffers::new(/*input_rate*/ 384_000, /*output_rate*/ 48_000); + let mut packer = FramePacker::default(); + let start = Instant::now(); + // 630 ms covers a pending batch's 500 ms deadline, an in-flight 100 ms + // send and callback/service margins. Consumption is deliberately paused. + let samples = 384_000 * 63 / 100; + for offset in (0..samples).step_by(17) { + assert!(packer.push( + Frame { + samples: [0.25; BLOCK], + len: (samples - offset).min(17), + at: start + Duration::from_secs_f64(offset as f64 / 384_000.0), + generation: 2, + }, + /*rate*/ 384_000.0, + &buffers.capture, + )); + } + assert_eq!(buffers.capture.len(), samples / BLOCK); + // Enlarging the queue must not turn overflow into unbounded accumulation. + for _ in buffers.capture.len()..=buffers.capture.capacity() { + let available = buffers.capture.len() < buffers.capture.capacity(); + assert_eq!( + packer.push( + Frame { + samples: [0.25; BLOCK], + len: BLOCK, + at: start, + generation: 2, + }, + /*rate*/ 384_000.0, + &buffers.capture, + ), + available, + ); + } +} diff --git a/codex-rs/voice-host/src/devices.rs b/codex-rs/voice-host/src/devices.rs index e217881ec4..ede1283af1 100644 --- a/codex-rs/voice-host/src/devices.rs +++ b/codex-rs/voice-host/src/devices.rs @@ -5,6 +5,10 @@ #[path = "device_buffers.rs"] mod buffers; +#[path = "capture_worker.rs"] +mod capture_worker; +#[path = "processing.rs"] +mod processing; use std::io; use std::sync::Arc; @@ -25,6 +29,7 @@ use buffers::Buffers; use buffers::CaptureBoundary; use buffers::Frame; use buffers::FramePacker; +use buffers::MAX_CALLBACK_FRAMES; use buffers::Playback; use buffers::QUEUE_CAPACITY; @@ -33,7 +38,7 @@ const MAX_CAPTURE_AGE: Duration = Duration::from_secs(/*secs*/ 1); pub(super) struct Devices { _input: cpal::Stream, _output: cpal::Stream, - buffers: Arc, + worker: capture_worker::CaptureWorker, } macro_rules! stream { @@ -79,7 +84,22 @@ impl Devices { } let input_stream_config = bounded_stream_config(&input_config)?; let output_stream_config = bounded_stream_config(&output_config)?; - let buffers = Arc::new(Buffers::new()); + let buffers = Arc::new(Buffers::new( + input_config.sample_rate(), + output_config.sample_rate(), + )); + let processor = + processing::Processor::new(input_config.sample_rate(), output_config.sample_rate()) + .map_err(io::Error::other)?; + let (cpal::BufferSize::Fixed(input_frames), cpal::BufferSize::Fixed(output_frames)) = ( + input_stream_config.buffer_size, + output_stream_config.buffer_size, + ) else { + return Err(io::Error::other("audio callback size unavailable")); + }; + processor + .validate_callback_timing(input_frames, output_frames) + .map_err(io::Error::other)?; let input = stream!( input_config.sample_format(), build_input, @@ -105,37 +125,26 @@ impl Devices { Ok(Self { _input: input, _output: output, - buffers, + worker: capture_worker::CaptureWorker { + buffers, + processor, + pending: Default::default(), + }, }) } pub(super) fn set_controls( - &self, + &mut self, controls: codex_realtime_webrtc::AudioControls, ) -> io::Result<()> { - Buffers::set_disabled(&self.buffers.microphone, controls.microphone_muted)?; - Buffers::set_disabled(&self.buffers.speaker, controls.speaker_suppressed) + self.worker.set_controls(controls) } - pub(super) fn service(&self) -> io::Result<()> { - // Some backends start callbacks during stream construction, before open returns. - self.buffers.serviced.store(true, Ordering::Release); - // Device-only stage: consume locally until the media worker is attached. - // Reading timestamps here also rejects a stalled callback before readiness is reused. - for queue in [&self.buffers.capture, &self.buffers.rendered] { - for _ in 0..queue.capacity() { - let Some(frame) = queue.pop() else { - break; - }; - if frame.at.elapsed() > MAX_CAPTURE_AGE { - return Err(io::Error::other("audio device processing fell behind")); - } - } - } - if self.buffers.failed.load(Ordering::Acquire) { - return Err(io::Error::other("audio device failed")); - } - Ok(()) + pub(super) async fn service( + &mut self, + audio: &mut crate::audio_track::AudioTrack, + ) -> io::Result { + self.worker.service(audio, Instant::now).await } } @@ -146,13 +155,8 @@ fn bounded_stream_config( return Err(io::Error::other("audio callback size range unavailable")); }; let mut config = supported.config(); - // Leave room for a partial block and smaller callbacks arriving before service. - let service_frames = (f64::from(config.sample_rate) - * super::DEVICE_SERVICE_INTERVAL.as_secs_f64()) - .ceil() as usize; - let callback_capacity = (BLOCK * QUEUE_CAPACITY).saturating_sub(service_frames + BLOCK); let min = min.max(1); - let max = max.min(callback_capacity as u32); + let max = max.min(MAX_CALLBACK_FRAMES as u32); if min > max { return Err(io::Error::other("unsupported audio callback size range")); } @@ -225,6 +229,7 @@ where .unwrap_or_default(), ) .unwrap_or_else(Instant::now); + capture.discard_capture_gap(start, rate); for (index, chunk) in data.chunks(BLOCK * channels).enumerate() { let mut frame = Frame { samples: [0.0; BLOCK], diff --git a/codex-rs/voice-host/src/devices_tests.rs b/codex-rs/voice-host/src/devices_tests.rs index 5a0235e4c4..0f9752f40e 100644 --- a/codex-rs/voice-host/src/devices_tests.rs +++ b/codex-rs/voice-host/src/devices_tests.rs @@ -3,7 +3,7 @@ use pretty_assertions::assert_eq; #[test] fn startup_output_waits_for_service_but_later_overflow_still_fails() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000); let mut state = OutputState::default(); let mut output = [1.0_f32; BLOCK * 2]; let start = Instant::now(); @@ -46,7 +46,7 @@ fn startup_output_waits_for_service_but_later_overflow_still_fails() { ), ); Buffers::set_disabled(&buffers.speaker, /*disabled*/ false).unwrap(); - for _ in 0..QUEUE_CAPACITY { + for _ in 0..buffers.rendered.capacity() { render_output( &mut output, /*channels*/ 2, @@ -96,7 +96,7 @@ fn callback_configuration_fits_supported_range_and_actual_queue() { buffer_size: cpal::BufferSize::Fixed(frames), } ); - let buffers = Buffers::new(); + let buffers = Buffers::new(rate, rate); for queue in [&buffers.capture, &buffers.rendered] { for offset in (0..frames as usize).step_by(BLOCK) { assert!( @@ -115,23 +115,20 @@ fn callback_configuration_fits_supported_range_and_actual_queue() { } #[test] -fn callback_configuration_reserves_room_for_partial_and_following_callbacks() { - for min in [6_016, 6_017, 8_192] { +fn callback_configuration_fits_dynamic_queue_with_following_callbacks() { + for min in [6_016, 8_192] { let supported = cpal::SupportedStreamConfig::new( /*channels*/ 1, 384_000, cpal::SupportedBufferSize::Range { min, max: 16_384 }, cpal::SampleFormat::F32, ); - let config = bounded_stream_config(&supported); - if min != 6_016 { - assert!(config.is_err(), "minimum callback size {min}"); - continue; - } - let cpal::BufferSize::Fixed(frames) = config.unwrap().buffer_size else { + let cpal::BufferSize::Fixed(frames) = + bounded_stream_config(&supported).unwrap().buffer_size + else { panic!("callback size must be bounded"); }; - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000); buffers.serviced.store(true, Ordering::Release); let mut output = OutputState::default(); let start = Instant::now(); @@ -146,7 +143,11 @@ fn callback_configuration_reserves_room_for_partial_and_following_callbacks() { &mut output, ); } - assert_eq!(buffers.rendered.len(), QUEUE_CAPACITY - 1); + assert_eq!( + buffers.rendered.len(), + (BLOCK - 1 + frames as usize + 15 * 128) / BLOCK, + ); + assert!(buffers.rendered.len() < buffers.rendered.capacity()); assert!(!buffers.failed.load(Ordering::Acquire)); } } @@ -191,9 +192,41 @@ fn callback_timing_must_fit_service_and_stale_limits() { } } +#[test] +fn callback_pair_must_fit_the_processing_deadline() { + for (input_rate, input_frames, output_rate, output_frames, accepted) in [ + (8_000, 7_680, 48_000, 480, false), + (8_000, 2_000, 8_000, 2_000, false), + (48_000, 480, 48_000, 480, true), + (44_100, 441, 48_000, 480, true), + (8_000, 1_600, 48_000, 480, true), + ] { + for (rate, frames) in [(input_rate, input_frames), (output_rate, output_frames)] { + let supported = cpal::SupportedStreamConfig::new( + /*channels*/ 1, + rate, + cpal::SupportedBufferSize::Range { + min: frames, + max: frames, + }, + cpal::SampleFormat::F32, + ); + assert!(bounded_stream_config(&supported).is_ok()); + } + let processor = processing::Processor::new(input_rate, output_rate).unwrap(); + assert_eq!( + processor + .validate_callback_timing(input_frames, output_frames) + .is_ok(), + accepted, + "input={input_rate}/{input_frames}, output={output_rate}/{output_frames}", + ); + } +} + #[test] fn tiny_output_callbacks_pack_references_without_delaying_rendering() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000); buffers.serviced.store(true, Ordering::Release); Buffers::set_disabled(&buffers.speaker, /*disabled*/ false).unwrap(); let generation = buffers.speaker.load(Ordering::Acquire); @@ -228,13 +261,37 @@ fn tiny_output_callbacks_pack_references_without_delaying_rendering() { assert!(!buffers.failed.load(Ordering::Acquire)); } +#[test] +fn rendered_queue_retains_high_rate_callbacks_during_bounded_service_pause() { + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000); + buffers.serviced.store(true, Ordering::Release); + let mut state = OutputState::default(); + let start = Instant::now(); + let samples = 384_000 * 63 / 100; + for offset in (0..samples).step_by(17) { + let mut output = [1.0_f32; 34]; + let len = (samples - offset).min(17) * 2; + render_output( + &mut output[..len], + /*channels*/ 2, + /*rate*/ 384_000.0, + start + Duration::from_secs_f64(offset as f64 / 384_000.0), + &buffers, + &mut state, + ); + assert_eq!(&output[..len], &[0.0; 34][..len]); + assert!(!buffers.failed.load(Ordering::Acquire)); + } + assert_eq!(buffers.rendered.len(), samples / BLOCK); +} + #[test] fn suppressed_output_does_not_fail_when_reference_queue_is_full() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000); buffers.serviced.store(true, Ordering::Release); let mut state = OutputState::default(); let mut output = [1.0_f32; BLOCK]; - for _ in 0..QUEUE_CAPACITY * 3 { + for _ in 0..buffers.rendered.capacity() * 3 { render_output( &mut output, /*channels*/ 1, @@ -245,7 +302,7 @@ fn suppressed_output_does_not_fail_when_reference_queue_is_full() { ); assert_eq!(output, [0.0; BLOCK]); } - assert_eq!(buffers.rendered.len(), QUEUE_CAPACITY); + assert_eq!(buffers.rendered.len(), buffers.rendered.capacity()); assert!(!buffers.failed.load(Ordering::Acquire)); Buffers::set_disabled(&buffers.speaker, /*disabled*/ false).unwrap(); render_output( @@ -261,7 +318,7 @@ fn suppressed_output_does_not_fail_when_reference_queue_is_full() { #[test] fn stream_xruns_recover_without_clearing_device_failure() { - let buffers = Buffers::new(); + let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000); handle_stream_error(&buffers, cpal::ErrorKind::Xrun.into()); assert!(!buffers.failed.load(Ordering::Acquire)); handle_stream_error(&buffers, cpal::ErrorKind::DeviceNotAvailable.into()); diff --git a/codex-rs/voice-host/src/devices_unavailable.rs b/codex-rs/voice-host/src/devices_unavailable.rs index 711dd0b817..5a59b6ef3f 100644 --- a/codex-rs/voice-host/src/devices_unavailable.rs +++ b/codex-rs/voice-host/src/devices_unavailable.rs @@ -15,7 +15,10 @@ impl Devices { Self::open().map(|_| ()) } - pub(super) fn service(&self) -> io::Result<()> { - Self::open().map(|_| ()) + pub(super) async fn service( + &mut self, + _: &mut crate::audio_track::AudioTrack, + ) -> io::Result { + Self::open().map(|_| 0) } } diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs index 0eb6339414..1d1b7a3470 100644 --- a/codex-rs/voice-host/src/main.rs +++ b/codex-rs/voice-host/src/main.rs @@ -1,5 +1,16 @@ //! Same-build helper lifecycle with privately owned runtime, transport and opt-in local devices. +//! Queued privacy controls take priority over starting another capture batch. +// Non-native helper targets negotiate transport but have no device backend to send audio. +#[cfg_attr( + not(any( + target_os = "macos", + all(target_os = "linux", target_env = "gnu"), + all(windows, target_env = "msvc") + )), + allow(dead_code) +)] +mod audio_track; #[cfg_attr( not(any( target_os = "macos", @@ -89,13 +100,19 @@ fn run( output.flush()?; let mut runtime = None; let executor = tokio::runtime::Runtime::new()?; - let mut transport = None; + let mut transport: Option = None; let mut answered = false; let mut devices: Option = None; loop { - let message = if let Some(devices) = &devices { - devices.service()?; - receiver.recv_timeout(DEVICE_SERVICE_INTERVAL) + let message = if let Some(devices) = &mut devices { + let peer = transport + .as_mut() + .ok_or_else(|| io::Error::other("voice peer not started"))?; + wait_for_control(&receiver, || { + executor.block_on(devices.service(&mut peer.audio))?; + Ok(()) + })? + .ok_or(mpsc::RecvTimeoutError::Disconnected) } else { receiver .recv() @@ -108,7 +125,7 @@ fn run( } Ok(Message::SetAudioControls { controls }) => { devices - .as_ref() + .as_mut() .ok_or_else(|| io::Error::other("audio devices not open"))? .set_controls(controls)?; Message::AudioControlsApplied {} @@ -167,6 +184,21 @@ fn run( } } +// Pending privacy controls and shutdown always precede the next capture batch. +// A command arriving after service begins cannot retract an already in-flight packet. +fn wait_for_control( + receiver: &mpsc::Receiver, + mut service: impl FnMut() -> io::Result<()>, +) -> io::Result> { + loop { + match receiver.recv_timeout(DEVICE_SERVICE_INTERVAL) { + Ok(message) => return Ok(Some(message)), + Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(None), + Err(mpsc::RecvTimeoutError::Timeout) => service()?, + } + } +} + #[cfg(test)] #[path = "main_tests.rs"] mod tests; diff --git a/codex-rs/voice-host/src/main_tests.rs b/codex-rs/voice-host/src/main_tests.rs index bd5074c671..eeeb797b34 100644 --- a/codex-rs/voice-host/src/main_tests.rs +++ b/codex-rs/voice-host/src/main_tests.rs @@ -1,4 +1,4 @@ -//! Verify the real parent-loss watchdog while the startup worker is deterministically blocked. +//! Verify control-first capture servicing and the real parent-loss watchdog during blocked startup. use super::*; use std::process::Stdio; @@ -12,6 +12,48 @@ use tokio::time::timeout; const STARTUP_BLOCKED: &[u8] = b"voice startup blocked\n"; const CHILD_ENV: &str = "CODEX_VOICE_WATCHDOG_TEST_CHILD"; +#[test] +fn queued_privacy_controls_and_close_precede_capture_service() { + for message in [ + Message::SetAudioControls { + controls: codex_realtime_webrtc::AudioControls { + microphone_muted: true, + speaker_suppressed: true, + }, + }, + Message::Close {}, + ] { + let expected = encode_frame(&message).unwrap(); + let (sender, receiver) = mpsc::sync_channel(/*bound*/ 1); + sender.send(message).unwrap(); + let next = wait_for_control(&receiver, || panic!("capture preceded queued control")) + .unwrap() + .unwrap(); + assert_eq!(encode_frame(&next).unwrap(), expected); + drop(sender); + assert!( + wait_for_control(&receiver, || panic!("capture after parent loss")) + .unwrap() + .is_none() + ); + } +} + +#[test] +fn idle_service_rechecks_controls_before_another_batch() { + let (sender, receiver) = mpsc::sync_channel(/*bound*/ 1); + let mut services = 0; + let next = wait_for_control(&receiver, || { + services += 1; + sender.send(Message::Close {}).unwrap(); + Ok(()) + }) + .unwrap() + .unwrap(); + assert_eq!(next, Message::Close {}); + assert_eq!(services, 1); +} + #[test] #[ignore = "subprocess fixture for the parent-loss watchdog test"] fn blocked_startup_fixture() { diff --git a/codex-rs/voice-host/src/processing.rs b/codex-rs/voice-host/src/processing.rs new file mode 100644 index 0000000000..bc8c16bce8 --- /dev/null +++ b/codex-rs/voice-host/src/processing.rs @@ -0,0 +1,251 @@ +//! Streaming device-rate conversion, echo/noise/gain processing, and 20 ms Opus encoding. +//! Mute transitions and capture gaps reset history; delayed pre-unmute buffers are discarded. + +use std::collections::VecDeque; +use std::time::Duration; +use std::time::Instant; + +use rubato::Resampler; +use rubato::audioadapter_buffers::direct::InterleavedSlice; +use sonora::AudioProcessing; +use sonora::StreamConfig; +use sonora::config::AdaptiveDigital; +use sonora::config::EchoCanceller; +use sonora::config::GainController2; +use sonora::config::NoiseSuppression; + +use super::buffers::CAPTURE_GAP_TOLERANCE; +use super::buffers::Frame; +use crate::audio_track::EncodedAudio; + +type Result = std::result::Result; +const BLOCK: usize = 480; +pub(super) const MAX_PROCESSING_DELAY: Duration = Duration::from_millis(/*millis*/ 500); + +struct Converter { + resampler: rubato::Async, + input: VecDeque, + output: VecDeque, + scratch: Vec, + rate: u32, + end: Option, +} + +impl Converter { + fn new(rate: u32) -> Result { + let resampler = rubato::Async::new_sinc( + 48_000.0 / f64::from(rate), + /*max_resample_ratio_relative*/ 1.0, + &rubato::SincInterpolationParameters::default(), + rate.div_ceil(100) as usize, + /*nbr_channels*/ 1, + rubato::FixedAsync::Input, + ) + .map_err(|_| "failed to create voice resampler")?; + let scratch = vec![0.0; resampler.output_frames_max()]; + Ok(Self { + resampler, + input: VecDeque::new(), + output: VecDeque::new(), + scratch, + rate, + end: None, + }) + } + + fn push(&mut self, frame: &Frame) -> Result<()> { + if frame.len > frame.samples.len() + || self.input.len() + frame.len > self.rate as usize + || self.output.len() > 48_000 + { + return Err("voice resampler backlog exceeded"); + } + self.end = + Some(frame.at + Duration::from_secs_f64(frame.len as f64 / f64::from(self.rate))); + self.input.extend(&frame.samples[..frame.len]); + while self.input.len() >= self.resampler.input_frames_next() { + let samples = self.input.make_contiguous(); + let input = InterleavedSlice::new(samples, /*channels*/ 1, samples.len()) + .map_err(|_| "invalid voice resampler input")?; + let capacity = self.scratch.len(); + let mut output = + InterleavedSlice::new_mut(&mut self.scratch, /*channels*/ 1, capacity) + .map_err(|_| "invalid voice resampler output")?; + let (consumed, produced) = self + .resampler + .process_into_buffer(&input, &mut output, /*indexing*/ None) + .map_err(|_| "failed to resample voice audio")?; + self.input.drain(..consumed); + self.output.extend(&self.scratch[..produced]); + } + Ok(()) + } + + fn next(&mut self) -> Option<(Instant, [f32; BLOCK])> { + if self.output.len() < BLOCK { + return None; + } + let delay = self.input.len() as f64 / f64::from(self.rate) + + (self.output.len() + self.resampler.output_delay()) as f64 / 48_000.0; + let end = self.end?; + let at = end + .checked_sub(Duration::from_secs_f64(delay)) + .unwrap_or(end); + let mut output = [0.0; BLOCK]; + for (out, sample) in output.iter_mut().zip(self.output.drain(..BLOCK)) { + *out = sample; + } + Some((at, output)) + } +} + +pub(super) struct Processor { + capture: Converter, + render: Converter, + apm: AudioProcessing, + encoder: opus::Encoder, + pending: Vec, + at: Instant, + cutoff: Instant, + render_delay: i64, +} + +impl Processor { + pub(super) fn new(input_rate: u32, output_rate: u32) -> Result { + Ok(Self { + capture: Converter::new(input_rate)?, + render: Converter::new(output_rate)?, + apm: AudioProcessing::builder() + .config(sonora::Config { + echo_canceller: Some(EchoCanceller::default()), + noise_suppression: Some(NoiseSuppression::default()), + gain_controller2: Some(GainController2 { + adaptive_digital: Some(AdaptiveDigital::default()), + ..Default::default() + }), + ..Default::default() + }) + .capture_config(StreamConfig::new( + /*sample_rate_hz*/ 48_000, /*num_channels*/ 1, + )) + .render_config(StreamConfig::new( + /*sample_rate_hz*/ 48_000, /*num_channels*/ 1, + )) + .build(), + encoder: opus::Encoder::new( + /*sample_rate*/ 48_000, + opus::Channels::Mono, + opus::Application::Voip, + ) + .map_err(|_| "failed to create voice encoder")?, + pending: Vec::with_capacity(960), + at: Instant::now(), + cutoff: Instant::now(), + render_delay: 0, + }) + } + + pub(super) fn reset(&mut self) -> Result<()> { + *self = Self::new(self.capture.rate, self.render.rate)?; + Ok(()) + } + + pub(super) fn validate_callback_timing( + &self, + input_frames: u32, + output_frames: u32, + ) -> Result<()> { + let mut delay = super::super::DEVICE_SERVICE_INTERVAL; + for (converter, frames) in [(&self.capture, input_frames), (&self.render, output_frames)] { + // Include callback and queue packing latency, plus the resampler's + // actual delay and one processing block, for both device directions. + delay += Duration::from_secs_f64( + (f64::from(frames) + super::buffers::BLOCK as f64) / f64::from(converter.rate) + + (converter.resampler.output_delay() + BLOCK) as f64 / 48_000.0, + ); + } + if delay >= MAX_PROCESSING_DELAY { + return Err("audio callback timing exceeds processing deadline"); + } + Ok(()) + } + + pub(super) fn render(&mut self, frame: &Frame) -> Result<()> { + self.render.push(frame)?; + while let Some((at, input)) = self.render.next() { + let mut output = [0.0; BLOCK]; + self.apm + .process_render_f32(&[&input], &mut [&mut output]) + .map_err(|_| "voice echo reference failed")?; + let now = Instant::now(); + self.render_delay = if at > now { + at.duration_since(now).as_millis() as i64 + } else { + -(now.duration_since(at).as_millis() as i64) + }; + } + Ok(()) + } + + pub(super) fn capture( + &mut self, + frame: &Frame, + now: impl Fn() -> Instant, + ) -> Result> { + if frame.at < self.cutoff { + return Ok(vec![]); + } + if self + .capture + .end + .is_some_and(|end| frame.at.saturating_duration_since(end) > CAPTURE_GAP_TOLERANCE) + { + // Drop resampler, APM and half-packet history without advancing the + // unmute boundary: this frame may already have waited in the queue. + let cutoff = self.cutoff; + self.reset()?; + self.cutoff = cutoff; + } + self.capture.push(frame)?; + let mut encoded = Vec::new(); + while let Some((at, input)) = self.capture.next() { + let capture_age = now().saturating_duration_since(at); + if capture_age > MAX_PROCESSING_DELAY { + return Err("voice processing fell behind"); + } + let delay = (capture_age.as_millis() as i64 + self.render_delay).max(0); + if delay > MAX_PROCESSING_DELAY.as_millis() as i64 { + return Err("voice processing fell behind"); + } + self.apm + .set_stream_delay_ms(delay as i32) + .map_err(|_| "invalid voice echo delay")?; + let mut output = [0.0; BLOCK]; + self.apm + .process_capture_f32(&[&input], &mut [&mut output]) + .map_err(|_| "voice capture processing failed")?; + if output.iter().any(|sample| !sample.is_finite()) { + return Err("invalid processed voice audio"); + } + if self.pending.is_empty() { + self.at = at; + } + self.pending.extend(output); + if self.pending.len() == 960 { + let mut data = vec![0; 1275]; + let len = self + .encoder + .encode_float(&self.pending, &mut data) + .map_err(|_| "failed to encode voice audio")?; + data.truncate(len); + encoded.push(EncodedAudio { data, at: self.at }); + self.pending.clear(); + } + } + Ok(encoded) + } +} + +#[cfg(test)] +#[path = "processing_tests.rs"] +mod tests; diff --git a/codex-rs/voice-host/src/processing_tests.rs b/codex-rs/voice-host/src/processing_tests.rs new file mode 100644 index 0000000000..15446365e7 --- /dev/null +++ b/codex-rs/voice-host/src/processing_tests.rs @@ -0,0 +1,285 @@ +use super::*; +use pretty_assertions::assert_eq; + +#[test] +fn rendered_reference_changes_decoded_echo_capture() { + let mut processors = [ + Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 44_100).unwrap(), + Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 44_100).unwrap(), + ]; + for processor in &mut processors { + // Isolate echo cancellation from independent noise/gain adaptation. + processor.apm.apply_config(sonora::Config { + echo_canceller: Some(EchoCanceller::default()), + ..Default::default() + }); + } + let start = Instant::now(); + let signal = |seconds: f32| { + [311.0, 719.0, 1_423.0] + .into_iter() + .map(|frequency| (std::f32::consts::TAU * frequency * seconds).sin() * 0.1) + .sum::() + }; + let mut packets = [Vec::new(), Vec::new()]; + // Two seconds of deterministic samples allow AEC startup, without sleeping. + for block in 0..200 { + for offset in [0, 256] { + let mut frame = Frame { + samples: [0.0; 256], + len: 256.min(441 - offset), + at: start - Duration::from_secs(/*secs*/ 10) + + Duration::from_secs_f64((block * 441 + offset) as f64 / 44_100.0), + generation: 0, + }; + for (index, sample) in frame.samples[..frame.len].iter_mut().enumerate() { + *sample = signal((block * 441 + offset + index) as f32 / 44_100.0); + } + processors[0].render(&frame).unwrap(); + } + // Keep delay metadata identical: only consuming reference samples may + // explain the difference in the decoded capture signal. + processors[1].render_delay = processors[0].render_delay; + for offset in [0, 256] { + let mut frame = Frame { + samples: [0.0; 256], + len: 256.min(480 - offset), + at: start + Duration::from_secs_f64((block * 480 + offset) as f64 / 48_000.0), + generation: 2, + }; + for (index, sample) in frame.samples[..frame.len].iter_mut().enumerate() { + *sample = signal((block * 480 + offset + index) as f32 / 48_000.0 - 0.05); + } + for (processor, packets) in processors.iter_mut().zip(&mut packets) { + packets.extend(processor.capture(&frame, || frame.at).unwrap()); + } + } + } + let decoded = packets.map(|packets| { + let mut decoder = opus::Decoder::new(/*sample_rate*/ 48_000, opus::Channels::Mono).unwrap(); + let mut decoded = Vec::new(); + for packet in packets { + let mut samples = [0.0; 960]; + assert_eq!( + decoder + .decode_float(&packet.data, &mut samples, /*fec*/ false) + .unwrap(), + 960 + ); + assert!(samples.iter().all(|sample| sample.is_finite())); + decoded.extend(samples); + } + decoded + }); + assert_eq!(decoded[0].len(), decoded[1].len()); + assert!(decoded[0].len() > 48_000); + let energy = decoded.map(|samples| { + samples[48_000..] + .iter() + .map(|sample| sample * sample) + .sum::() + }); + assert!( + energy[1] > 1.0, + "control capture must contain the echo signal" + ); + assert!( + energy[0] < energy[1] * 0.5, + "reference/control echo energy: {energy:?}" + ); +} + +#[test] +fn real_encoder_produces_twenty_millisecond_opus_packets() { + for rate in [44_100, 48_000] { + let mut processor = Processor::new(rate, /*output_rate*/ 48_000).unwrap(); + let mut packets = Vec::new(); + for block in 0..40 { + let mut frame = Frame { + samples: [0.0; 256], + len: 256, + at: Instant::now(), + generation: 2, + }; + for (index, sample) in frame.samples.iter_mut().enumerate() { + *sample = (std::f32::consts::TAU * 440.0 * (block as usize * 256 + index) as f32 + / rate as f32) + .sin() + * 0.1; + } + packets.extend(processor.capture(&frame, || frame.at).unwrap()); + } + assert!(!packets.is_empty()); + let mut decoder = opus::Decoder::new(/*sample_rate*/ 48_000, opus::Channels::Mono).unwrap(); + let mut energy = 0.0_f32; + for packet in packets { + let mut output = [0.0; 960]; + assert_eq!( + decoder + .decode_float(&packet.data, &mut output, /*fec*/ false) + .unwrap(), + 960 + ); + assert!(output.iter().all(|sample| sample.is_finite())); + energy += output.iter().map(|sample| sample * sample).sum::(); + } + assert!( + energy > 0.01, + "decoded capture must contain signal at {rate} Hz" + ); + } +} + +#[test] +fn unmute_reset_rejects_delayed_pre_unmute_audio_and_partial_history() { + let mut processor = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + let frame = Frame { + samples: [0.25; 256], + len: 256, + at: Instant::now(), + generation: 2, + }; + assert!(processor.capture(&frame, || frame.at).unwrap().is_empty()); + assert_eq!( + (0..10).find_map(|_| processor + .capture(&frame, || frame.at + Duration::from_secs(1)) + .err()), + Some("voice processing fell behind") + ); + processor.reset().unwrap(); + for _ in 0..10 { + assert!(processor.capture(&frame, || frame.at).unwrap().is_empty()); + } + assert!(processor.pending.is_empty()); + assert!(processor.capture.input.is_empty()); + assert!(processor.capture.output.is_empty()); +} + +#[test] +fn capture_gap_discards_old_partial_packet_and_preserves_unmute_cutoff() { + let mut processor = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + let mut fresh = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + let cutoff = processor.cutoff; + let start = Instant::now(); + let mut frame = Frame { + samples: [0.25; 256], + len: 240, + at: start, + generation: 2, + }; + for index in 0..5 { + frame.at = start + Duration::from_millis(index * 5); + assert!(processor.capture(&frame, || frame.at).unwrap().is_empty()); + } + assert!(!processor.pending.is_empty()); + assert!(!processor.capture.input.is_empty()); + + let resumed = start + Duration::from_millis(/*millis*/ 100); + frame.samples.fill(0.0); + let mut packets = Vec::new(); + let mut expected = Vec::new(); + for index in 0..12 { + frame.at = resumed + Duration::from_millis(index * 5); + let emitted = processor.capture(&frame, || frame.at).unwrap(); + if index < 2 { + assert!(emitted.is_empty()); + } + packets.extend(emitted.into_iter().map(|packet| (packet.at, packet.data))); + expected.extend( + fresh + .capture(&frame, || frame.at) + .unwrap() + .into_iter() + .map(|packet| (packet.at, packet.data)), + ); + } + assert!(!packets.is_empty()); + assert_eq!(packets, expected); + assert_eq!(processor.cutoff, cutoff); +} + +#[test] +fn ordinary_capture_timestamp_jitter_keeps_partial_audio() { + for jitter_ms in [-2_i64, 0, 2] { + let mut processor = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + let start = Instant::now(); + let mut frame = Frame { + samples: [0.1; 256], + len: 240, + at: start, + generation: 2, + }; + for index in 0..5 { + frame.at = start + Duration::from_millis(index * 5); + assert!(processor.capture(&frame, || frame.at).unwrap().is_empty()); + } + frame.at = start + Duration::from_millis((25 + jitter_ms) as u64); + assert_eq!(processor.capture(&frame, || frame.at).unwrap().len(), 1); + } +} + +#[test] +fn callback_gap_cannot_hide_in_a_packed_partial_block() { + let mut processor = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + let mut packer = super::super::buffers::FramePacker::default(); + let queue = crossbeam_queue::ArrayQueue::new(/*capacity*/ 8); + let start = Instant::now(); + // Leave both an incomplete callback block and a processed half-packet. + for index in 0..6 { + let at = start + Duration::from_millis(index * 5); + packer.discard_capture_gap(at, /*rate*/ 48_000.0); + assert!(packer.push( + Frame { + samples: [0.25; 256], + len: 240, + at, + generation: 2 + }, + /*rate*/ 48_000.0, + &queue + )); + while let Some(frame) = queue.pop() { + assert!(processor.capture(&frame, || frame.at).unwrap().is_empty()); + } + } + assert!(!processor.pending.is_empty()); + let resumed = start + Duration::from_millis(/*millis*/ 100); + packer.discard_capture_gap(resumed, /*rate*/ 48_000.0); + assert!(packer.push( + Frame { + samples: [0.0; 256], + len: 256, + at: resumed, + generation: 2 + }, + /*rate*/ 48_000.0, + &queue + )); + let frame = queue.pop().unwrap(); + assert_eq!( + (frame.samples, frame.len, frame.at, frame.generation), + ([0.0; 256], 256, resumed, 2) + ); + assert!(processor.capture(&frame, || resumed).unwrap().is_empty()); + assert!(processor.pending.is_empty()); +} + +#[test] +fn stale_echo_reference_cannot_cancel_capture_age() { + for render_delay in [0, -750] { + let mut processor = Processor::new(/*input_rate*/ 48_000, /*output_rate*/ 48_000).unwrap(); + processor.render_delay = render_delay; + let start = Instant::now(); + let now = start + Duration::from_millis(/*millis*/ 750); + let failure = (0..8).find_map(|index| { + let frame = Frame { + samples: [0.25; 256], + len: 256, + at: start + Duration::from_secs_f64((index * 256) as f64 / 48_000.0), + generation: 2, + }; + processor.capture(&frame, || now).err() + }); + assert_eq!(failure, Some("voice processing fell behind")); + } +} diff --git a/codex-rs/voice-host/src/transport.rs b/codex-rs/voice-host/src/transport.rs index 538ce753b8..c6f705efbc 100644 --- a/codex-rs/voice-host/src/transport.rs +++ b/codex-rs/voice-host/src/transport.rs @@ -60,6 +60,7 @@ impl PeerConnectionEventHandler for Events { } pub(crate) struct Transport { + pub(crate) audio: crate::audio_track::AudioTrack, connection: Arc, gathered: Arc, observer: JoinHandle<()>, @@ -72,6 +73,7 @@ impl Transport { } async fn with_runtime(runtime: Arc) -> Result { + let (media, audio) = crate::audio_track::AudioTrack::new()?; let gathered = Arc::new(Notify::new()); // Upstream defaults exhaust checks after 1.4s, including checks sent before // a TCP connection exists. Keep probing throughout our negotiation deadline. @@ -82,6 +84,7 @@ impl Transport { settings.set_ice_connection_attempts(Some(check_interval), Some(attempts)); let connection: Arc = Arc::new( PeerConnectionBuilder::new() + .with_media_engine(media) .with_runtime(runtime) .with_setting_engine(settings) .with_handler(Arc::new(Events(gathered.clone()))) @@ -93,6 +96,10 @@ impl Transport { .await .map_err(|_| "failed to create voice peer")?, ); + if connection.add_track(audio.track.clone()).await.is_err() { + let _ = timeout(Duration::from_secs(/*secs*/ 2), connection.close()).await; + return Err("failed to attach voice track"); + } let channel = match connection .create_data_channel("oai-events", /*options*/ None) .await @@ -121,6 +128,7 @@ impl Transport { sender.send_replace(false); }); Ok(Self { + audio, connection, gathered, observer, diff --git a/codex-rs/voice-host/src/transport_tests.rs b/codex-rs/voice-host/src/transport_tests.rs index 45f5c90aa1..9c9f338b71 100644 --- a/codex-rs/voice-host/src/transport_tests.rs +++ b/codex-rs/voice-host/src/transport_tests.rs @@ -55,7 +55,9 @@ async fn check_negotiation(runtime: Arc) { let gathered = Arc::new(Notify::new()); let mut settings = webrtc::peer_connection::SettingEngine::default(); settings.set_lite(/*lite*/ true); + let (media, _) = crate::audio_track::AudioTrack::new().unwrap(); let builder = PeerConnectionBuilder::new() + .with_media_engine(media) .with_setting_engine(settings) .with_handler(Arc::new(RemoteEvents(sender, gathered.clone()))); let remote = if tcp { diff --git a/codex-rs/voice-host/tests/lifecycle.rs b/codex-rs/voice-host/tests/lifecycle.rs index d241c3c3c0..ba2fe41a0b 100644 --- a/codex-rs/voice-host/tests/lifecycle.rs +++ b/codex-rs/voice-host/tests/lifecycle.rs @@ -27,6 +27,7 @@ use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::time::timeout; use webrtc::data_channel::DataChannel; +use webrtc::peer_connection::MediaEngine; use webrtc::peer_connection::PeerConnection; use webrtc::peer_connection::PeerConnectionBuilder; use webrtc::peer_connection::PeerConnectionEventHandler; @@ -307,7 +308,10 @@ async fn installed_client_negotiates_and_closes_over_udp_and_tcp() -> Result<()> let gathered = Arc::new(Notify::new()); let mut settings = SettingEngine::default(); settings.set_lite(/*lite*/ true); + let mut media = MediaEngine::default(); + media.register_default_codecs()?; let builder = PeerConnectionBuilder::new() + .with_media_engine(media) .with_setting_engine(settings) .with_handler(Arc::new(RemoteEvents { channels: sender,