diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5194ab6059..4536b2188b 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2761,6 +2761,7 @@ dependencies = [ "codex-thread-store", "codex-tools", "codex-utils-absolute-path", + "codex-utils-audio", "codex-utils-cache", "codex-utils-cargo-bin", "codex-utils-home-dir", @@ -2802,7 +2803,6 @@ dependencies = [ "sha1 0.10.6", "shlex", "similar", - "symphonia", "tempfile", "test-case", "test-log", @@ -4377,6 +4377,20 @@ dependencies = [ "codex-protocol", ] +[[package]] +name = "codex-utils-audio" +version = "0.0.0" +dependencies = [ + "base64 0.22.1", + "codex-protocol", + "codex-utils-cache", + "codex-utils-string", + "pretty_assertions", + "symphonia", + "thiserror 2.0.18", + "tracing", +] + [[package]] name = "codex-utils-cache" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 2f5588034b..9f971f7e96 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -94,6 +94,7 @@ members = [ "v8-poc", "websocket-client", "utils/absolute-path", + "utils/audio", "utils/path-uri", "utils/cargo-bin", "git-utils", @@ -246,6 +247,7 @@ codex-tui = { path = "tui" } codex-uds = { path = "uds" } codex-utils-absolute-path = { path = "utils/absolute-path" } codex-utils-approval-presets = { path = "utils/approval-presets" } +codex-utils-audio = { path = "utils/audio" } codex-utils-cache = { path = "utils/cache" } codex-utils-cargo-bin = { path = "utils/cargo-bin" } codex-utils-cli = { path = "utils/cli" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index cb524b4a93..3c8ad1b9fb 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -69,6 +69,7 @@ codex-terminal-detection = { workspace = true } codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-audio = { workspace = true } codex-utils-cache = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } @@ -103,7 +104,6 @@ serde_json = { workspace = true } sha1 = { workspace = true } shlex = { workspace = true } similar = { workspace = true } -symphonia = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 6f4c259feb..a3cf288ee5 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,4 +1,3 @@ -use crate::audio_preparation::estimate_audio_token_count; use crate::context::ContextualUserFragment; use crate::context::world_state::WorldState; use crate::context::world_state::WorldStateSnapshot; @@ -23,6 +22,7 @@ use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::WorldStateItem; +use codex_utils_audio::estimate_audio_token_count; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; use codex_utils_output_truncation::TruncationPolicy; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index c21bdc8c8d..862d98e9ca 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -7,7 +7,6 @@ mod apply_patch; mod apps; -mod audio_preparation; mod client; mod client_common; mod realtime_context; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index ec9c9cb1ea..df667895c4 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -17,7 +17,6 @@ use crate::agent::status::is_final; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; -use crate::audio_preparation::prepare_response_items as prepare_audio_response_items; use crate::compact; use crate::compact::CompactedHistoryMetadata; use crate::config::ManagedFeatures; @@ -152,6 +151,7 @@ use codex_thread_store::ReadThreadParams; use codex_thread_store::ResumeThreadParams; use codex_thread_store::ThreadPersistenceMetadata; use codex_thread_store::ThreadStore; +use codex_utils_audio::prepare_response_items as prepare_audio_response_items; use codex_utils_path_uri::PathUri; use futures::future::BoxFuture; use futures::future::Shared; diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index 5054f68c9c..57f1e6725c 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -24,7 +24,6 @@ use serde_json::Value as JsonValue; use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; -use crate::audio_preparation::estimate_audio_token_count; use crate::function_tool::FunctionCallError; use crate::original_image_detail::can_request_original_image_detail; use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items; @@ -41,6 +40,7 @@ use crate::tools::router::ToolCallSource; use crate::unified_exec::resolve_max_tokens; use codex_protocol::openai_models::ToolMode; use codex_tools::ToolName; +use codex_utils_audio::estimate_audio_token_count; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy; use codex_utils_output_truncation::truncate_function_output_items_with_policy; diff --git a/codex-rs/utils/audio/BUILD.bazel b/codex-rs/utils/audio/BUILD.bazel new file mode 100644 index 0000000000..eb2c15b509 --- /dev/null +++ b/codex-rs/utils/audio/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "audio", + crate_name = "codex_utils_audio", +) diff --git a/codex-rs/utils/audio/Cargo.toml b/codex-rs/utils/audio/Cargo.toml new file mode 100644 index 0000000000..90540228a6 --- /dev/null +++ b/codex-rs/utils/audio/Cargo.toml @@ -0,0 +1,23 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-utils-audio" +version.workspace = true + +[lints] +workspace = true + +[dependencies] +base64 = { workspace = true } +codex-protocol = { workspace = true } +codex-utils-cache = { workspace = true } +codex-utils-string = { workspace = true } +symphonia = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } + +[lib] +doctest = false diff --git a/codex-rs/core/src/audio_preparation_tests.rs b/codex-rs/utils/audio/src/audio_preparation_tests.rs similarity index 100% rename from codex-rs/core/src/audio_preparation_tests.rs rename to codex-rs/utils/audio/src/audio_preparation_tests.rs diff --git a/codex-rs/core/src/audio_preparation.rs b/codex-rs/utils/audio/src/lib.rs similarity index 95% rename from codex-rs/core/src/audio_preparation.rs rename to codex-rs/utils/audio/src/lib.rs index 9f44420470..ab836b5631 100644 --- a/codex-rs/core/src/audio_preparation.rs +++ b/codex-rs/utils/audio/src/lib.rs @@ -1,3 +1,5 @@ +//! Audio preparation and duration-based token estimates for model inputs. + use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::ContentItem; @@ -5,7 +7,7 @@ use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; -use codex_utils_output_truncation::approx_token_count; +use codex_utils_string::approx_token_count; use std::io::Cursor; use std::num::NonZeroUsize; use std::sync::LazyLock; @@ -58,7 +60,8 @@ impl AudioPreparationError { } } -pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { +/// Canonicalizes audio inputs and replaces unsupported inputs with text placeholders. +pub fn prepare_response_items(items: &mut [ResponseItem]) { for item in items { match item { ResponseItem::Message { content, .. } => prepare_message_content(content), @@ -141,7 +144,8 @@ fn canonical_audio_mime(mime: &str) -> Option<&'static str> { } } -pub(crate) fn estimate_audio_token_count(audio_url: &str) -> usize { +/// Estimates audio tokens from decoded duration, falling back to the data URL size. +pub fn estimate_audio_token_count(audio_url: &str) -> usize { let key = sha1_digest(audio_url.as_bytes()); AUDIO_TOKEN_ESTIMATE_CACHE.get_or_insert_with(key, || { let Some(duration_seconds) = audio_duration_seconds(audio_url) else {