Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-17 23:14:37 +00:00
1 parent e98765c212
commit 97dbd01a3d
49 files changed
+1330 -33

No files matched your search

@@ -308,6 +308,12 @@ pub enum UserInput {
detail: Option<ImageDetail>,
path: PathBuf,
},
Audio {
url: String,
},
LocalAudio {
path: PathBuf,
},
Skill {
name: String,
path: PathBuf,
@@ -333,6 +339,8 @@ impl UserInput {
detail,
},
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
UserInput::Audio { url } => CoreUserInput::Audio { audio_url: url },
UserInput::LocalAudio { path } => CoreUserInput::LocalAudio { path },
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
}
@@ -354,6 +362,8 @@ impl From<CoreUserInput> for UserInput {
detail,
},
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
CoreUserInput::Audio { audio_url } => UserInput::Audio { url: audio_url },
CoreUserInput::LocalAudio { path } => UserInput::LocalAudio { path },
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
_ => unreachable!("unsupported user input variant"),
@@ -367,6 +377,8 @@ impl UserInput {
UserInput::Text { text, .. } => text.chars().count(),
UserInput::Image { .. }
| UserInput::LocalImage { .. }
| UserInput::Audio { .. }
| UserInput::LocalAudio { .. }
| UserInput::Skill { .. }
| UserInput::Mention { .. } => 0,
}