Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-17 23:14:37 +00:00
1 parent e98765c212
commit 97dbd01a3d
49 files changed
+1330 -33

No files matched your search

@@ -2545,6 +2545,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
},
CoreUserInput::Audio {
audio_url: "data:audio/wav;base64,AAA".to_string(),
},
CoreUserInput::LocalAudio {
path: PathBuf::from("local/audio.mp3"),
},
CoreUserInput::Skill {
name: "skill-creator".to_string(),
path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"),
@@ -2574,6 +2580,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
},
UserInput::Audio {
url: "data:audio/wav;base64,AAA".to_string(),
},
UserInput::LocalAudio {
path: PathBuf::from("local/audio.mp3"),
},
UserInput::Skill {
name: "skill-creator".to_string(),
path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"),
@@ -3022,7 +3034,7 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() {
}
#[test]
fn user_input_into_core_preserves_image_detail() {
fn user_input_into_core_preserves_media_fields() {
assert_eq!(
UserInput::Image {
url: "https://example.com/image.png".to_string(),
@@ -3046,6 +3058,26 @@ fn user_input_into_core_preserves_image_detail() {
detail: Some(ImageDetail::Original),
}
);
assert_eq!(
UserInput::Audio {
url: "data:audio/wav;base64,AAA".to_string(),
}
.into_core(),
CoreUserInput::Audio {
audio_url: "data:audio/wav;base64,AAA".to_string(),
}
);
assert_eq!(
UserInput::LocalAudio {
path: PathBuf::from("local/audio.mp3"),
}
.into_core(),
CoreUserInput::LocalAudio {
path: PathBuf::from("local/audio.mp3"),
}
);
}
#[test]