Add audio variants to user input protocols (#33923)

## What changed

- Add remote and local audio variants to core and app-server user inputs, including generated JSON and TypeScript schemas.
- Preserve audio fields when converting app-server inputs and response content into thread items.
- Keep unsupported audio out of model requests, TUI rendering, text extraction, and truncated tool output, using explicit text placeholders where audio is submitted as a model input.

## Testing

- Cover app-server media conversion, unsupported-input placeholders, TUI omission, and tool-output truncation.

GitOrigin-RevId: 16152bf3af674575ce569764a8b472e032e7569d
This commit is contained in:
nhamidi-oai authored and copyberry committed 2026-07-17 23:14:37 +00:00
1 parent e98765c212
commit 97dbd01a3d
49 files changed
+1330 -33

No files matched your search

+5 -3
View File
@@ -252,7 +252,7 @@ fn has_subagent_notification(history_items: &[ResponseItem]) -> bool {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
SubagentNotification::matches_text(text)
}
ContentItem::InputImage { .. } => false,
ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false,
})
})
}
@@ -267,7 +267,7 @@ fn history_contains_text(history_items: &[ResponseItem], needle: &str) -> bool {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text.contains(needle)
}
ContentItem::InputImage { .. } => false,
ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false,
})
})
}
@@ -290,7 +290,9 @@ fn history_contains_assistant_inter_agent_communication(
.as_ref()
== Some(expected)
}
ContentItem::InputText { .. } | ContentItem::InputImage { .. } => false,
ContentItem::InputText { .. }
| ContentItem::InputImage { .. }
| ContentItem::InputAudio { .. } => false,
})
})
}