Annotate user input and contextual fragments with content kinds (#40196)

## What changed

- Preserve each contextual fragment's `ContentItemKind` when converting it to a response message.
- Classify user text, image, and audio content as `user.text`, `user.image`, and `user.audio` in their original order.
- Give additional context key-specific `additional_content.<key>` kinds and retain annotations on generated messages such as image resize notices and interrupted-turn markers.

## Testing

- Expand request-level coverage to verify content-kind annotations across mixed-modality turns, additional context, reminders, budgets, skills, model switching, subagents, and image resizing.

GitOrigin-RevId: 671e4545f39b335525c60d2a5df4608e8068cafc
This commit is contained in:
pakrym-oai authored and copyberry committed 2026-08-23 06:10:44 +00:00
1 parent 7b5b3bd5a2
commit 4582c0a498
21 files changed
+245 -93

No files matched your search

+23 -13
View File
@@ -22,6 +22,8 @@ use codex_protocol::mcp::ClientMcpExtensions;
use codex_protocol::mcp::MCP_APP_UI_EXTENSION_ID;
use codex_protocol::mcp::OPENAI_FORM_EXTENSION_ID;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelsResponse;
@@ -2153,19 +2155,27 @@ fn interrupted_snapshot_is_not_mid_turn() {
#[test]
fn multi_agent_v2_interrupted_marker_uses_developer_input_message() {
let marker = developer_interrupted_marker();
let ResponseItem::Message { role, content, .. } = marker else {
panic!("expected interrupted marker to be a message");
};
assert_eq!(role, "developer");
assert!(
matches!(
content.as_slice(),
[ContentItem::InputText { text }]
if text.contains(crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE)
),
"expected interrupted marker to use developer InputText content"
assert_eq!(
developer_interrupted_marker(),
ResponseItem::Message {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputText {
text: format!(
"<turn_aborted>\n{}\n</turn_aborted>",
crate::context::TurnAborted::INTERRUPTED_DEVELOPER_GUIDANCE
),
}],
phase: None,
internal_chat_message_metadata_passthrough: Some(
InternalChatMessageMetadataPassthrough {
content_item_kinds: Some(vec![ContentItemKind(
"generic.turn_aborted".to_string()
)]),
..Default::default()
}
),
}
);
}