github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-21 05:22:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor image content to use a shared ImageReference type (#45543)

Browse Source 
## What changed

Represent images in `ContentItem` and `FunctionCallOutputContentItem` with `ImageReference::Inline`, flattened to preserve the existing `image_url` wire format. Update image producers and consumers and regenerate app-server schemas and SDK artifacts.

Preserve the Python SDK's `InputImageContentItem` and `InputImageFunctionCallOutputContentItem` class names during generation.

## Testing

Add a regression test for stable Python image class names and adapt existing image tests to the shared representation.

GitOrigin-RevId: c38a780ac3314c2ac2deb3afc1b93b94b6f93fec
This commit is contained in:
Krish Chainani
2026-09-14 23:25:09 +00:00
committed by copyberry
parent 6ae5e71458
commit 5a66d460d3
79 changed files with 994 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-protocol/schema/json/ClientRequest.json
Generated
+28 -8
View File
@@ -660,6 +660,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -671,9 +685,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -683,7 +694,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageContentItem",
@@ -1420,6 +1430,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1431,9 +1455,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -1443,7 +1464,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/ServerNotification.json
Generated
+14 -4
View File
@@ -1759,6 +1759,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -1770,9 +1784,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -1782,7 +1793,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
Generated
+28 -8
View File
@@ -10472,6 +10472,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -10483,9 +10497,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -10495,7 +10506,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageContentItem",
@@ -12333,6 +12343,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -12344,9 +12368,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -12356,7 +12377,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
Generated
+28 -8
View File
@@ -6352,6 +6352,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -6363,9 +6377,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -6375,7 +6386,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageContentItem",
@@ -8213,6 +8223,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -8224,9 +8248,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -8236,7 +8257,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
Generated
+14 -4
View File
@@ -359,6 +359,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
Generated
+14 -4
View File
@@ -359,6 +359,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
Generated
+28 -8
View File
@@ -80,6 +80,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -91,9 +105,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -103,7 +114,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageContentItem",
@@ -188,6 +198,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -199,9 +223,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -211,7 +232,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
Generated
+14 -4
View File
@@ -499,6 +499,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
Generated
+14 -4
View File
@@ -580,6 +580,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -591,9 +605,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -603,7 +614,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
Generated
+14 -4
View File
@@ -359,6 +359,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
Generated
+28 -8
View File
@@ -142,6 +142,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -153,9 +167,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -165,7 +176,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageContentItem",
@@ -250,6 +260,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -261,9 +285,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -273,7 +294,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
Generated
+14 -4
View File
@@ -596,6 +596,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -607,9 +621,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -619,7 +630,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
Generated
+14 -4
View File
@@ -580,6 +580,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -591,9 +605,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -603,7 +614,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
Generated
+14 -4
View File
@@ -499,6 +499,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
Generated
+14 -4
View File
@@ -502,6 +502,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
Generated
+14 -4
View File
@@ -499,6 +499,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
Generated
+14 -4
View File
@@ -166,6 +166,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -177,9 +191,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -189,7 +200,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
Generated
+14 -4
View File
@@ -499,6 +499,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
Generated
+14 -4
View File
@@ -499,6 +499,20 @@
"type": "object"
},
{
"anyOf": [
{
"properties": {
"image_url": {
"type": "string"
}
},
"required": [
"image_url"
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
"image_url": {
"type": "string"
},
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
"image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst
Generated
BIN
View File
Binary file not shown.
codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
Generated
+1 -1
View File
@@ -3,4 +3,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageDetail } from "./ImageDetail";
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
Generated
+1 -1
View File
@@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail";
* Responses API compatible content items that can be returned by a tool call.
* This is a subset of ContentItem with the types we support as function call outputs.
*/
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
codex-rs/app-server/src/request_processors/turn_processor.rs
+3 -2
View File
@@ -8,6 +8,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind;
use codex_protocol::protocol::TurnSettingsUpdate;
@@ -36,7 +37,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
ResponseItem::Message { content, .. } => content.iter().any(|item| {
matches!(
item,
ContentItem::InputImage { image_url, .. } if is_remote_image_url(image_url)
ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
)
}),
ResponseItem::FunctionCallOutput { output, .. }
@@ -45,7 +46,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
content.iter().any(|item| {
matches!(
item,
FunctionCallOutputContentItem::InputImage { image_url, .. }
FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
if is_remote_image_url(image_url)
)
})
codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
+4 -1
View File
@@ -30,6 +30,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use core_test_support::load_default_config_for_test;
use core_test_support::responses;
@@ -715,7 +716,9 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
text: "dynamic-ok".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: TINY_PNG_DATA_URL.to_string(),
image: ImageReference::Inline {
image_url: TINY_PNG_DATA_URL.to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
codex-rs/app-server/tests/suite/v2/request_validation.rs
+4 -1
View File
@@ -14,6 +14,7 @@ use codex_app_server_protocol::ThreadStartResponse;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -173,7 +174,9 @@ async fn request_handlers_reject_remote_image_urls() -> Result<()> {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/tool.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/tool.png".to_string(),
},
detail: Some(ImageDetail::High),
},
]),
codex-rs/codex-api/src/endpoint/search.rs
+4 -1
View File
@@ -72,6 +72,7 @@ mod tests {
use codex_client::TransportError;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use http::StatusCode;
use pretty_assertions::assert_eq;
@@ -164,7 +165,9 @@ mod tests {
text: "find this".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: None,
},
],
codex-rs/core/src/client_common_tests.rs
+10 -3
View File
@@ -6,6 +6,7 @@ use codex_models_manager::model_info::model_info_from_slug;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
use serde_json::value::RawValue;
use std::sync::Arc;
@@ -27,7 +28,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
text: "Describe this image.".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail,
},
],
@@ -41,7 +44,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,function".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,function".to_string(),
},
detail,
},
]),
@@ -53,7 +58,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,custom".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,custom".to_string(),
},
detail,
},
]),
codex-rs/core/src/compact_remote_v2.rs
+26 -8
View File
@@ -45,6 +45,8 @@ use codex_protocol::items::ContextCompactionItem;
use codex_protocol::items::TurnItem;
use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::ContentItem;
#[cfg(test)]
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::TokenUsage;
@@ -981,11 +983,15 @@ mod tests {
text: "user".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
},
ContentItem::InputImage {
image_url: "data:image/png;base64,def".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,def".to_string(),
},
detail: None,
},
],
@@ -1047,7 +1053,9 @@ mod tests {
text: "abcdef".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
},
ContentItem::OutputText {
@@ -1057,7 +1065,9 @@ mod tests {
text: "discarded after the text budget is exhausted".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,def".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,def".to_string(),
},
detail: None,
},
],
@@ -1089,14 +1099,18 @@ mod tests {
text: "abcdef".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string()
},
detail: None,
},
ContentItem::OutputText {
text: "uv…1 tokens truncated…yz".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,def".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,def".to_string()
},
detail: None,
},
],
@@ -1123,7 +1137,9 @@ mod tests {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
}],
phase: None,
@@ -1147,7 +1163,9 @@ mod tests {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
}],
phase: None,
codex-rs/core/src/compact_remote_v2_image_budget_tests.rs
+4 -1
View File
@@ -1,5 +1,6 @@
use super::*;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::image_close_tag_text;
use codex_protocol::models::local_image_open_tag_text_with_path;
@@ -17,7 +18,9 @@ fn message(content: Vec<ContentItem>) -> ResponseItem {
fn image() -> ContentItem {
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
}
}
codex-rs/core/src/compact_remote_v2_images.rs
+7 -3
View File
@@ -4,6 +4,7 @@ use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::is_image_close_tag_text;
use codex_protocol::models::is_image_open_tag_text;
use codex_protocol::models::is_local_image_open_tag_text;
@@ -17,9 +18,12 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
approx_token_count(text)
}
ContentItem::InputImage { image_url, detail } => usize::try_from(
approx_tokens_from_byte_count_i64(estimate_image_bytes(image_url, *detail)),
)
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => usize::try_from(approx_tokens_from_byte_count_i64(estimate_image_bytes(
image_url, *detail,
)))
.unwrap_or(usize::MAX),
ContentItem::InputAudio { .. } => 0,
}
codex-rs/core/src/compact_tests.rs
+4 -1
View File
@@ -10,6 +10,7 @@ use codex_protocol::models::ContentItemKind;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ExecutedToolCall;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
@@ -207,7 +208,9 @@ fn content_items_to_text_joins_non_empty_segments() {
#[test]
fn content_items_to_text_ignores_image_only_content() {
let items = vec![ContentItem::InputImage {
image_url: "file://image.png".to_string(),
image: ImageReference::Inline {
image_url: "file://image.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
codex-rs/core/src/context/node_repl_review_evidence.rs
+4 -1
View File
@@ -12,6 +12,7 @@ use codex_guardian_context::NodeReplContext;
use codex_guardian_context::NodeReplResponse;
pub use codex_guardian_context::NodeReplReviewEvidenceMode;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_protocol::user_input::UserInput::Image;
use codex_protocol::user_input::UserInput::Text;
@@ -117,7 +118,9 @@ impl NodeReplReviewEvidence {
.filter_map(|item| match item {
Image { image_url, detail } if seen_images.insert(image_url) => {
Some(ContentItem::InputImage {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: *detail,
})
}
codex-rs/core/src/context_manager/history.rs
+9 -6
View File
@@ -43,6 +43,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::InterAgentCommunication;
@@ -850,9 +851,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text_bytes(text)
}
ContentItem::InputImage { image_url, detail } => {
estimate_image_bytes(image_url, *detail)
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
@@ -1055,9 +1057,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
.iter()
.map(|part| match part {
FunctionCallOutputContentItem::InputText { text } => text_bytes(text),
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
estimate_image_bytes(image_url, *detail)
}
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
codex-rs/core/src/context_manager/history_tests.rs
+40 -18
View File
@@ -18,6 +18,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
@@ -715,7 +716,9 @@ fn for_prompt_annotated_preserves_metadata_while_normalizing_item() {
text: "keep".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: None,
},
],
@@ -796,7 +799,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "look at this".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/img.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/img.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputAudio {
@@ -838,7 +843,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "image result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/result.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/result.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -865,7 +872,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "js repl result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/js-repl-result.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/js-repl-result.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -991,7 +1000,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "look".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/img.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/img.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2414,7 +2425,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
text: "Here is the screenshot".to_string(),
},
ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2455,7 +2466,7 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2484,7 +2495,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2628,7 +2639,9 @@ fn non_base64_image_urls_use_image_estimates() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "https://example.com/foo.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/foo.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2641,7 +2654,9 @@ fn non_base64_image_urls_use_image_estimates() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "file:///tmp/foo.png".to_string(),
image: ImageReference::Inline {
image_url: "file:///tmp/foo.png".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2746,7 +2761,10 @@ fn data_url_without_base64_marker_uses_image_estimate() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg'/>".to_string(),
image: ImageReference::Inline {
image_url: "data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg'/>"
.to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2771,7 +2789,7 @@ fn non_image_base64_data_url_uses_image_estimate() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2791,7 +2809,7 @@ fn mixed_case_data_url_markers_are_adjusted() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2818,11 +2836,15 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
text: "images".to_string(),
},
ContentItem::InputImage {
image_url: image_url_one,
image: ImageReference::Inline {
image_url: image_url_one,
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputImage {
image_url: image_url_two,
image: ImageReference::Inline {
image_url: image_url_two,
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2858,7 +2880,7 @@ fn original_detail_images_scale_with_dimensions() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
@@ -2891,7 +2913,7 @@ fn original_detail_images_are_capped_at_max_patch_count() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
@@ -2927,7 +2949,7 @@ fn original_detail_webp_images_scale_with_dimensions() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
codex-rs/core/src/event_mapping.rs
+5 -1
View File
@@ -5,6 +5,7 @@ use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -126,7 +127,10 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
text_elements: Vec::new(),
});
}
ContentItem::InputImage { image_url, detail } => {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => {
content.push(UserInput::Image {
image_url: image_url.clone(),
detail: *detail,
codex-rs/core/src/event_mapping_tests.rs
+13 -4
View File
@@ -12,6 +12,7 @@ use codex_protocol::items::WebSearchItem;
use codex_protocol::items::build_hook_prompt_message;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
@@ -83,11 +84,15 @@ fn parses_user_message_with_text_and_two_images() {
text: "Hello world".to_string(),
},
ContentItem::InputImage {
image_url: img1.clone(),
image: ImageReference::Inline {
image_url: img1.clone(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputImage {
image_url: img2.clone(),
image: ImageReference::Inline {
image_url: img2.clone(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -131,7 +136,9 @@ fn skips_local_image_label_text() {
content: vec![
ContentItem::InputText { text: label },
ContentItem::InputImage {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -258,7 +265,9 @@ fn skips_unnamed_image_label_text() {
content: vec![
ContentItem::InputText { text: label },
ContentItem::InputImage {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
codex-rs/core/src/guardian/review_session.rs
+3 -2
View File
@@ -35,6 +35,7 @@ use codex_protocol::items::TurnItem;
use codex_protocol::mcp::is_node_repl_backed_server;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
@@ -502,7 +503,7 @@ async fn run_review_on_session(
_ => &[],
})
.filter_map(|item| match item {
ContentItem::InputImage { image_url, .. } => Some(image_url.as_str()),
ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } => Some(image_url.as_str()),
_ => None,
})
.collect::<HashSet<_>>();
@@ -550,7 +551,7 @@ async fn run_review_on_session(
return false;
};
content.iter().any(|item| {
matches!(item, ContentItem::InputImage { image_url, .. }
matches!(item, ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
if !reviewer_image_urls.contains(image_url.as_str()))
})
});
codex-rs/core/src/guardian/tests.rs
+4 -1
View File
@@ -45,6 +45,7 @@ use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::SandboxPermissions;
@@ -1130,7 +1131,9 @@ fn collect_guardian_transcript_entries_preserves_named_unpaired_tool_sources() {
if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] {
*output = codex_protocol::models::FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,image".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,image".to_string(),
},
detail: None,
},
]);
codex-rs/core/src/image_preparation.rs
+11 -2
View File
@@ -15,6 +15,7 @@ use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_utils_image::ImageProcessingError;
@@ -185,7 +186,11 @@ fn prepare_message_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
if let ContentItem::InputImage { image_url, detail } = item.content_mut() {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item.content_mut()
{
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
@@ -226,7 +231,11 @@ fn prepare_tool_output_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item {
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
codex-rs/core/src/image_preparation_tests.rs
+68 -20
View File
@@ -5,6 +5,7 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_utils_image::data_url_from_bytes;
use image::DynamicImage;
@@ -41,11 +42,15 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: data_url,
image: ImageReference::Inline {
image_url: data_url,
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::Low),
},
],
@@ -56,7 +61,9 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputImage {
image_url: "https://example.com/developer-image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/developer-image.png".to_string(),
},
detail: Some(ImageDetail::High),
}],
phase: None,
@@ -73,7 +80,10 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
panic!("expected message");
};
let [
ContentItem::InputImage { image_url, .. },
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
},
ContentItem::InputText { text },
] = content.as_slice()
else {
@@ -135,7 +145,10 @@ fn detail_policies_apply_the_expected_budgets() {
let mut items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage { image_url, detail }],
content: vec![ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
}];
@@ -149,7 +162,13 @@ fn detail_policies_apply_the_expected_budgets() {
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
};
let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else {
let [
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
},
] = content.as_slice()
else {
panic!("expected image");
};
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
@@ -179,7 +198,7 @@ fn preparation_reports_tool_output_item_id() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::High),
},
]),
@@ -215,15 +234,21 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: small_image_url,
image: ImageReference::Inline {
image_url: small_image_url,
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image_url: large_image_url.clone(),
image: ImageReference::Inline {
image_url: large_image_url.clone(),
},
detail: Some(ImageDetail::High),
},
],
@@ -246,11 +271,15 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: large_image_url,
image: ImageReference::Inline {
image_url: large_image_url,
},
detail: Some(ImageDetail::High),
},
]),
@@ -290,14 +319,20 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
);
let [
ContentItem::InputImage {
image_url: small_message_image_url,
image:
ImageReference::Inline {
image_url: small_message_image_url,
},
..
},
ContentItem::InputText {
text: failed_message_image,
},
ContentItem::InputImage {
image_url: resized_message_image_url,
image:
ImageReference::Inline {
image_url: resized_message_image_url,
},
..
},
] = content.as_slice()
@@ -342,7 +377,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
text: failed_tool_image,
},
FunctionCallOutputContentItem::InputImage {
image_url: resized_tool_image_url,
image:
ImageReference::Inline {
image_url: resized_tool_image_url,
},
..
},
] = output.content_items().expect("tool output content items")
@@ -394,19 +432,27 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: data_url_from_bytes("image/png", b"not an image"),
image: ImageReference::Inline {
image_url: data_url_from_bytes("image/png", b"not an image"),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: valid_image_url.clone(),
image: ImageReference::Inline {
image_url: valid_image_url.clone(),
},
detail: Some(ImageDetail::Low),
},
FunctionCallOutputContentItem::InputImage {
image_url: valid_image_url,
image: ImageReference::Inline {
image_url: valid_image_url,
},
detail: Some(ImageDetail::High),
},
]),
@@ -442,7 +488,9 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: expected_valid_image_url,
image: ImageReference::Inline {
image_url: expected_valid_image_url
},
detail: Some(ImageDetail::High),
},
]),
codex-rs/core/src/session/tests.rs
+16 -5
View File
@@ -74,6 +74,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::openai_models::ModelServiceTier;
@@ -2575,11 +2576,15 @@ async fn prepares_image_failures_before_history_insertion() {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::High),
},
]),
@@ -2644,11 +2649,15 @@ async fn prepares_resumed_history_before_installing_it() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "data:image/png;base64,%%%".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
image: ImageReference::Inline {
image_url: "https://example.com/image.png".to_string(),
},
detail: Some(ImageDetail::High),
},
ContentItem::InputText {
@@ -11270,7 +11279,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
text: "late pending input".to_string(),
},
ContentItem::InputImage {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: Some(ImageDetail::Original),
},
],
codex-rs/core/src/tools/code_mode/response_adapter.rs
+2 -1
View File
@@ -2,6 +2,7 @@ use codex_code_mode::ImageDetail as CodeModeImageDetail;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
trait IntoProtocol<T> {
fn into_protocol(self) -> T;
@@ -36,7 +37,7 @@ impl IntoProtocol<FunctionCallOutputContentItem>
}
codex_code_mode::FunctionCallOutputContentItem::InputImage { image_url, detail } => {
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: detail
.map(IntoProtocol::into_protocol)
.or(Some(DEFAULT_IMAGE_DETAIL)),
codex-rs/core/src/tools/context_tests.rs
+10 -3
View File
@@ -1,5 +1,6 @@
use super::*;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageReference;
use codex_protocol::models::SearchToolCallParams;
use core_test_support::assert_regex_match;
use pretty_assertions::assert_eq;
@@ -217,7 +218,9 @@ fn mcp_tool_output_response_item_preserves_content_items() {
text: "Wall time: 0.5000 seconds\nOutput:".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string()
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
@@ -295,7 +298,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -315,7 +320,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
codex-rs/core/src/tools/handlers/view_image.rs
+4 -1
View File
@@ -7,6 +7,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::openai_models::InputModality;
use codex_utils_image::data_url_from_bytes;
@@ -238,7 +239,9 @@ impl ToolOutput for ViewImageOutput {
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let body =
FunctionCallOutputBody::ContentItems(vec![FunctionCallOutputContentItem::InputImage {
image_url: self.image_url.clone(),
image: ImageReference::Inline {
image_url: self.image_url.clone(),
},
detail: Some(self.image_detail),
}]);
let output = FunctionCallOutputPayload {
codex-rs/core/tests/suite/client.rs
+10 -3
View File
@@ -40,6 +40,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
use codex_protocol::models::LocalShellStatus;
@@ -1078,7 +1079,9 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: legacy_image_url.to_string(),
image: ImageReference::Inline {
image_url: legacy_image_url.to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -1218,7 +1221,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: Some(ImageDetail::Original),
},
]),
@@ -1247,7 +1252,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: image_url.to_string(),
image: ImageReference::Inline {
image_url: image_url.to_string(),
},
detail: Some(ImageDetail::Original),
},
]),
codex-rs/core/tests/suite/code_mode.rs
+4 -1
View File
@@ -42,6 +42,7 @@ use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
@@ -5662,7 +5663,9 @@ async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transc
assert_eq!(
evidence.images(),
vec![ContentItem::InputImage {
image_url: SCREENSHOT.to_owned(),
image: ImageReference::Inline {
image_url: SCREENSHOT.to_owned()
},
detail: Some(ImageDetail::Low),
}]
);
codex-rs/core/tests/suite/image_rollout.rs
+13 -5
View File
@@ -10,6 +10,7 @@ use codex_protocol::config_types::Settings;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
@@ -69,7 +70,10 @@ fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
fn extract_image_url(item: &ResponseItem) -> Option<String> {
match item {
ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span {
ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
_ => None,
}),
_ => None,
@@ -178,7 +182,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu
),
},
ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -269,7 +273,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -341,7 +345,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
historical_content.insert(
/*index*/ 0,
ContentItem::InputImage {
image_url: original_image_url.clone(),
image: ImageReference::Inline {
image_url: original_image_url.clone(),
},
detail: Some(ImageDetail::High),
},
);
@@ -480,7 +486,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
ResponseInputItem::Message {
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: original_image_url,
image: ImageReference::Inline {
image_url: original_image_url,
},
detail: Some(ImageDetail::High),
}],
phase: None,
codex-rs/core/tests/suite/rmcp_client.rs
+4 -1
View File
@@ -52,6 +52,7 @@ use codex_protocol::mcp_policy::McpServerRequirement;
use codex_protocol::mcp_policy::PluginMcpRequirements;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::PermissionProfileSnapshot;
use codex_protocol::models::ResponseItem;
@@ -1549,7 +1550,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
};
assert!(
content.contains(&ContentItem::InputImage {
image_url: OPENAI_PNG.to_string(),
image: ImageReference::Inline {
image_url: OPENAI_PNG.to_string()
},
detail: Some(ImageDetail::Original),
}),
"interrupted input must use the current model's unified image budget"
codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs
+2 -1
View File
@@ -2,6 +2,7 @@
use super::*;
use crate::async_scorer::authorization::ScoreAuthorization;
use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[derive(Clone, Copy)]
@@ -139,7 +140,7 @@ async fn assert_catalog_budget(evidence: BudgetEvidence) -> Result<()> {
unreachable!("user instruction is a message");
};
content.push(ContentItem::InputImage {
image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned(),
image: ImageReference::Inline { image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned() },
detail: None,
});
(None, vec!["optional old commentary ".repeat(/*n*/ 500)])
codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
+7 -2
View File
@@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
@@ -1641,7 +1642,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
text: "Review what is shown on screen.".to_owned(),
},
ContentItem::InputImage {
image_url: user_image.to_owned(),
image: ImageReference::Inline {
image_url: user_image.to_owned(),
},
detail: Some(ImageDetail::High),
},
],
@@ -1667,7 +1670,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
text: "Screenshot captured.".to_owned(),
},
FunctionCallOutputContentItem::InputImage {
image_url: tool_image.to_owned(),
image: ImageReference::Inline {
image_url: tool_image.to_owned(),
},
detail: Some(ImageDetail::High),
},
]),
codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs
+10 -3
View File
@@ -7,6 +7,7 @@ use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -1077,7 +1078,9 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] {
*output = FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,image".to_owned(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,image".to_owned(),
},
detail: None,
},
]);
@@ -1222,7 +1225,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
text: "Review this screenshot.".to_string(),
},
ContentItem::InputImage {
image_url: format!("data:image/png;base64,{oversized_image}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{oversized_image}"),
},
detail: None,
},
ContentItem::InputAudio {
@@ -1242,7 +1247,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
text: "Screenshot captured.".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,tool-image".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,tool-image".to_string(),
},
detail: None,
},
FunctionCallOutputContentItem::InputAudio {
codex-rs/ext/history-notes/src/tools.rs
+4 -1
View File
@@ -11,6 +11,7 @@ use codex_extension_api::parse_tool_input_schema;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_tools::JsonToolOutput;
use codex_tools::ResponsesApiNamespace;
@@ -367,7 +368,9 @@ impl HistoryNotesToolOutput {
serde_json::from_value(image.get("detail").cloned().unwrap_or(Value::Null))
.map_err(|_| invalid_image())?;
content.push(FunctionCallOutputContentItem::InputImage {
image_url: format!("data:{mime_type};base64,{data}"),
image: ImageReference::Inline {
image_url: format!("data:{mime_type};base64,{data}"),
},
detail,
});
}
codex-rs/ext/history-notes/src/tools_tests.rs
+13 -4
View File
@@ -1,6 +1,7 @@
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_tools::ToolOutput;
use codex_tools::ToolPayload;
@@ -73,19 +74,27 @@ fn preserves_images_as_separate_output_items_without_logging_bytes() {
encrypted_content: "enc_payload".to_string()
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,cG5n".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,cG5n".to_string()
},
detail: Some(ImageDetail::Original)
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/jpeg;base64,anBlZw==".to_string(),
image: ImageReference::Inline {
image_url: "data:image/jpeg;base64,anBlZw==".to_string()
},
detail: Some(ImageDetail::Low)
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/gif;base64,Z2lm".to_string(),
image: ImageReference::Inline {
image_url: "data:image/gif;base64,Z2lm".to_string()
},
detail: None
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/webp;base64,d2VicA==".to_string(),
image: ImageReference::Inline {
image_url: "data:image/webp;base64,d2VicA==".to_string()
},
detail: None
},
])
codex-rs/ext/image-generation/src/tests.rs
+13 -4
View File
@@ -12,6 +12,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_tools::ResponsesApiNamespaceTool;
@@ -266,7 +267,9 @@ fn generated_output_returns_image_input_and_output_hint() {
content_items,
vec![
FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{RESULT}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{RESULT}")
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: output_hint },
@@ -311,7 +314,9 @@ fn generated_output_omits_oversized_output_hint() {
assert_eq!(
content_items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{RESULT}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{RESULT}")
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -319,14 +324,18 @@ fn generated_output_omits_oversized_output_hint() {
fn input_image(image: &str) -> ContentItem {
ContentItem::InputImage {
image_url: format!("data:image/png;base64,{image}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{image}"),
},
detail: None,
}
}
fn image_output(image: &str) -> FunctionCallOutputPayload {
FunctionCallOutputPayload::from_content_items(vec![FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{image}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{image}"),
},
detail: None,
}])
}
codex-rs/ext/image-generation/src/tool.rs
+12 -3
View File
@@ -29,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
@@ -494,7 +495,10 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
match item {
ResponseItem::Message { content, .. } => {
image_urls.extend(content.iter().rev().filter_map(|item| match item {
ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => None,
@@ -542,7 +546,10 @@ fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator<Item =
.flatten()
.rev()
.filter_map(|item| match item {
FunctionCallOutputContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.clone()),
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => None,
@@ -651,7 +658,9 @@ impl ToolOutput for GeneratedImageOutput {
/// Returns generated bytes and persisted-artifact context for model follow-up.
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let mut content = vec![FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{}", self.result),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{}", self.result),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
if let Some(output_hint) = &self.output_hint {
codex-rs/ext/web-search/src/history.rs
+4 -1
View File
@@ -84,6 +84,7 @@ mod tests {
use codex_api::SearchInput;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
@@ -158,7 +159,9 @@ mod tests {
text: "previous user".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,image".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,image".to_string(),
},
detail: None,
},
],
codex-rs/guardian-context/src/budget.rs
+10 -2
View File
@@ -5,6 +5,7 @@
use std::io::Write;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::TruncationPolicy;
@@ -97,7 +98,10 @@ impl SectionCost {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
self.text_bytes = self.text_bytes.saturating_add(text.len());
}
ContentItem::InputImage { image_url, .. } => {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => {
self.image_bytes = self.image_bytes.saturating_add(image_url.len());
self.image_count = self.image_count.saturating_add(1);
}
@@ -206,7 +210,11 @@ pub(super) fn content_framing_tokens(item_count: usize) -> usize {
fn adjusted_tokens(mut bytes: usize, content: &[ContentItem]) -> usize {
for item in content {
if let ContentItem::InputImage { image_url, .. } = item {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} = item
{
let payload = ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));
if payload == usize::MAX {
return usize::MAX;
codex-rs/guardian-context/src/budget_tests.rs
+7 -2
View File
@@ -4,6 +4,7 @@ use super::*;
use crate::Budgeted;
use crate::composition::SectionOutput;
use crate::composition::user_message as message;
use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[test]
@@ -17,7 +18,9 @@ fn section_costs_keep_multimodal_payloads_separate() {
text: "évidence".to_owned(),
}),
Budgeted::required(ContentItem::InputImage {
image_url: "data:image/png;base64,AAAA".to_owned(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAAA".to_owned(),
},
detail: None,
}),
]),
@@ -59,7 +62,9 @@ fn section_costs_keep_multimodal_payloads_separate() {
fn request_estimate_reserves_images_independently_of_encoded_size() {
let image = |payload: &str| {
message(vec![ContentItem::InputImage {
image_url: format!("data:image/png;base64,{payload}"),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{payload}"),
},
detail: None,
}])
};
codex-rs/guardian-context/src/composition.rs
+9 -4
View File
@@ -6,6 +6,7 @@
use codex_context_fragments::ContextualUserFragment;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::TruncationPolicy;
use codex_protocol::user_input::UserInput;
@@ -214,7 +215,10 @@ impl CollectedContext {
Ok(Budgeted::required(ContentItem::InputText { text }))
}
UserInput::Image { image_url, detail } => Ok(Budgeted::optional(
ContentItem::InputImage { image_url, detail },
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
},
BudgetPriority::Image,
)),
_ => Err(SectionError::UnsupportedDelivery {
@@ -281,9 +285,10 @@ impl ComposedContext {
}));
continue;
}
ContentItem::InputImage { image_url, detail } => {
UserInput::Image { image_url, detail }
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => UserInput::Image { image_url, detail },
ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {
return Err(SectionError::UnsupportedDelivery {
section: section.id,
codex-rs/guardian-context/src/enforcement.rs
+9 -2
View File
@@ -7,6 +7,7 @@ use std::collections::HashSet;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::protocol::TruncationPolicy;
use crate::ComposedContext;
@@ -90,7 +91,10 @@ impl ComposedContext {
pub fn retain_images(&mut self, mut admit: impl FnMut(&str, &mut Option<ImageDetail>) -> bool) {
for section in &mut self.sections {
retain_content(section, &mut self.truncations, |_, item| match item {
ContentItem::InputImage { image_url, detail } => admit(image_url, detail),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => admit(image_url, detail),
_ => true,
});
}
@@ -273,7 +277,10 @@ fn retain_content(
if !keep {
let original_bytes = match &item.content {
ContentItem::InputText { text } | ContentItem::OutputText { text } => text.len(),
ContentItem::InputImage { image_url, .. } => image_url.len(),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => image_url.len(),
ContentItem::InputAudio { audio_url } => audio_url.len(),
};
truncations.push(TruncationObservation {
codex-rs/guardian-context/src/enforcement_tests.rs
+13 -4
View File
@@ -3,6 +3,7 @@
use super::*;
use crate::budget::section_tokens;
use crate::composition::user_message;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
@@ -146,7 +147,9 @@ fn budget_reserves_existing_context_and_preserves_required_messages() {
.unwrap()
.into_message();
let image = ContentItem::InputImage {
image_url: "data:image/png;base64,AAAA".to_owned(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAAA".to_owned(),
},
detail: None,
};
let make_context = || ComposedContext {
@@ -241,7 +244,9 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
delivery: SectionDelivery::UserContent(vec![
Budgeted::optional(
ContentItem::InputImage {
image_url: "rejected-image".to_owned(),
image: ImageReference::Inline {
image_url: "rejected-image".to_owned(),
},
detail: None,
},
BudgetPriority::Image,
@@ -307,11 +312,15 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
);
let older = ContentItem::InputImage {
image_url: "older-image".to_owned(),
image: ImageReference::Inline {
image_url: "older-image".to_owned(),
},
detail: None,
};
let newer = ContentItem::InputImage {
image_url: "newer-image".to_owned(),
image: ImageReference::Inline {
image_url: "newer-image".to_owned(),
},
detail: None,
};
let image_section = |images: Vec<ContentItem>| SectionOutput {
codex-rs/guardian-context/src/history_tests.rs
+4 -1
View File
@@ -1,4 +1,5 @@
use codex_protocol::models::ExecutedToolCall;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ReasoningItemContent;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -147,7 +148,9 @@ fn oversized_user_images_preserve_text_and_metadata_in_order() {
content.insert(
/*index*/ 1,
ContentItem::InputImage {
image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)),
image: ImageReference::Inline {
image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)),
},
detail: Some(codex_protocol::models::ImageDetail::Original),
},
);
codex-rs/guardian-context/src/images.rs
+23 -6
View File
@@ -10,6 +10,7 @@ use crate::SectionScope;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use std::collections::VecDeque;
@@ -65,7 +66,11 @@ impl TranscriptImages {
while images.len() >= MAX_TRANSCRIPT_IMAGES
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
{
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
let Some(ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
}) = images.pop_front()
else {
break;
};
image_bytes -= image_url.len();
@@ -73,7 +78,9 @@ impl TranscriptImages {
}
image_bytes += image_url.len();
images.push_back(ContentItem::InputImage {
image_url: image_url.to_owned(),
image: ImageReference::Inline {
image_url: image_url.to_owned(),
},
detail,
});
};
@@ -84,7 +91,11 @@ impl TranscriptImages {
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
if let ContentItem::InputImage { image_url, detail } = item {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
include_image(image_url, *detail);
}
}
@@ -95,8 +106,10 @@ impl TranscriptImages {
{
if let Some(content) = output.content_items() {
for item in content {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
item
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
include_image(image_url, *detail);
}
@@ -108,7 +121,11 @@ impl TranscriptImages {
}
if input.include_tool_outputs {
for image in input.node_repl_images {
if let ContentItem::InputImage { image_url, detail } = image {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = image
{
include_image(image_url, *detail);
}
}
codex-rs/guardian-context/src/images_tests.rs
+7 -2
View File
@@ -5,12 +5,15 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
fn image(url: &str) -> ContentItem {
ContentItem::InputImage {
image_url: url.into(),
image: ImageReference::Inline {
image_url: url.into(),
},
detail: Some(ImageDetail::High),
}
}
@@ -32,7 +35,9 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "tool".into(),
image: ImageReference::Inline {
image_url: "tool".into(),
},
detail: Some(ImageDetail::High),
},
]),
codex-rs/protocol/src/models.rs
+54 -21
View File
@@ -879,7 +879,8 @@ pub enum ContentItem {
text: String,
},
InputImage {
image_url: String,
#[serde(flatten)]
image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
@@ -892,6 +893,13 @@ pub enum ContentItem {
},
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
#[serde(untagged)]
#[ts(untagged)]
pub enum ImageReference {
Inline { image_url: String },
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum AgentMessageInputContent {
@@ -1843,7 +1851,7 @@ fn local_image_content_items(
});
}
items.push(ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(detail),
});
if label_number.is_some() {
@@ -2006,7 +2014,7 @@ impl ResponseInputItem {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
vec![ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(detail),
}]
}
@@ -2080,7 +2088,8 @@ pub enum FunctionCallOutputContentItem {
},
// Do not rename, these are serialized and used directly in the responses API.
InputImage {
image_url: String,
#[serde(flatten)]
image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
@@ -2138,7 +2147,7 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
}
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url } => {
Self::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}
}
@@ -2383,7 +2392,7 @@ fn convert_mcp_content_to_items(
format!("data:{mime_type};base64,{data}")
};
FunctionCallOutputContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: meta
.as_ref()
.and_then(serde_json::Value::as_object)
@@ -2656,7 +2665,9 @@ mod tests {
assert_eq!(
content_item,
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,abc".to_string(),
},
detail: Some(ImageDetail::Auto),
}
);
@@ -2711,7 +2722,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,Zm9v".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,Zm9v".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3043,7 +3056,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,Zm9v".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,Zm9v".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3100,7 +3115,9 @@ mod tests {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -3119,7 +3136,9 @@ mod tests {
text: " ".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -3148,7 +3167,9 @@ mod tests {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]);
@@ -3423,7 +3444,9 @@ mod tests {
text: "caption".into(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BASE64".into(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
@@ -3498,7 +3521,9 @@ mod tests {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BASE64".into(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -3563,7 +3588,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BASE64".into(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3595,7 +3622,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BASE64".into(),
},
detail: Some(ImageDetail::Original),
}]
);
@@ -3627,7 +3656,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BASE64".into(),
},
detail: Some(ImageDetail::High),
}]
);
@@ -3650,7 +3681,9 @@ mod tests {
text: "note".into(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,XYZ".into(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,XYZ".into(),
},
detail: None,
},
];
@@ -3859,7 +3892,7 @@ mod tests {
match item {
ResponseInputItem::Message { content, .. } => {
let expected = vec![ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
assert_eq!(content, expected);
@@ -4005,7 +4038,7 @@ mod tests {
assert_eq!(
content.first(),
Some(&ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
})
);
@@ -4208,7 +4241,7 @@ mod tests {
assert_eq!(
content.first(),
Some(&ContentItem::InputImage {
image_url,
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
})
);
codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
+5 -1
View File
@@ -12,6 +12,7 @@ use std::collections::HashSet;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
@@ -492,7 +493,10 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
for item in content {
match item {
ContentItem::InputText { text: item_text } => text.push_str(item_text),
ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => images.push(image_url.as_str()),
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
ContentItem::OutputText { .. } => return false,
}
codex-rs/tools/src/image_detail_tests.rs
+13 -4
View File
@@ -2,6 +2,7 @@ use super::*;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::ModelInfo;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -88,11 +89,15 @@ fn sanitize_original_falls_back_to_high_without_support() {
text: "header".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string(),
},
detail: Some(ImageDetail::Original),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BBB".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BBB".to_string(),
},
detail: Some(ImageDetail::Low),
},
];
@@ -106,11 +111,15 @@ fn sanitize_original_falls_back_to_high_without_support() {
text: "header".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,AAA".to_string()
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BBB".to_string(),
image: ImageReference::Inline {
image_url: "data:image/png;base64,BBB".to_string()
},
detail: Some(ImageDetail::Low),
},
]
codex-rs/tools/src/tool_output.rs
+12 -11
View File
@@ -2,6 +2,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use serde_json::Value as JsonValue;
@@ -219,12 +220,13 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue
| codex_protocol::models::ContentItem::OutputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
codex_protocol::models::ContentItem::InputImage { image_url, detail } => {
FunctionCallOutputContentItem::InputImage {
image_url,
detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
}
}
codex_protocol::models::ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
},
codex_protocol::models::ContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
@@ -254,11 +256,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
FunctionCallOutputContentItem::InputText { text } if !text.trim().is_empty() => {
Some(text.clone())
}
FunctionCallOutputContentItem::InputImage { image_url, .. }
if !image_url.trim().is_empty() =>
{
Some(image_url.clone())
}
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} if !image_url.trim().is_empty() => Some(image_url.clone()),
FunctionCallOutputContentItem::InputAudio { audio_url }
if !audio_url.trim().is_empty() =>
{
codex-rs/utils/output-truncation/src/lib.rs
+16 -7
View File
@@ -3,6 +3,7 @@
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageReference;
pub use codex_utils_string::approx_bytes_for_tokens;
pub use codex_utils_string::approx_token_count;
pub use codex_utils_string::approx_tokens_from_byte_count;
@@ -90,12 +91,15 @@ pub fn formatted_truncate_text_content_items_with_policy(
text: formatted_truncate_text(&combined, policy),
}];
out.extend(items.iter().filter_map(|item| match item {
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
Some(FunctionCallOutputContentItem::InputImage {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => Some(FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.clone(),
detail: *detail,
})
}
},
detail: *detail,
}),
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Some(FunctionCallOutputContentItem::InputAudio {
audio_url: audio_url.clone(),
@@ -159,9 +163,14 @@ pub fn truncate_function_output_items_with_policy(
remaining_budget = 0;
}
}
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => {
out.push(FunctionCallOutputContentItem::InputImage {
image_url: image_url.clone(),
image: ImageReference::Inline {
image_url: image_url.clone(),
},
detail: *detail,
});
}
codex-rs/utils/output-truncation/src/truncate_tests.rs
+18 -7
View File
@@ -7,6 +7,7 @@ use crate::truncate_function_output_items_with_policy;
use crate::truncate_text;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[test]
@@ -114,7 +115,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() {
FunctionCallOutputContentItem::InputText { text: t1.clone() },
FunctionCallOutputContentItem::InputText { text: t2.clone() },
FunctionCallOutputContentItem::InputImage {
image_url: "img:mid".to_string(),
image: ImageReference::Inline {
image_url: "img:mid".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: t3 },
@@ -142,7 +145,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() {
assert_eq!(
output[2],
FunctionCallOutputContentItem::InputImage {
image_url: "img:mid".to_string(),
image: ImageReference::Inline {
image_url: "img:mid".to_string()
},
detail: Some(DEFAULT_IMAGE_DETAIL),
}
);
@@ -183,7 +188,9 @@ fn truncate_function_output_items_with_policy_discards_empty_text() {
text: "caption".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "img:one".to_string(),
image: ImageReference::Inline {
image_url: "img:one".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -253,7 +260,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "abcd".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "img:one".to_string(),
image: ImageReference::Inline {
image_url: "img:one".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -266,7 +275,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "ijkl".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "img:two".to_string(),
image: ImageReference::Inline {
image_url: "img:two".to_string(),
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
@@ -281,14 +292,14 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "Warning: truncated output (original token count: 4)\nTotal output lines: 3\n\nabcd…6 chars truncated…ijkl".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "img:one".to_string(),
image: ImageReference::Inline { image_url: "img:one".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "img:two".to_string(),
image: ImageReference::Inline { image_url: "img:two".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
sdk/python/scripts/update_sdk_artifacts.py
+18
View File
@@ -575,6 +575,7 @@ def generate_v2_all(schema_dir: Path) -> None:
],
cwd=sdk_root(),
)
_preserve_inline_image_class_names(out_path)
_require_nullable_chatgpt_account_email(out_path)
_preserve_reasoning_effort_enum(out_path)
_preserve_thread_source_enum(out_path)
@@ -582,6 +583,23 @@ def generate_v2_all(schema_dir: Path) -> None:
_normalize_generated_timestamps(out_path)
def _preserve_inline_image_class_names(out_path: Path) -> None:
"""Keep the public class names used before ImageReference was introduced."""
source = out_path.read_text()
stable_names = {
"ImageUrlContentItem": "InputImageContentItem",
"ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem",
}
for generated_name, stable_name in stable_names.items():
if source.count(f"class {generated_name}(") != 1:
raise RuntimeError(f"Generated SDK is missing a unique {generated_name} class")
if re.search(rf"\b{re.escape(stable_name)}\b", source):
raise RuntimeError(f"Generated SDK already defines {stable_name}")
source = re.sub(rf"\b{re.escape(generated_name)}\b", stable_name, source)
out_path.write_text(source)
def _require_nullable_chatgpt_account_email(out_path: Path) -> None:
"""Preserve required-but-nullable email semantics in the generated SDK model."""
source = out_path.read_text()
sdk/python/src/openai_codex/generated/v2_all.py
+3 -5
View File
@@ -7784,8 +7784,8 @@ class InputImageContentItem(BaseModel):
populate_by_name=True,
)
detail: ImageDetail | None = None
image_url: str
type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")]
image_url: str
class ContentItem(
@@ -7796,9 +7796,7 @@ class ContentItem(
model_config = ConfigDict(
populate_by_name=True,
)
root: (
InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
)
root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
class ExperimentalFeature(BaseModel):
@@ -7980,10 +7978,10 @@ class InputImageFunctionCallOutputContentItem(BaseModel):
populate_by_name=True,
)
detail: ImageDetail | None = None
image_url: str
type: Annotated[
Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType")
]
image_url: str
class FunctionCallOutputContentItem(
sdk/python/tests/test_artifact_workflow_and_binaries.py
+14
View File
@@ -536,6 +536,20 @@ def test_generated_chatgpt_account_email_is_required_nullable() -> None:
ChatgptAccount.model_validate({"planType": "pro", "type": "chatgpt"})
def test_generated_inline_image_class_names_remain_stable() -> None:
"""Keep the existing Python class names when image references expand."""
from openai_codex.generated.v2_all import (
InputImageContentItem,
InputImageFunctionCallOutputContentItem,
)
assert InputImageContentItem.__name__ == "InputImageContentItem"
assert (
InputImageFunctionCallOutputContentItem.__name__
== "InputImageFunctionCallOutputContentItem"
)
def test_runtime_package_template_has_no_checked_in_binaries() -> None:
runtime_root = ROOT.parent / "python-runtime" / "src" / "codex_cli_bin"
assert sorted(