Forward history note images to the model (#41292)

## What changed

- Convert history backend `images` into `input_image` function-call output items alongside encrypted or plaintext history content.
- Keep image data out of logged output and post-tool-use hook responses.
- Reject malformed image attachments instead of silently dropping them.

## Testing

- Cover encrypted and plaintext outputs, supported image detail values, malformed attachments, and propagation into the next model request.

GitOrigin-RevId: a8eee10eb09c637bcfb06a759dd6313caf707e51

codex-rs/ext/history-notes/src/tools.rs

@@ -8,6 +8,7 @@ use codex_extension_api::ToolOutput;
 use codex_extension_api::ToolPayload;
 use codex_extension_api::ToolSpec;
 use codex_extension_api::parse_tool_input_schema;
+use codex_protocol::models::FunctionCallOutputBody;
 use codex_protocol::models::FunctionCallOutputContentItem;
 use codex_protocol::models::FunctionCallOutputPayload;
 use codex_protocol::models::ResponseInputItem;
@@ -280,7 +281,7 @@ impl HistoryNotesTool {
             .await
             .map_err(FunctionCallError::RespondToModel)?;
 
-        Ok(Box::new(HistoryNotesToolOutput { result }))
+        Ok(Box::new(HistoryNotesToolOutput::new(result)?))
     }
 }
 
@@ -324,6 +325,56 @@ impl<'call> ToolExecutor<ToolCall<'call>> for HistoryNotesTool {
 
 struct HistoryNotesToolOutput {
     result: Value,
+    output: FunctionCallOutputPayload,
+}
+
+impl HistoryNotesToolOutput {
+    fn new(mut result: Value) -> Result<Self, FunctionCallError> {
+        // Separate attachments before serializing any text or retaining log output.
+        let images = result.as_object_mut().and_then(|map| map.remove("images"));
+        // The server applies the requested output budget before encryption.
+        let mut output = match result.get("encrypted_output").and_then(Value::as_str) {
+            Some(encrypted_content) => FunctionCallOutputPayload::from_content_items(vec![
+                FunctionCallOutputContentItem::EncryptedContent {
+                    encrypted_content: encrypted_content.to_string(),
+                },
+            ]),
+            None => FunctionCallOutputPayload::from_text(result.to_string()),
+        };
+        if let Some(images) = images {
+            let invalid_image = || {
+                FunctionCallError::RespondToModel(
+                    "History backend returned invalid image content.".to_string(),
+                )
+            };
+            let images = images.as_array().ok_or_else(invalid_image)?;
+            let mut content = match output.body {
+                FunctionCallOutputBody::Text(text) => {
+                    vec![FunctionCallOutputContentItem::InputText { text }]
+                }
+                FunctionCallOutputBody::ContentItems(content) => content,
+            };
+            for image in images {
+                let data = image
+                    .get("data")
+                    .and_then(Value::as_str)
+                    .ok_or_else(invalid_image)?;
+                let mime_type = image
+                    .get("mime_type")
+                    .and_then(Value::as_str)
+                    .ok_or_else(invalid_image)?;
+                let detail =
+                    serde_json::from_value(image.get("detail").cloned().unwrap_or(Value::Null))
+                        .map_err(|_| invalid_image())?;
+                content.push(FunctionCallOutputContentItem::InputImage {
+                    image_url: format!("data:{mime_type};base64,{data}"),
+                    detail,
+                });
+            }
+            output = FunctionCallOutputPayload::from_content_items(content);
+        }
+        Ok(Self { result, output })
+    }
 }
 
 impl ToolOutput for HistoryNotesToolOutput {
@@ -335,20 +386,15 @@ impl ToolOutput for HistoryNotesToolOutput {
         true
     }
 
-    fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
-        // The server applies the requested output budget before encryption.
-        let output = match self.result.get("encrypted_output").and_then(Value::as_str) {
-            Some(encrypted_content) => FunctionCallOutputPayload::from_content_items(vec![
-                FunctionCallOutputContentItem::EncryptedContent {
-                    encrypted_content: encrypted_content.to_string(),
-                },
-            ]),
-            None => FunctionCallOutputPayload::from_text(self.result.to_string()),
-        };
+    fn post_tool_use_response(&self, _call_id: &str, _payload: &ToolPayload) -> Option<Value> {
+        // Hooks must not receive model-only image attachments.
+        Some(self.result.clone())
+    }
 
+    fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
         ResponseInputItem::FunctionCallOutput {
             call_id: call_id.to_string(),
-            output,
+            output: self.output.clone(),
         }
     }
 

codex-rs/ext/history-notes/src/tools_tests.rs

@@ -1,4 +1,6 @@
 use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageDetail;
 use codex_protocol::models::ResponseInputItem;
 use codex_tools::ToolOutput;
 use codex_tools::ToolPayload;
@@ -9,15 +11,14 @@ use super::HistoryNotesToolOutput;
 
 #[test]
 fn preserves_encrypted_history_output() {
-    let result = HistoryNotesToolOutput {
-        result: json!({"encrypted_output": "enc_payload"}),
-    }
-    .to_response_item(
-        "call-1",
-        &ToolPayload::Function {
-            arguments: "{}".to_string(),
-        },
-    );
+    let result = HistoryNotesToolOutput::new(json!({"encrypted_output": "enc_payload"}))
+        .expect("valid output")
+        .to_response_item(
+            "call-1",
+            &ToolPayload::Function {
+                arguments: "{}".to_string(),
+            },
+        );
 
     let ResponseInputItem::FunctionCallOutput { output, .. } = result else {
         panic!("expected function-call output");
@@ -32,3 +33,114 @@ fn preserves_encrypted_history_output() {
         )
     );
 }
+
+#[test]
+fn preserves_images_as_separate_output_items_without_logging_bytes() {
+    let output = HistoryNotesToolOutput::new(json!({
+        "encrypted_output": "enc_payload",
+        "images": [
+            {"data": "cG5n", "mime_type": "image/png", "detail": "original"},
+            {"data": "anBlZw==", "mime_type": "image/jpeg", "detail": "low"},
+            {"data": "Z2lm", "mime_type": "image/gif"},
+            {"data": "d2VicA==", "mime_type": "image/webp", "detail": null}
+        ]
+    }))
+    .expect("valid image output");
+    assert_eq!(
+        output.log_output(),
+        json!({"encrypted_output": "enc_payload"}).to_string()
+    );
+    assert_eq!(
+        output.post_tool_use_response(
+            "call-1",
+            &ToolPayload::Function {
+                arguments: "{}".to_string()
+            }
+        ),
+        Some(json!({"encrypted_output": "enc_payload"}))
+    );
+    assert_eq!(
+        output.to_response_item(
+            "call-1",
+            &ToolPayload::Function {
+                arguments: "{}".to_string()
+            }
+        ),
+        ResponseInputItem::FunctionCallOutput {
+            call_id: "call-1".to_string(),
+            output: FunctionCallOutputPayload::from_content_items(vec![
+                FunctionCallOutputContentItem::EncryptedContent {
+                    encrypted_content: "enc_payload".to_string()
+                },
+                FunctionCallOutputContentItem::InputImage {
+                    image_url: "data:image/png;base64,cG5n".to_string(),
+                    detail: Some(ImageDetail::Original)
+                },
+                FunctionCallOutputContentItem::InputImage {
+                    image_url: "data:image/jpeg;base64,anBlZw==".to_string(),
+                    detail: Some(ImageDetail::Low)
+                },
+                FunctionCallOutputContentItem::InputImage {
+                    image_url: "data:image/gif;base64,Z2lm".to_string(),
+                    detail: None
+                },
+                FunctionCallOutputContentItem::InputImage {
+                    image_url: "data:image/webp;base64,d2VicA==".to_string(),
+                    detail: None
+                },
+            ])
+        }
+    );
+}
+
+#[test]
+fn accepts_empty_attachments_and_legacy_plaintext_results() {
+    for (result, expected) in [
+        (
+            json!({"encrypted_output": "enc_payload", "images": []}),
+            FunctionCallOutputPayload::from_content_items(vec![
+                FunctionCallOutputContentItem::EncryptedContent {
+                    encrypted_content: "enc_payload".to_string(),
+                },
+            ]),
+        ),
+        (
+            json!({"text": "legacy result"}),
+            FunctionCallOutputPayload::from_text(json!({"text": "legacy result"}).to_string()),
+        ),
+    ] {
+        let output = HistoryNotesToolOutput::new(result).expect("valid output");
+        assert_eq!(
+            output.to_response_item(
+                "call-1",
+                &ToolPayload::Function {
+                    arguments: "{}".to_string()
+                }
+            ),
+            ResponseInputItem::FunctionCallOutput {
+                call_id: "call-1".to_string(),
+                output: expected
+            }
+        );
+    }
+}
+
+#[test]
+fn rejects_malformed_attachments_instead_of_silently_dropping_them() {
+    for images in [
+        json!(null),
+        json!({}),
+        json!([null]),
+        json!([{"mime_type": "image/png"}]),
+        json!([{"data": "private-image-bytes"}]),
+        json!([{"data": "private-image-bytes", "mime_type": "image/png", "detail": "invalid"}]),
+    ] {
+        let result = HistoryNotesToolOutput::new(
+            json!({"encrypted_output": "enc_payload", "images": images}),
+        );
+        let Err(codex_extension_api::FunctionCallError::RespondToModel(message)) = result else {
+            panic!("expected a model-facing image error");
+        };
+        assert_eq!(message, "History backend returned invalid image content.");
+    }
+}