Track image preparation details in turn analytics (#36388)

## What changed

- Record the effective detail setting and source and prepared dimensions for
  successfully decoded images.
- Associate message images with their role and tool-output images with their
  originating call ID.
- Include the collected image preparation metadata in the corresponding turn
  analytics event.

## Testing

- Cover image resize metadata, tool-output attribution, turn aggregation, event
  serialization, and app-server analytics output.

GitOrigin-RevId: dbfb5ba1ce7cb02dd93d9b579c5109817f16aef5

codex-rs/core/src/image_preparation.rs

@@ -1,3 +1,5 @@
+use codex_analytics::ImageDetailSetting;
+use codex_analytics::ImagePreparationMetadata;
 use codex_protocol::models::ContentItem;
 use codex_protocol::models::FunctionCallOutputContentItem;
 use codex_protocol::models::ImageDetail;
@@ -24,6 +26,13 @@ const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits
     max_dimension: 6000,
     max_patches: 10_000,
 };
+
+#[derive(Clone, Copy, Debug)]
+struct ImageOrigin<'a> {
+    message_role: Option<&'a str>,
+    item_id: Option<&'a str>,
+}
+
 #[derive(Debug, thiserror::Error)]
 enum ImagePreparationError {
     #[error("remote image URLs are not supported")]
@@ -47,14 +56,35 @@ impl ImagePreparationError {
     }
 }
 
-pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
+pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) -> Vec<ImagePreparationMetadata> {
+    let mut metadata = Vec::new();
     for item in items {
         match item {
-            ResponseItem::Message { content, .. } => prepare_message_content(content),
-            ResponseItem::FunctionCallOutput { output, .. }
-            | ResponseItem::CustomToolCallOutput { output, .. } => {
+            ResponseItem::Message { role, content, .. } => {
+                prepare_message_content(
+                    content,
+                    ImageOrigin {
+                        message_role: Some(role),
+                        item_id: None,
+                    },
+                    &mut metadata,
+                );
+            }
+            ResponseItem::FunctionCallOutput {
+                call_id, output, ..
+            }
+            | ResponseItem::CustomToolCallOutput {
+                call_id, output, ..
+            } => {
                 if let Some(content) = output.content_items_mut() {
-                    prepare_tool_output_content(content);
+                    prepare_tool_output_content(
+                        content,
+                        ImageOrigin {
+                            message_role: None,
+                            item_id: Some(call_id),
+                        },
+                        &mut metadata,
+                    );
                 }
             }
             ResponseItem::AdditionalTools { .. }
@@ -73,12 +103,17 @@ pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
             | ResponseItem::Other => {}
         }
     }
+    metadata
 }
 
-fn prepare_message_content(items: &mut [ContentItem]) {
+fn prepare_message_content(
+    items: &mut [ContentItem],
+    origin: ImageOrigin<'_>,
+    metadata: &mut Vec<ImagePreparationMetadata>,
+) {
     for item in items {
         if let ContentItem::InputImage { image_url, detail } = item
-            && let Err(error) = prepare_image(image_url, *detail)
+            && let Err(error) = prepare_image(image_url, *detail, origin, metadata)
         {
             warn!(%error, "failed to prepare message image");
             *item = ContentItem::InputText {
@@ -88,10 +123,14 @@ fn prepare_message_content(items: &mut [ContentItem]) {
     }
 }
 
-fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
+fn prepare_tool_output_content(
+    items: &mut [FunctionCallOutputContentItem],
+    origin: ImageOrigin<'_>,
+    metadata: &mut Vec<ImagePreparationMetadata>,
+) {
     for item in items {
         if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item
-            && let Err(error) = prepare_image(image_url, *detail)
+            && let Err(error) = prepare_image(image_url, *detail, origin, metadata)
         {
             warn!(%error, "failed to prepare tool output image");
             *item = FunctionCallOutputContentItem::InputText {
@@ -116,6 +155,8 @@ fn is_data_url(image_url: &str) -> bool {
 fn prepare_image(
     image_url: &mut String,
     detail: Option<ImageDetail>,
+    origin: ImageOrigin<'_>,
+    metadata: &mut Vec<ImagePreparationMetadata>,
 ) -> Result<(), ImagePreparationError> {
     if is_remote_image_url(image_url) {
         return Err(ImagePreparationError::RemoteUrlUnsupported);
@@ -124,12 +165,23 @@ fn prepare_image(
         return Ok(());
     }
 
-    let limits = match detail {
-        None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS,
-        Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS,
+    let (effective_detail, limits) = match detail {
+        None | Some(ImageDetail::Auto | ImageDetail::High) => {
+            (ImageDetailSetting::High, HIGH_DETAIL_LIMITS)
+        }
+        Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS),
         Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
     };
     let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;
+    metadata.push(ImagePreparationMetadata {
+        message_role: origin.message_role.map(str::to_string),
+        item_id: origin.item_id.map(str::to_string),
+        effective_detail,
+        source_width: image.source_width,
+        source_height: image.source_height,
+        prepared_width: image.width,
+        prepared_height: image.height,
+    });
     *image_url = image.into_data_url();
     Ok(())
 }

codex-rs/core/src/image_preparation_tests.rs

@@ -69,12 +69,32 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
 
 #[test]
 fn detail_policies_apply_the_expected_budgets() {
-    for (detail, input_dimensions, expected_dimensions) in [
-        (Some(ImageDetail::High), (2048, 2048), (1600, 1600)),
-        (Some(ImageDetail::Original), (6401, 100), (6000, 94)),
-        (Some(ImageDetail::Original), (3201, 3201), (3200, 3200)),
-        (Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)),
-        (None, (2048, 2048), (1600, 1600)),
+    for (detail, effective_detail, input_dimensions, expected_dimensions) in [
+        (
+            Some(ImageDetail::High),
+            ImageDetailSetting::High,
+            (2048, 2048),
+            (1600, 1600),
+        ),
+        (
+            Some(ImageDetail::Original),
+            ImageDetailSetting::Original,
+            (6401, 100),
+            (6000, 94),
+        ),
+        (
+            Some(ImageDetail::Original),
+            ImageDetailSetting::Original,
+            (3201, 3201),
+            (3200, 3200),
+        ),
+        (
+            Some(ImageDetail::Auto),
+            ImageDetailSetting::High,
+            (2048, 2048),
+            (1600, 1600),
+        ),
+        (None, ImageDetailSetting::High, (2048, 2048), (1600, 1600)),
     ] {
         let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1);
         let mut items = vec![ResponseItem::Message {
@@ -85,7 +105,7 @@ fn detail_policies_apply_the_expected_budgets() {
             internal_chat_message_metadata_passthrough: None,
         }];
 
-        prepare_response_items(&mut items);
+        let metadata = prepare_response_items(&mut items);
 
         let ResponseItem::Message { content, .. } = &items[0] else {
             panic!("expected message");
@@ -94,9 +114,52 @@ fn detail_policies_apply_the_expected_budgets() {
             panic!("expected image");
         };
         assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
+        assert_eq!(
+            metadata,
+            vec![ImagePreparationMetadata {
+                message_role: Some("user".to_string()),
+                item_id: None,
+                effective_detail,
+                source_width: input_dimensions.0,
+                source_height: input_dimensions.1,
+                prepared_width: expected_dimensions.0,
+                prepared_height: expected_dimensions.1,
+            }]
+        );
     }
 }
 
+#[test]
+fn preparation_reports_tool_output_item_id() {
+    let call_id = "call-image";
+    let (image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
+    let mut items = vec![ResponseItem::FunctionCallOutput {
+        id: None,
+        call_id: call_id.to_string(),
+        output: FunctionCallOutputPayload::from_content_items(vec![
+            FunctionCallOutputContentItem::InputImage {
+                image_url,
+                detail: Some(ImageDetail::High),
+            },
+        ]),
+        internal_chat_message_metadata_passthrough: None,
+    }];
+    let metadata = prepare_response_items(&mut items);
+
+    assert_eq!(
+        metadata,
+        vec![ImagePreparationMetadata {
+            message_role: None,
+            item_id: Some(call_id.to_string()),
+            effective_detail: ImageDetailSetting::High,
+            source_width: 64,
+            source_height: 32,
+            prepared_width: 64,
+            prepared_height: 32,
+        }]
+    );
+}
+
 #[test]
 fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
     let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);

codex-rs/core/src/session/mod.rs

@@ -47,6 +47,8 @@ use async_channel::Sender;
 use chrono::Local;
 use chrono::Utc;
 use codex_analytics::AnalyticsEventsClient;
+use codex_analytics::ImagePreparationFact;
+use codex_analytics::ImagePreparationMetadata;
 use codex_analytics::SubAgentThreadStartedInput;
 use codex_analytics::TurnCodexErrorFact;
 use codex_async_utils::OrCancelExt;
@@ -2907,15 +2909,18 @@ impl Session {
         &self,
         turn_context: &TurnContext,
         items: &'a [ResponseItem],
-    ) -> Cow<'a, [ResponseItem]> {
+    ) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {
         let mut items = Cow::Borrowed(items);
-        prepare_image_response_items(items.to_mut());
+        let image_preparations = prepare_image_response_items(items.to_mut());
         prepare_audio_response_items(items.to_mut());
         // Most response items get their passthrough turn ID at the durable history boundary.
         for item in items.to_mut() {
             item.set_turn_id_if_missing(&turn_context.sub_id);
         }
-        Self::assign_missing_response_item_ids(items)
+        (
+            Self::assign_missing_response_item_ids(items),
+            image_preparations,
+        )
     }
 
     fn assign_missing_response_item_ids(items: Cow<'_, [ResponseItem]>) -> Cow<'_, [ResponseItem]> {
@@ -2963,7 +2968,8 @@ impl Session {
         turn_context: &TurnContext,
         items: &[ResponseItem],
     ) {
-        let items = self.prepare_conversation_items_for_history(turn_context, items);
+        let (items, image_preparations) =
+            self.prepare_conversation_items_for_history(turn_context, items);
         let items = items.as_ref();
         {
             let mut state = self.state.lock().await;
@@ -2973,6 +2979,14 @@ impl Session {
                 turn_context.model_info.truncation_policy.into(),
             );
         }
+        for image in image_preparations {
+            self.services
+                .analytics_events_client
+                .track_image_preparation(ImagePreparationFact {
+                    turn_id: turn_context.sub_id.clone(),
+                    metadata: image,
+                });
+        }
         self.persist_rollout_response_items(items).await;
         self.send_raw_response_items(turn_context, items).await;
     }
@@ -3124,7 +3138,7 @@ impl Session {
     ) {
         communication.set_turn_id_if_missing(&turn_context.sub_id);
         let response_item = communication.to_model_input_item();
-        let items = self.prepare_conversation_items_for_history(
+        let (items, _) = self.prepare_conversation_items_for_history(
             turn_context,
             std::slice::from_ref(&response_item),
         );

codex-rs/core/src/session/tests.rs

@@ -285,7 +285,7 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
     let (session, turn_context) = make_session_and_context().await;
     let response_item = user_message("hello");
 
-    let items = session.prepare_conversation_items_for_history(
+    let (items, _) = session.prepare_conversation_items_for_history(
         &turn_context,
         std::slice::from_ref(&response_item),
     );