Preserve harness metadata across conversation history (#38058)

## What changed

- Wrap response items with optional harness-owned metadata while keeping the persisted response payload shape backward compatible.
- Store compacted-history metadata in an aligned sidecar and reject malformed sidecars.
- Carry metadata through history normalization, compaction, resume, fork, truncation, migration, and rollout recording without sending it to model providers.

## Testing

- Cover legacy rollout compatibility, metadata serialization and validation, compaction retention, and resume/fork reconstruction.

GitOrigin-RevId: 17a6127c9048ff984cb4df4d97c0751a38c29f5f

codex-rs/core/src/compact.rs

@@ -28,6 +28,7 @@ use codex_analytics::CompactionStatus;
 use codex_analytics::CompactionStrategy;
 use codex_analytics::CompactionTrigger;
 use codex_analytics::now_unix_seconds;
+use codex_history::CodexHarnessMetadata;
 use codex_history::ResponseItemEnvelope;
 use codex_protocol::error::CodexErr;
 use codex_protocol::error::CodexErrorDetails;
@@ -85,7 +86,7 @@ pub(crate) struct CompactedHistoryMetadata {
 pub(crate) async fn build_compaction_initial_context(
     sess: &Session,
     initial_context_injection: &InitialContextInjection,
-) -> (Vec<ResponseItem>, Option<Arc<WorldState>>) {
+) -> (Vec<ResponseItemEnvelope>, Option<Arc<WorldState>>) {
     // Return the rendered state with its items so history and its baseline stay identical.
     match initial_context_injection {
         InitialContextInjection::BeforeLastUserMessage {
@@ -98,7 +99,10 @@ pub(crate) async fn build_compaction_initial_context(
                     world_state.as_ref(),
                 )
                 .await;
-            (items, Some(Arc::clone(world_state)))
+            (
+                items.into_iter().map(ResponseItemEnvelope::new).collect(),
+                Some(Arc::clone(world_state)),
+            )
         }
         InitialContextInjection::DoNotInject => (Vec::new(), None),
     }
@@ -517,11 +521,15 @@ pub fn content_items_to_text(content: &[ContentItem]) -> Option<String> {
 pub(crate) struct CompactedUserMessage {
     message: String,
     internal_chat_message_metadata_passthrough: Option<InternalChatMessageMetadataPassthrough>,
+    harness_metadata: Option<CodexHarnessMetadata>,
 }
 
 #[cfg(test)]
 pub(crate) fn collect_user_messages(items: &[ResponseItem]) -> Vec<CompactedUserMessage> {
-    items.iter().filter_map(compacted_user_message).collect()
+    items
+        .iter()
+        .filter_map(|item| compacted_user_message(item, /*harness_metadata*/ None))
+        .collect()
 }
 
 pub(crate) fn collect_annotated_user_messages(
@@ -529,11 +537,14 @@ pub(crate) fn collect_annotated_user_messages(
 ) -> Vec<CompactedUserMessage> {
     items
         .iter()
-        .filter_map(|envelope| compacted_user_message(&envelope.item))
+        .filter_map(|envelope| compacted_user_message(&envelope.item, envelope.metadata.clone()))
         .collect()
 }
 
-fn compacted_user_message(item: &ResponseItem) -> Option<CompactedUserMessage> {
+fn compacted_user_message(
+    item: &ResponseItem,
+    harness_metadata: Option<CodexHarnessMetadata>,
+) -> Option<CompactedUserMessage> {
     let Some(TurnItem::UserMessage(user)) = crate::event_mapping::parse_turn_item(item) else {
         return None;
     };
@@ -549,6 +560,7 @@ fn compacted_user_message(item: &ResponseItem) -> Option<CompactedUserMessage> {
             } => internal_chat_message_metadata_passthrough.clone(),
             _ => None,
         },
+        harness_metadata,
     })
 }
 
@@ -567,13 +579,13 @@ pub(crate) fn is_summary_message(message: &str) -> bool {
 ///   that item remains last (remote compaction may return only compaction items).
 /// - If there are no user messages or compaction items, append the context.
 pub(crate) fn insert_initial_context_before_last_real_user_or_summary(
-    mut compacted_history: Vec<ResponseItem>,
-    initial_context: Vec<ResponseItem>,
-) -> Vec<ResponseItem> {
+    mut compacted_history: Vec<ResponseItemEnvelope>,
+    initial_context: Vec<ResponseItemEnvelope>,
+) -> Vec<ResponseItemEnvelope> {
     let mut last_user_or_summary_index = None;
     let mut last_real_user_index = None;
     for (i, item) in compacted_history.iter().enumerate().rev() {
-        if let ResponseItem::AgentMessage { content, .. } = item
+        if let ResponseItem::AgentMessage { content, .. } = &item.item
             && !matches!(
                 content.first(),
                 Some(AgentMessageInputContent::InputText { text })
@@ -583,7 +595,8 @@ pub(crate) fn insert_initial_context_before_last_real_user_or_summary(
             last_real_user_index = Some(i);
             break;
         }
-        let Some(TurnItem::UserMessage(user)) = crate::event_mapping::parse_turn_item(item) else {
+        let Some(TurnItem::UserMessage(user)) = crate::event_mapping::parse_turn_item(&item.item)
+        else {
             continue;
         };
         // Compaction summaries are encoded as user messages, so track both:
@@ -601,7 +614,7 @@ pub(crate) fn insert_initial_context_before_last_real_user_or_summary(
         .rev()
         .find_map(|(i, item)| {
             matches!(
-                item,
+                &item.item,
                 ResponseItem::Compaction { .. } | ResponseItem::ContextCompaction { .. }
             )
             .then_some(i)
@@ -624,10 +637,10 @@ pub(crate) fn insert_initial_context_before_last_real_user_or_summary(
 }
 
 pub(crate) fn build_compacted_history(
-    initial_context: Vec<ResponseItem>,
+    initial_context: Vec<ResponseItemEnvelope>,
     user_messages: &[CompactedUserMessage],
     summary_text: &str,
-) -> Vec<ResponseItem> {
+) -> Vec<ResponseItemEnvelope> {
     build_compacted_history_with_limit(
         initial_context,
         user_messages,
@@ -637,11 +650,11 @@ pub(crate) fn build_compacted_history(
 }
 
 fn build_compacted_history_with_limit(
-    mut history: Vec<ResponseItem>,
+    mut history: Vec<ResponseItemEnvelope>,
     user_messages: &[CompactedUserMessage],
     summary_text: &str,
     max_tokens: usize,
-) -> Vec<ResponseItem> {
+) -> Vec<ResponseItemEnvelope> {
     let mut selected_messages: Vec<CompactedUserMessage> = Vec::new();
     if max_tokens > 0 {
         let mut remaining = max_tokens;
@@ -661,6 +674,7 @@ fn build_compacted_history_with_limit(
                     internal_chat_message_metadata_passthrough: message
                         .internal_chat_message_metadata_passthrough
                         .clone(),
+                    harness_metadata: message.harness_metadata.clone(),
                 });
                 break;
             }
@@ -669,16 +683,19 @@ fn build_compacted_history_with_limit(
     }
 
     for message in &selected_messages {
-        history.push(ResponseItem::Message {
-            id: None,
-            role: "user".to_string(),
-            content: vec![ContentItem::InputText {
-                text: message.message.clone(),
-            }],
-            phase: None,
-            internal_chat_message_metadata_passthrough: message
-                .internal_chat_message_metadata_passthrough
-                .clone(),
+        history.push(ResponseItemEnvelope {
+            item: ResponseItem::Message {
+                id: None,
+                role: "user".to_string(),
+                content: vec![ContentItem::InputText {
+                    text: message.message.clone(),
+                }],
+                phase: None,
+                internal_chat_message_metadata_passthrough: message
+                    .internal_chat_message_metadata_passthrough
+                    .clone(),
+            },
+            metadata: message.harness_metadata.clone(),
         });
     }
 
@@ -688,13 +705,13 @@ fn build_compacted_history_with_limit(
         summary_text.to_string()
     };
 
-    history.push(ResponseItem::Message {
+    history.push(ResponseItemEnvelope::new(ResponseItem::Message {
         id: None,
         role: "user".to_string(),
         content: vec![ContentItem::InputText { text: summary_text }],
         phase: None,
         internal_chat_message_metadata_passthrough: None,
-    });
+    }));
 
     history
 }