Preserve harness metadata across conversation history (#38058)

## What changed

- Wrap response items with optional harness-owned metadata while keeping the persisted response payload shape backward compatible.
- Store compacted-history metadata in an aligned sidecar and reject malformed sidecars.
- Carry metadata through history normalization, compaction, resume, fork, truncation, migration, and rollout recording without sending it to model providers.

## Testing

- Cover legacy rollout compatibility, metadata serialization and validation, compaction retention, and resume/fork reconstruction.

GitOrigin-RevId: 17a6127c9048ff984cb4df4d97c0751a38c29f5f

codex-rs/core/src/compact_tests.rs

@@ -1,4 +1,5 @@
 use super::*;
+use codex_history::CodexHarnessMetadata;
 use codex_history::ResponseItemEnvelope;
 use codex_protocol::ResponseItemId;
 use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
@@ -39,6 +40,17 @@ async fn process_compacted_history_with_test_session(
     (refreshed, initial_context)
 }
 
+fn annotated(items: Vec<ResponseItem>) -> Vec<ResponseItemEnvelope> {
+    items.into_iter().map(ResponseItemEnvelope::new).collect()
+}
+
+fn raw(items: Vec<ResponseItemEnvelope>) -> Vec<ResponseItem> {
+    items
+        .into_iter()
+        .map(ResponseItemEnvelope::into_item)
+        .collect()
+}
+
 fn user_message(text: &str) -> ResponseItem {
     ResponseItem::Message {
         id: None,
@@ -55,6 +67,7 @@ fn compacted_user_message(text: &str) -> CompactedUserMessage {
     CompactedUserMessage {
         message: text.to_string(),
         internal_chat_message_metadata_passthrough: None,
+        harness_metadata: None,
     }
 }
 
@@ -121,13 +134,23 @@ fn collect_user_messages_extracts_user_text_only() {
 #[test]
 fn collect_annotated_user_messages_extracts_user_text_only() {
     let items = vec![
-        ResponseItemEnvelope::new(user_message("first")),
+        ResponseItemEnvelope {
+            item: user_message("first"),
+            metadata: Some(CodexHarnessMetadata::default()),
+        },
         ResponseItemEnvelope::new(ResponseItem::Other),
     ];
 
     let collected = collect_annotated_user_messages(&items);
 
-    assert_eq!(vec![compacted_user_message("first")], collected);
+    assert_eq!(
+        vec![CompactedUserMessage {
+            message: "first".to_string(),
+            internal_chat_message_metadata_passthrough: None,
+            harness_metadata: Some(CodexHarnessMetadata::default()),
+        }],
+        collected
+    );
 }
 
 #[test]
@@ -198,7 +221,11 @@ fn build_token_limited_compacted_history_truncates_overlong_user_messages() {
     // that oversized user content is truncated.
     let max_tokens = 16;
     let big = "word ".repeat(200);
-    let user_message = compacted_user_message(&big);
+    let user_message = CompactedUserMessage {
+        message: big.clone(),
+        internal_chat_message_metadata_passthrough: None,
+        harness_metadata: Some(CodexHarnessMetadata::default()),
+    };
     let history = super::build_compacted_history_with_limit(
         Vec::new(),
         std::slice::from_ref(&user_message),
@@ -207,8 +234,8 @@ fn build_token_limited_compacted_history_truncates_overlong_user_messages() {
     );
     assert_eq!(history.len(), 2);
 
-    let truncated_message = &history[0];
-    let summary_message = &history[1];
+    let truncated_message = &history[0].item;
+    let summary_message = &history[1].item;
 
     let truncated_text = match truncated_message {
         ResponseItem::Message { role, content, .. } if role == "user" => {
@@ -233,11 +260,13 @@ fn build_token_limited_compacted_history_truncates_overlong_user_messages() {
         other => panic!("unexpected item in history: {other:?}"),
     };
     assert_eq!(summary_text, "SUMMARY");
+    assert_eq!(history[0].metadata, Some(CodexHarnessMetadata::default()));
+    assert_eq!(history[1].metadata, None);
 }
 
 #[test]
 fn build_token_limited_compacted_history_appends_summary_message() {
-    let initial_context: Vec<ResponseItem> = Vec::new();
+    let initial_context: Vec<ResponseItemEnvelope> = Vec::new();
     let user_messages = vec![compacted_user_message("first user message")];
     let summary_text = "summary text";
 
@@ -248,7 +277,7 @@ fn build_token_limited_compacted_history_appends_summary_message() {
     );
 
     let last = history.last().expect("history should have a summary entry");
-    let summary = match last {
+    let summary = match &last.item {
         ResponseItem::Message { role, content, .. } if role == "user" => {
             content_items_to_text(content).unwrap_or_default()
         }
@@ -269,12 +298,15 @@ fn build_compacted_history_preserves_user_message_passthrough_metadata() {
                     ..Default::default()
                 },
             ),
+            harness_metadata: Some(CodexHarnessMetadata::default()),
         }],
         "summary text",
     );
 
     assert_eq!(history[0].turn_id(), Some("turn-1"));
     assert_eq!(history[1].turn_id(), None);
+    assert_eq!(history[0].metadata, Some(CodexHarnessMetadata::default()));
+    assert_eq!(history[1].metadata, None);
 }
 
 #[tokio::test]
@@ -623,8 +655,10 @@ fn insert_initial_context_before_last_real_user_or_summary_keeps_summary_last()
         internal_chat_message_metadata_passthrough: None,
     }];
 
-    let refreshed =
-        insert_initial_context_before_last_real_user_or_summary(compacted_history, initial_context);
+    let refreshed = raw(insert_initial_context_before_last_real_user_or_summary(
+        annotated(compacted_history),
+        annotated(initial_context),
+    ));
     let expected = vec![
         ResponseItem::Message {
             id: None,
@@ -694,8 +728,10 @@ fn insert_initial_context_before_last_real_user_or_summary_keeps_compaction_last
         internal_chat_message_metadata_passthrough: None,
     }];
 
-    let refreshed =
-        insert_initial_context_before_last_real_user_or_summary(compacted_history, initial_context);
+    let refreshed = raw(insert_initial_context_before_last_real_user_or_summary(
+        annotated(compacted_history),
+        annotated(initial_context),
+    ));
     let expected = vec![
         ResponseItem::Message {
             id: None,