Preserve harness metadata across conversation history (#38058)

## What changed

- Wrap response items with optional harness-owned metadata while keeping the persisted response payload shape backward compatible.
- Store compacted-history metadata in an aligned sidecar and reject malformed sidecars.
- Carry metadata through history normalization, compaction, resume, fork, truncation, migration, and rollout recording without sending it to model providers.

## Testing

- Cover legacy rollout compatibility, metadata serialization and validation, compaction retention, and resume/fork reconstruction.

GitOrigin-RevId: 17a6127c9048ff984cb4df4d97c0751a38c29f5f

codex-rs/history/src/tests.rs

@@ -35,6 +35,7 @@ fn response_item_envelope_accessors_preserve_item() {
 }
 
 #[test]
+/// Keeps legacy response-item rollout lines readable and byte-shape compatible.
 fn response_item_rollout_line_preserves_shape() -> Result<()> {
     let legacy_line = json!({
         "timestamp": "2025-01-03T12:00:00.000Z",
@@ -51,15 +52,88 @@ fn response_item_rollout_line_preserves_shape() -> Result<()> {
     });
 
     let line = serde_json::from_value::<RolloutLine>(legacy_line.clone())?;
-    let RolloutItem::ResponseItem(item) = &line.item else {
+    let RolloutItem::ResponseItem(envelope) = &line.item else {
         panic!("expected response item");
     };
-    assert!(matches!(item, ResponseItem::Message { .. }));
+    assert!(matches!(&envelope.item, ResponseItem::Message { .. }));
+    assert_eq!(envelope.metadata, None);
     assert_eq!(serde_json::to_value(line)?, legacy_line);
     Ok(())
 }
 
 #[test]
+/// Keeps harness metadata beside, rather than inside, response-item payloads.
+fn response_item_envelope_stores_metadata_beside_rollout_payload() -> Result<()> {
+    let response_item = response_message("developer");
+    let line = RolloutLine {
+        timestamp: "2025-01-03T12:00:00.000Z".to_string(),
+        ordinal: Some(7),
+        item: RolloutItem::ResponseItem(ResponseItemEnvelope {
+            item: response_item.clone(),
+            metadata: Some(CodexHarnessMetadata {}),
+        }),
+    };
+    let serialized = serde_json::to_value(&line)?;
+
+    assert_eq!(
+        serialized,
+        json!({
+            "timestamp": "2025-01-03T12:00:00.000Z",
+            "ordinal": 7,
+            "type": "response_item",
+            "payload": response_item,
+            "metadata": {},
+        })
+    );
+    assert_eq!(serialized["payload"].get("metadata"), None);
+
+    let restored = serde_json::from_value::<RolloutLine>(serialized)?;
+    let RolloutItem::ResponseItem(envelope) = restored.item else {
+        panic!("expected response item");
+    };
+    assert_eq!(envelope.metadata, Some(CodexHarnessMetadata {}));
+    Ok(())
+}
+
+#[test]
+/// Keeps future metadata fields from making older binaries reject persisted items.
+fn response_item_envelope_ignores_unknown_harness_metadata_fields() -> Result<()> {
+    let line = serde_json::from_value::<RolloutLine>(json!({
+        "timestamp": "2025-01-03T12:00:00.000Z",
+        "ordinal": 7,
+        "type": "response_item",
+        "payload": {
+            "type": "message",
+            "role": "user",
+            "content": [{
+                "type": "input_text",
+                "text": "hello",
+            }],
+        },
+        "metadata": {
+            "future_field": "value",
+        },
+    }))?;
+
+    let RolloutItem::ResponseItem(envelope) = line.item else {
+        panic!("expected response item");
+    };
+    assert_eq!(envelope.metadata, Some(CodexHarnessMetadata {}));
+
+    let compacted = serde_json::from_value::<CompactedItem>(json!({
+        "message": "summary",
+        "replacement_history": [response_message("user")],
+        "replacement_history_metadata": [{ "future_field": "value" }],
+    }))?;
+    assert_eq!(
+        compacted.replacement_history.expect("replacement history")[0].metadata,
+        Some(CodexHarnessMetadata {})
+    );
+    Ok(())
+}
+
+#[test]
+/// Keeps legacy compacted replacement histories readable and shape-compatible.
 fn response_item_replacement_history_preserves_shape() -> Result<()> {
     let legacy_item = json!({
         "message": "summary",
@@ -79,14 +153,254 @@ fn response_item_replacement_history_preserves_shape() -> Result<()> {
         .as_ref()
         .expect("replacement history");
     assert!(matches!(
-        &replacement_history[0],
+        &replacement_history[0].item,
         ResponseItem::Message { .. }
     ));
+    assert_eq!(replacement_history[0].metadata, None);
     assert_eq!(serde_json::to_value(item)?, legacy_item);
     Ok(())
 }
 
 #[test]
+/// Stores complete aligned checkpoint metadata without modifying response items.
+fn compacted_replacement_history_stores_metadata_in_an_aligned_sidecar() -> Result<()> {
+    let developer_message = response_message("developer");
+    let compaction_item = ResponseItem::Compaction {
+        id: None,
+        encrypted_content: "opaque".to_string(),
+        internal_chat_message_metadata_passthrough: None,
+    };
+    let item = CompactedItem {
+        message: "summary".to_string(),
+        replacement_history: Some(vec![
+            ResponseItemEnvelope {
+                item: developer_message.clone(),
+                metadata: Some(CodexHarnessMetadata {}),
+            },
+            ResponseItemEnvelope::new(compaction_item.clone()),
+        ]),
+        window_number: None,
+        first_window_id: None,
+        previous_window_id: None,
+        window_id: None,
+    };
+
+    let serialized = serde_json::to_value(item)?;
+    assert_eq!(
+        serialized,
+        json!({
+            "message": "summary",
+            "replacement_history": [developer_message, compaction_item],
+            "replacement_history_metadata": [{}, {}],
+        })
+    );
+
+    let restored = serde_json::from_value::<CompactedItem>(serialized)?;
+    assert_eq!(
+        restored.replacement_history,
+        Some(vec![
+            ResponseItemEnvelope {
+                item: developer_message,
+                metadata: Some(CodexHarnessMetadata {}),
+            },
+            ResponseItemEnvelope {
+                item: compaction_item,
+                metadata: Some(CodexHarnessMetadata {}),
+            },
+        ])
+    );
+    Ok(())
+}
+
+#[test]
+/// Rejects checkpoint sidecars that cannot be paired unambiguously with history.
+fn compacted_replacement_history_rejects_misaligned_metadata() {
+    let malformed_items = [
+        json!({
+            "message": "summary",
+            "replacement_history": [response_message("user")],
+            "replacement_history_metadata": [],
+        }),
+        json!({
+            "message": "summary",
+            "replacement_history": [response_message("user")],
+            "replacement_history_metadata": [{}, {}],
+        }),
+        json!({
+            "message": "summary",
+            "replacement_history_metadata": [{}],
+        }),
+    ];
+
+    for malformed in malformed_items {
+        let error = serde_json::from_value::<CompactedItem>(malformed)
+            .expect_err("misaligned checkpoint metadata must be rejected");
+        assert!(
+            error.to_string().contains("replacement_history_metadata"),
+            "error: {error}"
+        );
+    }
+}
+
+#[test]
+/// Keeps annotated checkpoints readable by binaries expecting raw response items.
+fn compacted_metadata_remains_compatible_with_legacy_response_item_readers() -> Result<()> {
+    #[derive(Deserialize)]
+    #[serde(tag = "type", content = "payload", rename_all = "snake_case")]
+    enum LegacyRolloutItem {
+        ResponseItem(Box<ResponseItem>),
+        Compacted(LegacyCompactedItem),
+    }
+
+    #[derive(Deserialize)]
+    struct LegacyCompactedItem {
+        replacement_history: Vec<ResponseItem>,
+    }
+
+    let response_item = response_message("developer");
+    let envelope = ResponseItemEnvelope {
+        item: response_item.clone(),
+        metadata: Some(CodexHarnessMetadata {}),
+    };
+    let response_line = serde_json::to_value(RolloutItem::ResponseItem(envelope.clone()))?;
+    let LegacyRolloutItem::ResponseItem(legacy_response) =
+        serde_json::from_value::<LegacyRolloutItem>(response_line)?
+    else {
+        panic!("expected legacy response item");
+    };
+    assert_eq!(*legacy_response, response_item);
+
+    let compacted_line = serde_json::to_value(RolloutItem::Compacted(CompactedItem {
+        message: "summary".to_string(),
+        replacement_history: Some(vec![envelope]),
+        window_number: None,
+        first_window_id: None,
+        previous_window_id: None,
+        window_id: None,
+    }))?;
+
+    let LegacyRolloutItem::Compacted(legacy) =
+        serde_json::from_value::<LegacyRolloutItem>(compacted_line)?
+    else {
+        panic!("expected legacy compacted item");
+    };
+    assert_eq!(legacy.replacement_history, vec![response_item]);
+    Ok(())
+}
+
+#[test]
+/// Preserves the established tagged payload representation for every rollout variant.
+fn rollout_item_variants_preserve_existing_payload_shapes() -> Result<()> {
+    let cwd = std::env::current_dir()?;
+    let fixtures = [
+        json!({
+            "type": "session_meta",
+            "payload": SessionMetaLine {
+                meta: SessionMeta::default(),
+                git: None,
+            },
+        }),
+        json!({
+            "type": "response_item",
+            "payload": response_message("user"),
+        }),
+        json!({
+            "type": "inter_agent_communication",
+            "payload": {
+                "author": "/root",
+                "recipient": "/root/child",
+                "other_recipients": [],
+                "content": "hello",
+                "trigger_turn": false,
+            },
+        }),
+        json!({
+            "type": "inter_agent_communication_metadata",
+            "payload": { "trigger_turn": true },
+        }),
+        json!({
+            "type": "compacted",
+            "payload": { "message": "summary" },
+        }),
+        json!({
+            "type": "turn_context",
+            "payload": {
+                "cwd": cwd,
+                "approval_policy": "never",
+                "sandbox_policy": { "type": "danger-full-access" },
+                "model": "gpt-5",
+                "summary": "auto",
+            },
+        }),
+        json!({
+            "type": "world_state",
+            "payload": { "full": true, "state": { "cwd": "/tmp" } },
+        }),
+        json!({
+            "type": "event_msg",
+            "payload": { "type": "warning", "message": "heads up" },
+        }),
+    ];
+
+    for expected in fixtures {
+        let item = serde_json::from_value::<RolloutItem>(expected.clone())?;
+        assert_eq!(serde_json::to_value(item)?, expected);
+    }
+    Ok(())
+}
+
+#[test]
+/// Keeps the generated schema aligned with each variant's actual persisted shape.
+fn rollout_item_schema_matches_tagged_payload_and_sibling_metadata() -> Result<()> {
+    let schema = serde_json::to_value(schemars::schema_for!(RolloutItem))?;
+    let variants = schema["oneOf"].as_array().expect("rollout variants");
+    assert_eq!(variants.len(), 8);
+
+    for variant in variants {
+        let required = variant["required"].as_array().expect("required fields");
+        assert!(required.contains(&json!("type")), "schema: {variant}");
+        assert!(required.contains(&json!("payload")), "schema: {variant}");
+    }
+
+    let response_item = variants
+        .iter()
+        .find(|variant| variant["properties"]["type"]["enum"] == json!(["response_item"]))
+        .expect("response item schema");
+    assert!(response_item["properties"].get("metadata").is_some());
+    assert_eq!(
+        response_item["properties"]["payload"]["$ref"],
+        json!("#/definitions/ResponseItem")
+    );
+
+    let compacted = &schema["definitions"]["CompactedItem"];
+    assert_eq!(
+        compacted["properties"]["replacement_history"]["items"]["$ref"],
+        json!("#/definitions/ResponseItem")
+    );
+    assert_eq!(
+        compacted["properties"]["replacement_history_metadata"]["items"]["$ref"],
+        json!("#/definitions/CodexHarnessMetadata")
+    );
+    let required = compacted["required"].as_array().expect("required fields");
+    assert!(!required.contains(&json!("replacement_history")));
+    assert!(!required.contains(&json!("replacement_history_metadata")));
+    Ok(())
+}
+
+fn response_message(role: &str) -> ResponseItem {
+    ResponseItem::Message {
+        id: None,
+        role: role.to_string(),
+        content: vec![ContentItem::InputText {
+            text: "hello".to_string(),
+        }],
+        phase: None,
+        internal_chat_message_metadata_passthrough: None,
+    }
+}
+
+#[test]
+/// Preserves the stored compacted-item window metadata shape.
 fn compacted_item_serializes_window_number_and_id() -> Result<()> {
     let item = CompactedItem {
         message: "summary".to_string(),
@@ -111,6 +425,7 @@ fn compacted_item_serializes_window_number_and_id() -> Result<()> {
 }
 
 #[test]
+/// Keeps legacy numeric window IDs readable in stored compacted items.
 fn compacted_item_migrates_legacy_numeric_window_id() -> Result<()> {
     let item = serde_json::from_value::<CompactedItem>(json!({
         "message": "summary",