Persist response token usage in rollout history (#41912)

## Why

Resumed threads need to continue per-turn and per-thread token totals without scanning arbitrarily far beyond the latest compaction checkpoint.

## What changed

- Add durable `TokenUsageRecord` rollout items with response, turn, thread, session, and root-turn attribution.
- Restore accumulated usage on resume and snapshot the latest record plus the compaction response ID in compaction checkpoints.
- Preserve root-turn lineage in persisted turn context, while ensuring forked child threads start with their own usage totals.

## Testing

- Cover usage accumulation across multiple responses and resumed turns.
- Cover local and remote compaction checkpoints, invalid remote compaction output, rollout reconstruction, and fork isolation.

GitOrigin-RevId: ef9e0c4a9102a08a2c382be4cdac68c84353c90a

codex-rs/core/tests/suite/compact.rs

@@ -989,7 +989,7 @@ async fn manual_compact_uses_custom_prompt() {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn manual_compact_emits_api_and_local_token_usage_events() {
+async fn manual_compact_records_durable_and_local_token_usage() {
     skip_if_no_network!();
 
     let server = start_mock_server().await;
@@ -1008,7 +1008,9 @@ async fn manual_compact_emits_api_and_local_token_usage_events() {
         config.model_provider = model_provider;
         set_test_compact_prompt(config);
     });
-    let codex = builder.build(&server).await.unwrap().codex;
+    let test = builder.build(&server).await.unwrap();
+    let rollout_path = test.codex.rollout_path().expect("rollout path");
+    let codex = test.codex;
 
     // Trigger manual compact and collect TokenCount events for the compact turn.
     codex.submit(Op::Compact).await.unwrap();
@@ -1044,6 +1046,36 @@ async fn manual_compact_emits_api_and_local_token_usage_events() {
         last > 0,
         "second TokenCount should reflect a non-zero estimated context size after compaction"
     );
+    let rollout_items = fs::read_to_string(rollout_path)
+        .expect("read rollout")
+        .lines()
+        .filter_map(|line| serde_json::from_str::<RolloutLine>(line).ok())
+        .map(|line| line.item)
+        .collect::<Vec<_>>();
+    let records = rollout_items
+        .iter()
+        .filter_map(|item| match item {
+            RolloutItem::TokenUsageRecord(record) => Some(record),
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    assert_eq!(records.len(), 1);
+    assert_eq!(records[0].response_id, "r1");
+    assert_eq!(records[0].usage.total_tokens, 0);
+    let compacted = rollout_items
+        .iter()
+        .find_map(|item| match item {
+            RolloutItem::Compacted(compacted) => Some(compacted),
+            _ => None,
+        })
+        .expect("compaction checkpoint");
+    assert_eq!(
+        (
+            compacted.compaction_response_id.as_deref(),
+            compacted.latest_token_usage_record.as_ref(),
+        ),
+        (Some("r1"), records.first().copied())
+    );
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

codex-rs/core/tests/suite/compact_remote.rs

@@ -685,6 +685,67 @@ async fn remote_compact_v2_retains_only_client_developer_messages_when_enabled(
         })
         .count();
     assert_eq!(retained_client_developers, usize::from(enabled));
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn remote_compact_v2_records_usage_before_output_validation() -> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let harness = TestCodexHarness::with_auto_env_builder(
+        test_codex()
+            .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
+            .with_config(|config| {
+                config
+                    .features
+                    .enable(Feature::RemoteCompactionV2)
+                    .expect("remote compaction v2 should be configurable");
+            }),
+    )
+    .await?;
+    let codex = &harness.test().codex;
+    let rollout_path = codex.rollout_path().context("rollout path")?;
+    responses::mount_sse_sequence(
+        harness.server(),
+        vec![
+            sse(vec![responses::ev_completed("before-compact")]),
+            sse(vec![
+                json!({
+                    "type": "response.output_item.done",
+                    "item": {
+                        "type": "compaction",
+                        "encrypted_content": "FIRST_SUMMARY",
+                    },
+                }),
+                json!({
+                    "type": "response.output_item.done",
+                    "item": {
+                        "type": "compaction",
+                        "encrypted_content": "SECOND_SUMMARY",
+                    },
+                }),
+                responses::ev_completed_with_tokens("invalid-compact", /*total_tokens*/ 8_200),
+            ]),
+        ],
+    )
+    .await;
+
+    harness.test().submit_turn("before compact").await?;
+    codex.submit(Op::Compact).await?;
+    wait_for_event(codex, |event| matches!(event, EventMsg::Error(_))).await;
+    wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
+    codex.shutdown_and_wait().await?;
+
+    let record = fs::read_to_string(&rollout_path)?
+        .lines()
+        .filter_map(|line| serde_json::from_str::<RolloutLine>(line).ok())
+        .filter_map(|line| match line.item {
+            RolloutItem::TokenUsageRecord(record) => Some(record),
+            _ => None,
+        })
+        .find(|record| record.response_id == "invalid-compact")
+        .context("remote compaction usage record")?;
+    assert_eq!(record.usage.total_tokens, 8_200);
 
     Ok(())
 }

codex-rs/core/tests/suite/hooks.rs

@@ -1843,7 +1843,11 @@ async fn async_hook_finishing_while_idle_waits_for_the_next_turn(
     assert_ne!(first_turn_id, second_turn_id);
     responses::assert_root_turn(
         &requests[1].body_json(),
-        (!automatic_continuation).then_some(second_turn_id.as_str()),
+        Some(if automatic_continuation {
+            first_turn_id.as_str()
+        } else {
+            second_turn_id.as_str()
+        }),
     )?;
     assert_eq!(
         warning_event

codex-rs/core/tests/suite/mod.rs

@@ -166,6 +166,7 @@ mod stream_no_completed;
 mod subagent_notifications;
 mod subagent_service_tier;
 mod token_budget;
+mod token_usage_rollout;
 mod tool_harness;
 mod tool_lifecycle;
 mod tool_parallelism;

codex-rs/core/tests/suite/pending_input.rs

@@ -1102,8 +1102,8 @@ async fn injected_response_item_reopens_turn_after_final_answer() {
     responses::assert_root_turn(&first, Some(first_turn_id))
         .expect("initial root should be trusted");
     let second: Value = from_slice(&requests[1]).expect("parse second request");
-    responses::assert_root_turn(&second, /*expected*/ None)
-        .expect("external injection should invalidate the active turn root");
+    responses::assert_root_turn(&second, Some(first_turn_id))
+        .expect("external injection should preserve the active turn root");
     let relevant_user_input = message_input_texts(&second, "user")
         .into_iter()
         .filter(|text| {

codex-rs/core/tests/suite/resume_warning.rs

@@ -30,6 +30,7 @@ fn resume_history(
     let turn_id = "resume-warning-seed-turn".to_string();
     let turn_ctx = TurnContextItem {
         turn_id: Some(turn_id.clone()),
+        root_turn_id: None,
         cwd: config.cwd.clone(),
         workspace_roots: None,
         current_date: None,

codex-rs/core/tests/suite/token_usage_rollout.rs

@@ -0,0 +1,117 @@
+//! Verifies observed Responses API usage is durably recorded in rollout history.
+
+use anyhow::Result;
+use codex_history::RolloutItem;
+use codex_history::RolloutLine;
+use codex_protocol::SessionId;
+use codex_protocol::protocol::TokenUsageRecord;
+use core_test_support::responses::ev_assistant_message;
+use core_test_support::responses::ev_completed_with_tokens;
+use core_test_support::responses::ev_function_call;
+use core_test_support::responses::ev_response_created;
+use core_test_support::responses::mount_sse_sequence;
+use core_test_support::responses::sse;
+use core_test_support::responses::start_mock_server;
+use core_test_support::skip_if_no_network;
+use core_test_support::test_codex::test_codex;
+use pretty_assertions::assert_eq;
+use serde_json::json;
+
+fn token_usage_records(path: &std::path::Path) -> Vec<TokenUsageRecord> {
+    std::fs::read_to_string(path)
+        .expect("read rollout")
+        .lines()
+        .filter_map(|line| serde_json::from_str::<RolloutLine>(line).ok())
+        .filter_map(|line| match line.item {
+            RolloutItem::TokenUsageRecord(record) => Some(record),
+            _ => None,
+        })
+        .collect()
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn observed_response_usage_accumulates_per_turn_and_thread() -> Result<()> {
+    skip_if_no_network!(Ok(()));
+
+    let server = start_mock_server().await;
+    let plan_args = json!({
+        "plan": [{
+            "step": "keep sampling",
+            "status": "in_progress"
+        }]
+    })
+    .to_string();
+    mount_sse_sequence(
+        &server,
+        vec![
+            sse(vec![
+                ev_response_created("response-a"),
+                ev_function_call("call-a", "update_plan", &plan_args),
+                ev_completed_with_tokens("response-a", /*total_tokens*/ 120),
+            ]),
+            sse(vec![
+                ev_response_created("response-b"),
+                ev_assistant_message("message-b", "done"),
+                ev_completed_with_tokens("response-b", /*total_tokens*/ 80),
+            ]),
+            sse(vec![
+                ev_response_created("response-c"),
+                ev_assistant_message("message-c", "next"),
+                ev_completed_with_tokens("response-c", /*total_tokens*/ 30),
+            ]),
+            sse(vec![
+                ev_response_created("response-without-usage"),
+                ev_assistant_message("message-d", "no usage"),
+                json!({
+                    "type": "response.completed",
+                    "response": {
+                        "id": "response-without-usage"
+                    }
+                }),
+            ]),
+        ],
+    )
+    .await;
+    let test = test_codex().build_with_auto_env(&server).await?;
+    let rollout_path = test.codex.rollout_path().expect("rollout path");
+    let home = test.home.clone();
+
+    test.submit_turn("first").await?;
+    test.codex.shutdown_and_wait().await?;
+
+    let resumed = test_codex()
+        .resume(&server, home, rollout_path.clone())
+        .await?;
+    for prompt in ["second", "third"] {
+        resumed.submit_turn(prompt).await?;
+    }
+    resumed.codex.shutdown_and_wait().await?;
+
+    let records = token_usage_records(&rollout_path);
+    assert_eq!(records.len(), 3);
+    assert_eq!(
+        records
+            .iter()
+            .map(|record| {
+                (
+                    record.response_id.as_str(),
+                    record.turn_token_usage.total_tokens,
+                    record.thread_token_usage.total_tokens,
+                )
+            })
+            .collect::<Vec<_>>(),
+        vec![
+            ("response-a", 120, 120),
+            ("response-b", 200, 200),
+            ("response-c", 30, 230),
+        ]
+    );
+    assert_eq!(records[0].turn_id, records[1].turn_id);
+    assert_ne!(records[1].turn_id, records[2].turn_id);
+    assert!(records.iter().all(|record| {
+        record.session_id == SessionId::from(record.thread_id)
+            && record.root_turn_id == record.turn_id
+    }));
+
+    Ok(())
+}