Attribute turn metrics to the models used during the turn (#44656)

## Why

Turn completion metrics used session-level telemetry, which could label usage with the wrong model after a model switch. A turn can also include compaction and responses from different models, so a single token total cannot attribute usage correctly.

## What changed

- Accumulate token usage by model using each response's resolved step settings, and emit one histogram sample per model and token type per turn.
- Preserve zero-valued samples when no usage is reported, without adding a sample for an unused selected model when only compaction ran.
- Use turn telemetry for tool-call, memory, network-proxy, and running-process metrics.

## Testing

Add regression tests for model switches, step-level model changes, compaction followed by a response or an aborted turn, and turns without responses. Verify repeated requests aggregate into one sample per model and earlier session usage is excluded.

GitOrigin-RevId: 03ab1dcde7da0b16b3a500a9a05e4c86583ee585
This commit is contained in:
Andrei Eternal authored and copyberry committed 2026-09-10 21:58:58 +00:00
1 parent 8570091e14
commit c8a8295e79
8 files changed
+352 -46

No files matched your search

+13 -1
View File
@@ -4533,7 +4533,7 @@ impl Session {
token_usage: Option<&TokenUsage>,
) -> CodexResult<()> {
let result = self
.record_token_usage_info(turn_context, token_usage)
.record_token_usage_info(turn_context, &turn_context.initial_settings, token_usage)
.await;
self.send_token_count_event(turn_context).await;
result
@@ -4576,6 +4576,7 @@ impl Session {
pub(crate) async fn record_token_usage_info(
&self,
turn_context: &TurnContext,
settings: &ResolvedStepSettings,
token_usage: Option<&TokenUsage>,
) -> CodexResult<()> {
if let Some(token_usage) = token_usage {
@@ -4591,6 +4592,17 @@ impl Session {
}
state.token_info()
};
let turn_state = self
.input_queue
.turn_state_for_sub_id(&self.active_turn, &turn_context.sub_id)
.await;
if let Some(turn_state) = turn_state {
turn_state.lock().await.token_usage_by_model.record(
settings.selected_collaboration_mode().model(),
settings.telemetry(&turn_context.session_telemetry),
token_usage,
);
}
let budget_result = self.record_rollout_budget_usage(token_usage);
if let Some(token_info) = token_info.as_ref() {
for contributor in self.services.extensions.token_usage_contributors() {
+10 -2
View File
@@ -3121,11 +3121,19 @@ async fn record_token_usage_info_notifies_extension_contributors() {
};
session
.record_token_usage_info(&turn_context, Some(&first_usage))
.record_token_usage_info(
&turn_context,
&turn_context.initial_settings,
Some(&first_usage),
)
.await
.expect("first usage should be recorded");
session
.record_token_usage_info(&turn_context, Some(&second_usage))
.record_token_usage_info(
&turn_context,
&turn_context.initial_settings,
Some(&second_usage),
)
.await
.expect("second usage should be recorded");
+5 -1
View File
@@ -2811,7 +2811,11 @@ async fn try_run_sampling_request(
)
.await;
let budget_result = sess
.record_token_usage_info(&turn_context, token_usage.as_ref())
.record_token_usage_info(
&turn_context,
&step_context.settings,
token_usage.as_ref(),
)
.await;
should_emit_token_count = true;
should_emit_turn_diff = true;