mirror of
https://github.com/openai/codex.git
synced 2026-11-09 00:39:32 +00:00
Attribute turn metrics to the models used during the turn (#44656)
## Why Turn completion metrics used session-level telemetry, which could label usage with the wrong model after a model switch. A turn can also include compaction and responses from different models, so a single token total cannot attribute usage correctly. ## What changed - Accumulate token usage by model using each response's resolved step settings, and emit one histogram sample per model and token type per turn. - Preserve zero-valued samples when no usage is reported, without adding a sample for an unused selected model when only compaction ran. - Use turn telemetry for tool-call, memory, network-proxy, and running-process metrics. ## Testing Add regression tests for model switches, step-level model changes, compaction followed by a response or an aborted turn, and turns without responses. Verify repeated requests aggregate into one sample per model and earlier session usage is excluded. GitOrigin-RevId: 03ab1dcde7da0b16b3a500a9a05e4c86583ee585
This commit is contained in:
1 parent
8570091e14
commit
c8a8295e79
8 files changed
+352
-46
No files matched your search
@@ -4533,7 +4533,7 @@ impl Session {
|
||||
token_usage: Option<&TokenUsage>,
|
||||
) -> CodexResult<()> {
|
||||
let result = self
|
||||
.record_token_usage_info(turn_context, token_usage)
|
||||
.record_token_usage_info(turn_context, &turn_context.initial_settings, token_usage)
|
||||
.await;
|
||||
self.send_token_count_event(turn_context).await;
|
||||
result
|
||||
@@ -4576,6 +4576,7 @@ impl Session {
|
||||
pub(crate) async fn record_token_usage_info(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
settings: &ResolvedStepSettings,
|
||||
token_usage: Option<&TokenUsage>,
|
||||
) -> CodexResult<()> {
|
||||
if let Some(token_usage) = token_usage {
|
||||
@@ -4591,6 +4592,17 @@ impl Session {
|
||||
}
|
||||
state.token_info()
|
||||
};
|
||||
let turn_state = self
|
||||
.input_queue
|
||||
.turn_state_for_sub_id(&self.active_turn, &turn_context.sub_id)
|
||||
.await;
|
||||
if let Some(turn_state) = turn_state {
|
||||
turn_state.lock().await.token_usage_by_model.record(
|
||||
settings.selected_collaboration_mode().model(),
|
||||
settings.telemetry(&turn_context.session_telemetry),
|
||||
token_usage,
|
||||
);
|
||||
}
|
||||
let budget_result = self.record_rollout_budget_usage(token_usage);
|
||||
if let Some(token_info) = token_info.as_ref() {
|
||||
for contributor in self.services.extensions.token_usage_contributors() {
|
||||
|
||||
@@ -3121,11 +3121,19 @@ async fn record_token_usage_info_notifies_extension_contributors() {
|
||||
};
|
||||
|
||||
session
|
||||
.record_token_usage_info(&turn_context, Some(&first_usage))
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&first_usage),
|
||||
)
|
||||
.await
|
||||
.expect("first usage should be recorded");
|
||||
session
|
||||
.record_token_usage_info(&turn_context, Some(&second_usage))
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&turn_context.initial_settings,
|
||||
Some(&second_usage),
|
||||
)
|
||||
.await
|
||||
.expect("second usage should be recorded");
|
||||
|
||||
|
||||
@@ -2811,7 +2811,11 @@ async fn try_run_sampling_request(
|
||||
)
|
||||
.await;
|
||||
let budget_result = sess
|
||||
.record_token_usage_info(&turn_context, token_usage.as_ref())
|
||||
.record_token_usage_info(
|
||||
&turn_context,
|
||||
&step_context.settings,
|
||||
token_usage.as_ref(),
|
||||
)
|
||||
.await;
|
||||
should_emit_token_count = true;
|
||||
should_emit_turn_diff = true;
|
||||
|
||||
Reference in new issue
Block a user