mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
[codex-analytics] add turn tool counts to turn events (#21431)
## Summary - accumulate completed tool-item counts per turn from the item lifecycle - populate the reserved count fields on `codex_turn_event` - add reducer coverage for zero-count turns and mixed completed tool items ## Why PR #17090 moved tool-item analytics onto the item lifecycle, so the turn reducer can now derive the per-turn tool counts from the same completed items instead of leaving the reserved fields null. ## Validation - `just fmt` - `cargo test -p codex-analytics`
This commit is contained in:
1 parent
0dbad2a348
commit
cf6342b75b
3 files changed
+246
-10
No files matched your search
@@ -69,15 +69,20 @@ use codex_app_server_protocol::ClientInfo;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponsePayload;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::CollabAgentTool;
|
||||
use codex_app_server_protocol::CollabAgentToolCallStatus;
|
||||
use codex_app_server_protocol::CommandAction;
|
||||
use codex_app_server_protocol::CommandExecutionSource;
|
||||
use codex_app_server_protocol::CommandExecutionStatus;
|
||||
use codex_app_server_protocol::DynamicToolCallStatus;
|
||||
use codex_app_server_protocol::InitializeCapabilities;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::ItemCompletedNotification;
|
||||
use codex_app_server_protocol::ItemStartedNotification;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::McpToolCallStatus;
|
||||
use codex_app_server_protocol::NonSteerableTurnKind;
|
||||
use codex_app_server_protocol::PatchApplyStatus;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
@@ -1709,6 +1714,14 @@ async fn item_lifecycle_notifications_publish_command_execution_event() {
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
|
||||
"thread-1", "turn-1",
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
|
||||
@@ -2119,6 +2132,15 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() {
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
|
||||
"thread-subagent",
|
||||
"turn-subagent",
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
@@ -2993,6 +3015,17 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||
assert_eq!(payload["event_params"]["status"], json!("completed"));
|
||||
assert_eq!(payload["event_params"]["steer_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["total_tool_call_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["shell_command_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["file_change_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(0));
|
||||
assert_eq!(
|
||||
payload["event_params"]["subagent_tool_call_count"],
|
||||
json!(0)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["web_search_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["image_generation_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["started_at"], json!(455));
|
||||
assert_eq!(payload["event_params"]["completed_at"], json!(456));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
@@ -3006,6 +3039,158 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_event_counts_completed_tool_items() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
|
||||
let completed_tool_items = vec![
|
||||
sample_command_execution_item(CommandExecutionStatus::Completed, Some(0), Some(1)),
|
||||
ThreadItem::FileChange {
|
||||
id: "file-change-1".to_string(),
|
||||
changes: Vec::new(),
|
||||
status: PatchApplyStatus::Completed,
|
||||
},
|
||||
ThreadItem::McpToolCall {
|
||||
id: "mcp-1".to_string(),
|
||||
server: "server".to_string(),
|
||||
tool: "search".to_string(),
|
||||
status: McpToolCallStatus::Completed,
|
||||
arguments: json!({}),
|
||||
mcp_app_resource_uri: None,
|
||||
result: None,
|
||||
error: None,
|
||||
duration_ms: Some(2),
|
||||
},
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "dynamic-1".to_string(),
|
||||
namespace: None,
|
||||
tool: "render".to_string(),
|
||||
arguments: json!({}),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: None,
|
||||
success: Some(true),
|
||||
duration_ms: Some(3),
|
||||
},
|
||||
ThreadItem::CollabAgentToolCall {
|
||||
id: "collab-1".to_string(),
|
||||
tool: CollabAgentTool::SpawnAgent,
|
||||
status: CollabAgentToolCallStatus::Completed,
|
||||
sender_thread_id: "thread-2".to_string(),
|
||||
receiver_thread_ids: vec!["thread-child".to_string()],
|
||||
prompt: Some("help".to_string()),
|
||||
model: Some("gpt-5".to_string()),
|
||||
reasoning_effort: None,
|
||||
agents_states: Default::default(),
|
||||
},
|
||||
ThreadItem::WebSearch {
|
||||
id: "web-1".to_string(),
|
||||
query: "codex".to_string(),
|
||||
action: None,
|
||||
},
|
||||
ThreadItem::ImageGeneration {
|
||||
id: "image-1".to_string(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: None,
|
||||
result: "ok".to_string(),
|
||||
saved_path: None,
|
||||
},
|
||||
];
|
||||
|
||||
for item in completed_tool_items {
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
completed_at_ms: 1_000,
|
||||
item,
|
||||
},
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
let turn_event = out
|
||||
.iter()
|
||||
.find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
|
||||
.expect("turn event should be emitted");
|
||||
let payload = serde_json::to_value(turn_event).expect("serialize turn event");
|
||||
assert_eq!(payload["event_params"]["total_tool_call_count"], json!(7));
|
||||
assert_eq!(payload["event_params"]["shell_command_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["file_change_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(1));
|
||||
assert_eq!(
|
||||
payload["event_params"]["subagent_tool_call_count"],
|
||||
json!(1)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["web_search_count"], json!(1));
|
||||
assert_eq!(payload["event_params"]["image_generation_count"], json!(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn item_completed_without_turn_state_does_not_create_turn_state() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
completed_at_ms: 1_000,
|
||||
item: sample_command_execution_item(
|
||||
CommandExecutionStatus::Completed,
|
||||
Some(0),
|
||||
Some(1),
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_steers_increment_turn_steer_count() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
|
||||
Reference in new issue
Block a user