mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
[codex-analytics] emit tool item events from item lifecycle (#17090)
## Why After the tool-item schemas are in place, analytics needs to emit them from the app-server item lifecycle rather than requiring bespoke tracking at each callsite. The reducer should also reuse the shared thread analytics context introduced below it in the stack so later event families do not repeat the same reducer joins or missing-state ladder. ## What changed - Tracks tool-item completion notifications and emits the matching tool analytics event when a terminal item arrives. - Derives event-specific payload details for command execution, file changes, MCP calls, dynamic tools, collaboration tools, web search, and image generation. - Denormalizes thread, app-server client, runtime, and subagent provenance metadata through the shared thread analytics context. - Adds reducer coverage for item lifecycle emission and subagent metadata inheritance. ## Duration semantics `duration_ms` is computed from the app-server item lifecycle timestamps: `completed_at_ms - started_at_ms`. That makes it the duration of the lifecycle Codex observed locally, not necessarily the upstream provider's full execution time. - Web search usually has a meaningful observed lifecycle because Responses can send `response.output_item.added` before `response.output_item.done`; in that case `started_at_ms` comes from the added event and `completed_at_ms` comes from the done event. - Image generation can be much less precise. In the current observed stream, image generation often arrives only as a completed `response.output_item.done`; when there is no earlier added event, Codex synthesizes the started item immediately before completion, so `duration_ms` can be `0` even though upstream image generation took longer. - Standalone web search and standalone image generation work is expected to land after this stack. Those paths may introduce more direct lifecycle events or timing points, so the current web-search/image-generation duration semantics should be treated as the best available item-lifecycle approximation, not the final latency contract for those tool families. - `execution_duration_ms` is populated only where the completed item already carries a native execution duration; otherwise it remains `null` while `duration_ms` still reflects the local lifecycle interval. ## Currently placeholder / partial fields Some fields are included in the schema for the intended steady-state contract, but this PR does not yet populate them from real approval/review state: - `review_count`, `guardian_review_count`, and `user_review_count` currently default to `0`. - `final_approval_outcome` currently defaults to `unknown`. - `requested_additional_permissions` and `requested_network_access` currently default to `false`. ## Verification - `cargo test -p codex-analytics` --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/17090). * #18748 * #18747 * __->__ #17090 * #17089 * #20514
This commit is contained in:
rhan-oai authored and GitHub committed
1 parent
21295f47e2
commit
fbdbc6b2fe
11 files changed
+980
-79
No files matched your search
@@ -12,7 +12,6 @@ use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexToolItemEventBase;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
use crate::events::CommandExecutionSource;
|
||||
use crate::events::GuardianApprovalRequestSource;
|
||||
use crate::events::GuardianReviewDecision;
|
||||
use crate::events::GuardianReviewEventParams;
|
||||
@@ -67,8 +66,13 @@ use codex_app_server_protocol::ClientInfo;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponsePayload;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::CommandAction;
|
||||
use codex_app_server_protocol::CommandExecutionSource;
|
||||
use codex_app_server_protocol::CommandExecutionStatus;
|
||||
use codex_app_server_protocol::InitializeCapabilities;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::ItemCompletedNotification;
|
||||
use codex_app_server_protocol::ItemStartedNotification;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::NonSteerableTurnKind;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
@@ -78,6 +82,7 @@ use codex_app_server_protocol::SessionSource as AppServerSessionSource;
|
||||
use codex_app_server_protocol::Thread;
|
||||
use codex_app_server_protocol::ThreadArchiveParams;
|
||||
use codex_app_server_protocol::ThreadArchiveResponse;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_app_server_protocol::ThreadSource as AppServerThreadSource;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
@@ -598,6 +603,90 @@ async fn ingest_turn_prerequisites(
|
||||
}
|
||||
}
|
||||
|
||||
async fn ingest_tool_review_prerequisites(
|
||||
reducer: &mut AnalyticsReducer,
|
||||
events: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
reducer
|
||||
.ingest(sample_initialize_fact(/*connection_id*/ 7), events)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ClientResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(1),
|
||||
response: Box::new(sample_thread_start_response(
|
||||
"thread-1", /*ephemeral*/ false, "gpt-5",
|
||||
)),
|
||||
},
|
||||
events,
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
}
|
||||
|
||||
fn sample_initialize_fact(connection_id: u64) -> AnalyticsFact {
|
||||
AnalyticsFact::Initialize {
|
||||
connection_id,
|
||||
params: InitializeParams {
|
||||
client_info: ClientInfo {
|
||||
name: "codex-tui".to_string(),
|
||||
title: None,
|
||||
version: "1.0.0".to_string(),
|
||||
},
|
||||
capabilities: Some(InitializeCapabilities {
|
||||
experimental_api: false,
|
||||
opt_out_notification_methods: None,
|
||||
}),
|
||||
},
|
||||
product_client_id: DEFAULT_ORIGINATOR.to_string(),
|
||||
runtime: CodexRuntimeMetadata {
|
||||
codex_rs_version: "0.99.0".to_string(),
|
||||
runtime_os: "linux".to_string(),
|
||||
runtime_os_version: "24.04".to_string(),
|
||||
runtime_arch: "x86_64".to_string(),
|
||||
},
|
||||
rpc_transport: AppServerRpcTransport::Websocket,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_command_execution_item(
|
||||
status: CommandExecutionStatus,
|
||||
exit_code: Option<i32>,
|
||||
duration_ms: Option<i64>,
|
||||
) -> ThreadItem {
|
||||
ThreadItem::CommandExecution {
|
||||
id: "item-1".to_string(),
|
||||
command: "echo hi".to_string(),
|
||||
cwd: test_path_buf("/tmp").abs(),
|
||||
process_id: Some("pid-1".to_string()),
|
||||
source: CommandExecutionSource::Agent,
|
||||
status,
|
||||
command_actions: Vec::new(),
|
||||
aggregated_output: None,
|
||||
exit_code,
|
||||
duration_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_command_execution_item_with_actions(
|
||||
status: CommandExecutionStatus,
|
||||
exit_code: Option<i32>,
|
||||
duration_ms: Option<i64>,
|
||||
command_actions: Vec<CommandAction>,
|
||||
) -> ThreadItem {
|
||||
let mut item = sample_command_execution_item(status, exit_code, duration_ms);
|
||||
let ThreadItem::CommandExecution {
|
||||
command_actions: item_command_actions,
|
||||
..
|
||||
} = &mut item
|
||||
else {
|
||||
unreachable!("sample command execution item should be CommandExecution");
|
||||
};
|
||||
*item_command_actions = command_actions;
|
||||
item
|
||||
}
|
||||
|
||||
fn expected_absolute_path(path: &PathBuf) -> String {
|
||||
std::fs::canonicalize(path)
|
||||
.unwrap_or_else(|_| path.to_path_buf())
|
||||
@@ -930,6 +1019,7 @@ fn command_execution_event_serializes_expected_shape() {
|
||||
started_at_ms: 123_000,
|
||||
completed_at_ms: 125_000,
|
||||
duration_ms: Some(2000),
|
||||
execution_duration_ms: Some(1900),
|
||||
review_count: 0,
|
||||
guardian_review_count: 0,
|
||||
user_review_count: 0,
|
||||
@@ -978,6 +1068,7 @@ fn command_execution_event_serializes_expected_shape() {
|
||||
"started_at_ms": 123000,
|
||||
"completed_at_ms": 125000,
|
||||
"duration_ms": 2000,
|
||||
"execution_duration_ms": 1900,
|
||||
"review_count": 0,
|
||||
"guardian_review_count": 0,
|
||||
"user_review_count": 0,
|
||||
@@ -1404,6 +1495,114 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() {
|
||||
assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn item_lifecycle_notifications_publish_command_execution_event() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
|
||||
ItemStartedNotification {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
started_at_ms: 1_000,
|
||||
item: sample_command_execution_item(
|
||||
CommandExecutionStatus::InProgress,
|
||||
/*exit_code*/ None,
|
||||
/*duration_ms*/ None,
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
events.is_empty(),
|
||||
"tool item event should emit on completion"
|
||||
);
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
completed_at_ms: 1_045,
|
||||
item: sample_command_execution_item_with_actions(
|
||||
CommandExecutionStatus::Completed,
|
||||
Some(0),
|
||||
Some(42),
|
||||
vec![
|
||||
CommandAction::Read {
|
||||
command: "cat README.md".to_string(),
|
||||
name: "README.md".to_string(),
|
||||
path: test_path_buf("/tmp/README.md").abs(),
|
||||
},
|
||||
CommandAction::ListFiles {
|
||||
command: "ls".to_string(),
|
||||
path: None,
|
||||
},
|
||||
CommandAction::Search {
|
||||
command: "rg TODO".to_string(),
|
||||
query: Some("TODO".to_string()),
|
||||
path: None,
|
||||
},
|
||||
CommandAction::Unknown {
|
||||
command: "cargo test".to_string(),
|
||||
},
|
||||
],
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
let payload = serde_json::to_value(&events).expect("serialize events");
|
||||
assert_eq!(payload.as_array().expect("events array").len(), 1);
|
||||
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
|
||||
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-1");
|
||||
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-1");
|
||||
assert_eq!(payload[0]["event_params"]["item_id"], "item-1");
|
||||
assert_eq!(payload[0]["event_params"]["tool_name"], "shell");
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["command_execution_source"],
|
||||
"agent"
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["terminal_status"], "completed");
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["final_approval_outcome"],
|
||||
"unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["failure_kind"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["exit_code"], 0);
|
||||
assert_eq!(payload[0]["event_params"]["command_total_action_count"], 4);
|
||||
assert_eq!(payload[0]["event_params"]["command_read_action_count"], 1);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["command_list_files_action_count"],
|
||||
1
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["command_search_action_count"], 1);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["command_unknown_action_count"],
|
||||
1
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["started_at_ms"], 1_000);
|
||||
assert_eq!(payload[0]["event_params"]["completed_at_ms"], 1_045);
|
||||
assert_eq!(payload[0]["event_params"]["duration_ms"], 45);
|
||||
assert_eq!(payload[0]["event_params"]["execution_duration_ms"], 42);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["app_server_client"]["client_name"],
|
||||
"codex-tui"
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["thread_source"], "user");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subagent_thread_started_review_serializes_expected_shape() {
|
||||
let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request(
|
||||
@@ -1687,6 +1886,79 @@ async fn subagent_thread_started_inherits_parent_connection_for_new_thread() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_tool_items_inherit_parent_connection_metadata() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::SubAgentThreadStarted(
|
||||
SubAgentThreadStartedInput {
|
||||
thread_id: "thread-subagent".to_string(),
|
||||
parent_thread_id: Some("thread-1".to_string()),
|
||||
product_client_id: "codex-tui".to_string(),
|
||||
client_name: "codex-tui".to_string(),
|
||||
client_version: "1.0.0".to_string(),
|
||||
model: "gpt-5".to_string(),
|
||||
ephemeral: false,
|
||||
subagent_source: SubAgentSource::Review,
|
||||
created_at: 128,
|
||||
},
|
||||
)),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
|
||||
ItemStartedNotification {
|
||||
thread_id: "thread-subagent".to_string(),
|
||||
turn_id: "turn-subagent".to_string(),
|
||||
started_at_ms: 1_000,
|
||||
item: sample_command_execution_item(
|
||||
CommandExecutionStatus::InProgress,
|
||||
/*exit_code*/ None,
|
||||
/*duration_ms*/ None,
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
|
||||
ItemCompletedNotification {
|
||||
thread_id: "thread-subagent".to_string(),
|
||||
turn_id: "turn-subagent".to_string(),
|
||||
completed_at_ms: 1_042,
|
||||
item: sample_command_execution_item(
|
||||
CommandExecutionStatus::Completed,
|
||||
Some(0),
|
||||
Some(42),
|
||||
),
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
let payload = serde_json::to_value(&events).expect("serialize events");
|
||||
assert_eq!(payload.as_array().expect("events array").len(), 1);
|
||||
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
|
||||
assert_eq!(payload[0]["event_params"]["thread_source"], "subagent");
|
||||
assert_eq!(payload[0]["event_params"]["subagent_source"], "review");
|
||||
assert_eq!(payload[0]["event_params"]["parent_thread_id"], "thread-1");
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["app_server_client"]["client_name"],
|
||||
"codex-tui"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plugin_used_event_serializes_expected_shape() {
|
||||
let tracking = TrackEventsContext {
|
||||
|
||||
Reference in new issue
Block a user