mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Correlate code mode tool analytics with model responses (#36729)
## What changed - Emit dynamic tool-call analytics for code mode `exec` and `wait` calls, including duration and terminal status. - Associate code-mode cells and their nested tool calls with the originating and subsequent model response IDs. - Buffer correlated tool events until the next response is known, while flushing pending events when turns, threads, or the analytics queue close. ## Testing - Add reducer coverage for correlating `exec`, `wait`, and nested tool events across turns. - Add an app-server test that verifies production analytics include the cell and response IDs for a code-mode `exec` call. GitOrigin-RevId: 8b34ab59c1ad68ebbd57f9c8a05f19ab04df48a4
This commit is contained in:
1 parent
dae2122214
commit
155f1ca9e5
13 files changed
+797
-35
No files matched your search
@@ -47,6 +47,8 @@ use crate::facts::AnalyticsJsonRpcError;
|
||||
use crate::facts::AppInvocation;
|
||||
use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodeModeToolCallFact;
|
||||
use crate::facts::CodeModeToolCallStatus;
|
||||
use crate::facts::CodexCompactionEvent;
|
||||
use crate::facts::CodexErrKind;
|
||||
use crate::facts::CompactionImplementation;
|
||||
@@ -1665,6 +1667,10 @@ fn command_execution_event_serializes_expected_shape() {
|
||||
session_id: "session-thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
item_id: "item-1".to_string(),
|
||||
cell_id: None,
|
||||
parent_call_id: None,
|
||||
originating_response_id: None,
|
||||
subsequent_response_id: None,
|
||||
app_server_client: CodexAppServerClientMetadata {
|
||||
product_client_id: "codex_tui".to_string(),
|
||||
client_name: Some("codex-tui".to_string()),
|
||||
@@ -1717,6 +1723,10 @@ fn command_execution_event_serializes_expected_shape() {
|
||||
"session_id": "session-thread-1",
|
||||
"turn_id": "turn-1",
|
||||
"item_id": "item-1",
|
||||
"cell_id": null,
|
||||
"parent_call_id": null,
|
||||
"originating_response_id": null,
|
||||
"subsequent_response_id": null,
|
||||
"app_server_client": {
|
||||
"product_client_id": "codex_tui",
|
||||
"client_name": "codex-tui",
|
||||
@@ -2634,6 +2644,110 @@ async fn command_execution_approval_response_publishes_user_review_event() {
|
||||
assert_eq!(payload[0]["event_params"]["duration_ms"], 42);
|
||||
}
|
||||
|
||||
async fn ingest_code_mode_facts(
|
||||
reducer: &mut AnalyticsReducer,
|
||||
events: &mut Vec<TrackEventRequest>,
|
||||
facts: impl IntoIterator<Item = CodeModeToolCallFact>,
|
||||
) {
|
||||
for fact in facts {
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::CodeModeToolCall(fact)),
|
||||
events,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn sampling_response(
|
||||
turn_id: &str,
|
||||
response_id: &str,
|
||||
tool_call_ids: &[&str],
|
||||
) -> CodeModeToolCallFact {
|
||||
CodeModeToolCallFact::SamplingResponseCompleted {
|
||||
thread_id: "thread-1".into(),
|
||||
turn_id: turn_id.into(),
|
||||
response_id: response_id.into(),
|
||||
tool_call_ids: tool_call_ids.iter().map(|id| (*id).into()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn code_mode_exec_wait_and_child_events_share_cell_and_response_ids() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
ingest_review_prerequisites(&mut reducer, &mut events).await;
|
||||
|
||||
let completed =
|
||||
|turn_id: &str, call_id: &str, tool_name: &str| CodeModeToolCallFact::Completed {
|
||||
thread_id: "thread-1".into(),
|
||||
turn_id: turn_id.into(),
|
||||
call_id: call_id.into(),
|
||||
cell_id: Some("cell-1".into()),
|
||||
tool_name: tool_name.into(),
|
||||
started_at_ms: 1_000,
|
||||
completed_at_ms: 1_010,
|
||||
status: CodeModeToolCallStatus::Completed,
|
||||
};
|
||||
ingest_code_mode_facts(
|
||||
&mut reducer,
|
||||
&mut events,
|
||||
[
|
||||
sampling_response("turn-1", "resp-a", &["exec-1"]),
|
||||
CodeModeToolCallFact::CellStarted {
|
||||
thread_id: "thread-1".into(),
|
||||
turn_id: "turn-1".into(),
|
||||
call_id: "exec-1".into(),
|
||||
cell_id: "cell-1".into(),
|
||||
},
|
||||
CodeModeToolCallFact::ChildStarted {
|
||||
thread_id: "thread-1".into(),
|
||||
turn_id: "turn-1".into(),
|
||||
call_id: "child-1".into(),
|
||||
cell_id: "cell-1".into(),
|
||||
},
|
||||
completed("turn-1", "exec-1", "exec"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
ingest_completed_command_execution_item(&mut reducer, &mut events, "thread-1", "child-1").await;
|
||||
assert!(events.is_empty());
|
||||
|
||||
ingest_code_mode_facts(
|
||||
&mut reducer,
|
||||
&mut events,
|
||||
[
|
||||
sampling_response("turn-1", "resp-b", &[]),
|
||||
sampling_response("turn-2", "resp-c", &["wait-1"]),
|
||||
completed("turn-2", "wait-1", "wait"),
|
||||
sampling_response("turn-2", "resp-d", &[]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let actual = events
|
||||
.iter()
|
||||
.map(|event| {
|
||||
let event = serde_json::to_value(event).expect("serialize tool event");
|
||||
serde_json::json!({
|
||||
"item": event["event_params"]["item_id"],
|
||||
"cell": event["event_params"]["cell_id"],
|
||||
"parent": event["event_params"]["parent_call_id"],
|
||||
"origin": event["event_params"]["originating_response_id"],
|
||||
"subsequent": event["event_params"]["subsequent_response_id"],
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![
|
||||
serde_json::json!({"item":"exec-1","cell":"cell-1","parent":null,"origin":"resp-a","subsequent":"resp-b"}),
|
||||
serde_json::json!({"item":"child-1","cell":"cell-1","parent":"exec-1","origin":"resp-a","subsequent":"resp-b"}),
|
||||
serde_json::json!({"item":"wait-1","cell":"cell-1","parent":"exec-1","origin":"resp-c","subsequent":"resp-d"}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn permissions_reviews_emit_events_without_denormalizing_onto_tool_items() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
|
||||
Reference in new issue
Block a user