Attribute command and plugin analytics to the invoking model (#45445)

## Why

Command execution and plugin measurement events lack model and reasoning-effort labels. Attribution needs to reflect the step that invoked the command, even when model settings change before a background process finishes.

## What changed

- Add `model_slug` and `reasoning_effort` to command execution and plugin measurement analytics.
- Capture model context from resolved step settings and carry it through execution, approval, Guardian review, and plugin metrics collection.
- Preserve the first command-start model context when subsequent start notifications arrive.
- Keep the carried context out of serialized protocol items and generated schemas.

## Testing

Extend analytics tests to cover model switches before invocation and during background execution, default reasoning effort, Guardian-denied commands, and repeated start notifications retaining the original model context.

GitOrigin-RevId: af90e1c0d39bab625f2e89786085b61a9b96c0ce
This commit is contained in:
jwang-openai authored and copyberry committed 2026-09-14 15:30:22 +00:00
1 parent b6a5d5bb14
commit 4d8eca1ff3
40 files changed
+355 -43

No files matched your search

+123 -16
View File
@@ -14,11 +14,15 @@ use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxPolicy;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
use codex_app_server_protocol::ThreadReadResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::TurnSettingsUpdateParams;
use codex_app_server_protocol::TurnSettingsUpdateResponse;
use codex_app_server_protocol::TurnSettingsUpdateStatus;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::UserInput;
@@ -29,7 +33,10 @@ use codex_core::config::ConfigBuilder;
use codex_core_plugins::loader::curated_plugin_cache_version;
use codex_core_plugins::store::PluginStore;
use codex_features::Feature;
use codex_models_manager::bundled_models_response;
use codex_plugin::PluginId;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::openai_models::ReasoningEffort;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::skip_if_remote;
@@ -135,6 +142,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
text: "Review three commands".to_string(),
text_elements: Vec::new(),
}],
effort: Some(ReasoningEffort::High),
..Default::default()
})
.await?;
@@ -166,6 +174,19 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
timeout(READ_TIMEOUT, app_server.shutdown_gracefully()).await??;
let events = captured_analytics_events(&server).await;
assert!(!serde_json::to_string(&events)?.contains(PRIVATE));
let parent_command = events
.iter()
.find(|event| {
event["event_type"] == "codex_command_execution_event"
&& event["event_params"]["thread_id"] == thread.id
&& event["event_params"]["item_id"] == "parent-first"
})
.expect("denied parent command analytics");
let params = &parent_command["event_params"];
assert_eq!(
json!([params["model_slug"], params["reasoning_effort"]]),
json!(["mock-model", "high"])
);
let children = events
.iter()
.filter(|event| event["event_params"]["subagent_source"] == "guardian")
@@ -692,6 +713,7 @@ operations:
&script_path,
r#"test -n "$CODEX_PLUGIN_METRICS_OUTPUT"
sleep "${1:-0.3}"
while [ -n "${2:-}" ] && [ ! -f "$2" ]; do sleep 0.01; done
printf '%s' '{"version":1,"measurements":[{"name":"findings","value":3,"dimensions":{"severity":"high"}},{"name":"files_scanned","value":17}]}' > "$CODEX_PLUGIN_METRICS_OUTPUT"
"#,
)?;
@@ -724,12 +746,14 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
let codex_home = TempDir::new()?;
let script_path = write_curated_metrics_plugin(codex_home.path())?.canonicalize()?;
let release_path = codex_home.path().join("release-command");
let mut command = vec![
"/bin/sh".to_string(),
script_path.to_string_lossy().into_owned(),
];
if background {
command.push("1.0".to_string());
command.push(release_path.to_string_lossy().into_owned());
}
let call_id = "curated-plugin-metrics";
let arguments = serde_json::to_string(&json!({
@@ -742,8 +766,32 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
responses::ev_completed("resp-1"),
]);
let final_response = create_final_assistant_message_sse_response("done")?;
let server =
create_mock_responses_server_sequence(vec![command_response, final_response]).await;
let pause_response = |id: &str| {
responses::sse(vec![
responses::ev_response_created(id),
responses::ev_function_call(
id,
"request_user_input",
&json!({
"questions": [{
"id": "continue", "header": "Continue", "question": "Continue?",
"options": [
{"label": "Yes", "description": "Continue the turn."},
{"label": "No", "description": "Stop the turn."}
]
}]
})
.to_string(),
),
responses::ev_completed(id),
])
};
let mut response_sequence = vec![pause_response("before-command"), command_response];
if background {
response_sequence.push(pause_response("after-launch"));
}
response_sequence.push(final_response);
let server = create_mock_responses_server_sequence(response_sequence).await;
let analytics_server = responses::start_mock_server().await;
write_mock_responses_config_toml_with_chatgpt_base_url(
@@ -753,11 +801,40 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
)?;
let config_path = codex_home.path().join("config.toml");
let config = std::fs::read_to_string(&config_path)?;
let model = bundled_models_response()?
.models
.into_iter()
.find(|model| model.slug == "gpt-5.4")
.expect("bundled gpt-5.4 model");
let models = [
("initial-model", ReasoningEffort::Low),
("invoking-model", ReasoningEffort::High),
]
.into_iter()
.map(|(slug, effort)| {
let mut model = model.clone();
model.slug = slug.to_string();
model.default_reasoning_level = Some(effort);
model
})
.collect();
let catalog_path = codex_home.path().join("measurement-models.json");
std::fs::write(
&catalog_path,
serde_json::to_vec(&ModelsResponse { models })?,
)?;
let catalog_config = format!(
"model_catalog_json = {}",
serde_json::to_string(&catalog_path)?
);
std::fs::write(
config_path,
format!(
r#"{config}
r#"{catalog_config}
{config}
[features]
step_model_switching = true
default_mode_request_user_input = true
plugins = true
remote_plugin = false
unified_exec = true
@@ -787,7 +864,7 @@ enabled = true
timeout(Duration::from_secs(10), mcp.initialize()).await??;
let thread_request = mcp
.send_thread_start_request_with_auto_env(ThreadStartParams {
model: Some("mock-model".to_string()),
model: Some("initial-model".to_string()),
service_name: Some("codex_work_desktop".to_string()),
..Default::default()
})
@@ -813,11 +890,43 @@ enabled = true
..Default::default()
})
.await?;
timeout(
let turn_response = timeout(
Duration::from_secs(10),
mcp.read_stream_until_response_message(RequestId::Integer(turn_request)),
)
.await??;
let turn_response: TurnStartResponse = to_response(turn_response)?;
for model in if background {
vec!["invoking-model", "initial-model"]
} else {
vec!["invoking-model"]
} {
let request = timeout(
Duration::from_secs(10),
mcp.read_stream_until_request_message(),
)
.await??;
let ServerRequest::ToolRequestUserInput { request_id, .. } = request else {
anyhow::bail!("expected request_user_input, received {request:?}");
};
let response: TurnSettingsUpdateResponse = mcp
.request(|request_id| ClientRequest::TurnSettingsUpdate {
request_id,
params: TurnSettingsUpdateParams {
thread_id: thread_id.clone(),
turn_id: turn_response.turn.id.clone(),
model: Some(model.to_string()),
..Default::default()
},
})
.await?;
assert_eq!(response.status, TurnSettingsUpdateStatus::Applied);
mcp.send_response(
request_id,
json!({"answers": {"continue": {"answers": ["Yes"]}}}),
)
.await?;
}
let completed_turn = timeout(
Duration::from_secs(10),
mcp.read_stream_until_notification_message("turn/completed"),
@@ -830,17 +939,7 @@ enabled = true
.expect("completed turn id");
if background {
let model_request_bodies = server
.received_requests()
.await
.unwrap_or_default()
.into_iter()
.filter(|request| request.url.path().ends_with("/responses"))
.map(|request| serde_json::from_slice::<Value>(&request.body))
.collect::<Result<Vec<_>, _>>()?;
let request_text = serde_json::to_string(&model_request_bodies)?;
assert!(request_text.contains("Process running with session ID "));
assert!(!request_text.contains("Process exited with code 0"));
std::fs::write(release_path, "release")?;
}
for measurement_name in ["findings", "files_scanned"] {
@@ -859,12 +958,16 @@ enabled = true
.await?;
assert_eq!(
json!({
"model_slug": command_event["event_params"]["model_slug"],
"reasoning_effort": command_event["event_params"]["reasoning_effort"],
"plugin_id": command_event["event_params"]["plugin_id"],
"script_path": command_event["event_params"]["script_path"],
"item_id": command_event["event_params"]["item_id"],
"exit_code": command_event["event_params"]["exit_code"],
}),
json!({
"model_slug": "invoking-model",
"reasoning_effort": "high",
"plugin_id": METRICS_PLUGIN_ID,
"script_path": "scripts/run.sh",
"item_id": call_id,
@@ -952,10 +1055,12 @@ enabled = true
"execution_id",
"item_id",
"measurement_name",
"model_slug",
"number_value",
"operation",
"originator",
"plugin_id",
"reasoning_effort",
"thread_id",
"turn_id",
]
@@ -963,6 +1068,8 @@ enabled = true
assert_eq!(event_params["thread_id"], thread_id);
assert_eq!(event_params["turn_id"], turn_id);
assert_eq!(event_params["originator"], "codex_work_desktop");
assert_eq!(event_params["model_slug"], "invoking-model");
assert_eq!(event_params["reasoning_effort"], "high");
}
Ok(())