mirror of
https://github.com/openai/codex.git
synced 2026-11-07 16:35:32 +00:00
Add bounded plugin measurement analytics (#38239)
## What changed - Add `PluginMeasurementsInput` and `PluginMeasurementRow`, plus an analytics client method for recording batches and checking whether analytics are enabled. - Emit each valid row as a `codex_plugin_measurement_event` with thread, turn, item, plugin, execution, and operation context. - Bound batches at 100 rows and reject non-finite values, invalid identifiers, and rows with more than eight dimensions. - Treat measurement events as plugin events when routing API-key analytics. ## Testing - Cover batch limits, row filtering, serialized event payloads, and API-key plugin-event routing. GitOrigin-RevId: f949ed06246ee1e22466df2660076f2d4f556444
This commit is contained in:
1 parent
dc8562d672
commit
d6eefb26a6
7 files changed
+318
No files matched your search
@@ -20,6 +20,10 @@ use crate::events::CodexMcpToolCallEventParams;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::events::CodexMcpToolCallEventRequest;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::events::CodexPluginMeasurementEventParams;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::events::CodexPluginMeasurementEventRequest;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::events::CodexPluginMetadata;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::events::CodexPluginUsedEventRequest;
|
||||
@@ -49,9 +53,13 @@ use crate::facts::AnalyticsFact;
|
||||
use crate::facts::ArtifactOperation;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::facts::ArtifactOperationLifecycle;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::InvocationType;
|
||||
use crate::facts::PluginMeasurementRow;
|
||||
use crate::facts::PluginMeasurementsInput;
|
||||
#[cfg(debug_assertions)]
|
||||
use crate::facts::TrackEventsContext;
|
||||
use crate::reducer::MAX_PLUGIN_MEASUREMENTS_PER_BATCH;
|
||||
use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer;
|
||||
use codex_app_server_protocol::AskForApproval as AppServerAskForApproval;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
@@ -83,6 +91,7 @@ use codex_login::AuthManager;
|
||||
use codex_utils_absolute_path::test_support::PathBufExt;
|
||||
use codex_utils_absolute_path::test_support::test_path_buf;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::HashSet;
|
||||
#[cfg(debug_assertions)]
|
||||
use std::fs;
|
||||
@@ -407,6 +416,22 @@ async fn api_key_auth_sends_only_plugin_events_to_codex_backend() {
|
||||
let auth_manager = codex_login::AuthManager::from_auth_for_testing(
|
||||
codex_login::CodexAuth::from_api_key("sk-test"),
|
||||
);
|
||||
let plugin_measurement = |thread_id: &str, plugin_id: &str| {
|
||||
TrackEventRequest::PluginMeasurement(CodexPluginMeasurementEventRequest {
|
||||
event_type: "codex_plugin_measurement_event",
|
||||
event_params: CodexPluginMeasurementEventParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
item_id: "item-1".to_string(),
|
||||
plugin_id: plugin_id.to_string(),
|
||||
execution_id: "execution-1".to_string(),
|
||||
operation: "security_scan".to_string(),
|
||||
measurement_name: "findings".to_string(),
|
||||
number_value: 1.0,
|
||||
dimensions: None,
|
||||
},
|
||||
})
|
||||
};
|
||||
|
||||
send_track_events(
|
||||
&auth_manager,
|
||||
@@ -415,6 +440,7 @@ async fn api_key_auth_sends_only_plugin_events_to_codex_backend() {
|
||||
sample_regular_track_event("non-plugin-skill"),
|
||||
sample_mcp_tool_call_event("non-plugin-mcp", /*plugin_id*/ None),
|
||||
sample_plugin_used_track_event("non-plugin-used", /*plugin_id*/ None),
|
||||
plugin_measurement("non-plugin-measurement", /*plugin_id*/ ""),
|
||||
sample_accepted_line_fingerprint_event("other-event"),
|
||||
TrackEventRequest::ThreadArchive(ThreadArchiveEvent {
|
||||
event_type: "codex_thread_archive_event",
|
||||
@@ -428,6 +454,7 @@ async fn api_key_auth_sends_only_plugin_events_to_codex_backend() {
|
||||
sample_skill_track_event("plugin-skill", Some("sample@test")),
|
||||
sample_mcp_tool_call_event("plugin-mcp", Some("sample@test")),
|
||||
sample_artifact_operation_event("plugin-artifact"),
|
||||
plugin_measurement("plugin-measurement", "sample@test"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
@@ -482,6 +509,11 @@ async fn api_key_auth_sends_only_plugin_events_to_codex_backend() {
|
||||
"plugin_id": "presentations@openai-primary-runtime",
|
||||
"thread_id": "plugin-artifact",
|
||||
}),
|
||||
serde_json::json!({
|
||||
"event_type": "codex_plugin_measurement_event",
|
||||
"plugin_id": "sample@test",
|
||||
"thread_id": "plugin-measurement",
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -658,6 +690,66 @@ fn sample_turn_steer_response() -> ClientResponsePayload {
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_plugin_measurements_rejects_unbounded_inputs_before_queueing() {
|
||||
let (client, mut receiver) = client_with_receiver();
|
||||
let measurements = |row_count| PluginMeasurementsInput {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
item_id: "item-1".to_string(),
|
||||
plugin_id: "sample@openai-curated".to_string(),
|
||||
execution_id: "execution-1".to_string(),
|
||||
operation: "security_scan".to_string(),
|
||||
rows: vec![
|
||||
PluginMeasurementRow {
|
||||
measurement_name: "finding_count".to_string(),
|
||||
number_value: 1.0,
|
||||
dimensions: BTreeMap::new(),
|
||||
};
|
||||
row_count
|
||||
],
|
||||
};
|
||||
|
||||
client.track_plugin_measurements(measurements(MAX_PLUGIN_MEASUREMENTS_PER_BATCH + 1));
|
||||
assert!(matches!(receiver.try_recv(), Err(TryRecvError::Empty)));
|
||||
|
||||
let mut oversized_operation = measurements(1);
|
||||
oversized_operation.operation = "o".repeat(65);
|
||||
client.track_plugin_measurements(oversized_operation);
|
||||
assert!(matches!(receiver.try_recv(), Err(TryRecvError::Empty)));
|
||||
|
||||
let mut mixed_rows = measurements(4);
|
||||
mixed_rows.rows[0].measurement_name = "m".repeat(65);
|
||||
mixed_rows.rows[1]
|
||||
.dimensions
|
||||
.insert("d".repeat(65), "valid".to_string());
|
||||
mixed_rows.rows[2]
|
||||
.dimensions
|
||||
.insert("valid".to_string(), "v".repeat(65));
|
||||
client.track_plugin_measurements(mixed_rows);
|
||||
assert!(matches!(
|
||||
receiver.try_recv(),
|
||||
Ok(AnalyticsEventsQueueMessage::Fact(fact))
|
||||
if matches!(
|
||||
fact.as_ref(),
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::PluginMeasurements(input))
|
||||
if input.rows.len() == 1
|
||||
&& input.rows[0].measurement_name == "finding_count"
|
||||
)
|
||||
));
|
||||
|
||||
client.track_plugin_measurements(measurements(MAX_PLUGIN_MEASUREMENTS_PER_BATCH));
|
||||
assert!(matches!(
|
||||
receiver.try_recv(),
|
||||
Ok(AnalyticsEventsQueueMessage::Fact(fact))
|
||||
if matches!(
|
||||
fact.as_ref(),
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::PluginMeasurements(input))
|
||||
if input.rows.len() == MAX_PLUGIN_MEASUREMENTS_PER_BATCH
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_request_only_enqueues_analytics_relevant_requests() {
|
||||
let (client, mut receiver) = client_with_receiver();
|
||||
|
||||
Reference in new issue
Block a user