Add Guardian V2 analytics events (#42144)

## What changed

- Emit `codex_guardian_v2_classification` events with the outcome, risk
  level, duration, model, and thread attribution.
- Emit `codex_guardian_v2_fast_decision` events when Guardian V2 approves an
  action without a full review.
- Add `guardian_v2_enabled` to turn analytics, accounting for review scope
  and model requirements.

## Testing

- Extend the Guardian V2 app-server coverage to verify event payloads,
  attribution, enablement state, and scoped event emission.

GitOrigin-RevId: 0805975f8e8f9d954aa1058ea8c60545a090ae74
This commit is contained in:
jif authored and copyberry committed 2026-09-01 22:02:42 +00:00
1 parent a876a9e494
commit 86b7d12742
17 files changed
+272 -20

No files matched your search

+1
View File
@@ -13,6 +13,7 @@ doctest = false
workspace = true
[dependencies]
codex-analytics = { workspace = true }
codex-api = { workspace = true }
codex-core = { workspace = true }
codex-extension-api = { workspace = true }
@@ -7,6 +7,9 @@ use std::sync::atomic::Ordering;
use std::time::Instant;
use std::time::SystemTime;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::GuardianV2Event;
use codex_analytics::GuardianV2EventKind;
use codex_core::ThreadManager;
use codex_core::config::Config;
use codex_core::context::GuardianReviewEvidence;
@@ -18,6 +21,7 @@ use codex_extension_api::ExtensionFuture;
use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ExtensionWarning;
use codex_extension_api::GuardianV2Enabled;
use codex_extension_api::ResponseItem;
use codex_extension_api::SkillInvocationContributor;
use codex_extension_api::SkillInvocationInput;
@@ -103,8 +107,6 @@ pub enum StrictReviewReason {
StaleScore,
}
struct GuardianV2Enabled;
enum ClassificationOutcome {
Scored,
Superseded,
@@ -196,6 +198,10 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
let sampler = input
.thread_store
.get_or_init(|| LunaSampler::new(sampler_config));
let guardian_v2_enabled = GuardianV2Enabled {
computer_use_only: guardian_config.review_scope
== GuardianV2ReviewScope::ComputerUseOnly,
};
input.thread_store.insert(guardian_config);
input.thread_store.insert(GuardianV2ScoreProgress {
metrics: input.extension_metrics.clone(),
@@ -205,7 +211,7 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
input
.thread_store
.insert(TrustedSkillRoots::from_config(input.config));
input.thread_store.insert(GuardianV2Enabled);
input.thread_store.insert(guardian_v2_enabled);
tokio::spawn(async move {
sampler.prewarm().await;
@@ -431,6 +437,7 @@ impl GuardianV2Extension {
.fetch_add(/*val*/ 1, Ordering::Relaxed);
}
let metrics = score_progress.metrics.clone();
let analytics = input.session_store.get::<AnalyticsEventsClient>();
let sampled_at = SystemTime::now();
let tool_call_index = score_progress
.latest_tool_call
@@ -673,6 +680,7 @@ impl GuardianV2Extension {
format!("{planned_action}\n"),
">>> APPROVAL REQUEST END\n".to_owned(),
]);
let mut classification_risk = None;
let mut classification_finished_at = None;
let result: Result<ClassificationOutcome, String> = async {
let review_model_messages = if config.guardian_policy_config.is_none() {
@@ -722,11 +730,12 @@ impl GuardianV2Extension {
}
Err(error) => return Err(error.to_string()),
};
let action_risk = match output.as_str() {
"high" => 1.0,
"low" => 0.0,
let (action_risk, risk_level) = match output.as_str() {
"high" => (1.0, "high"),
"low" => (0.0, "low"),
_ => return Err("invalid Guardian V2 classification".to_owned()),
};
classification_risk = Some(risk_level);
let score = SecurityRiskScore {
scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]),
call_id: Some(call_id.clone()),
@@ -794,19 +803,29 @@ impl GuardianV2Extension {
if result.is_err() {
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
}
record_classification(
metrics.as_deref(),
classification_finished_at
.map(|finished_at: Instant| {
finished_at.duration_since(classification_started_at)
})
.unwrap_or_else(|| classification_started_at.elapsed()),
match &result {
Ok(ClassificationOutcome::Scored) => "success",
Ok(ClassificationOutcome::Superseded) => "superseded",
Err(_) => "failure",
},
);
let duration = classification_finished_at
.map(|finished_at: Instant| finished_at.duration_since(classification_started_at))
.unwrap_or_else(|| classification_started_at.elapsed());
let outcome = match &result {
Ok(ClassificationOutcome::Scored) => "success",
Ok(ClassificationOutcome::Superseded) => "superseded",
Err(_) => "failure",
};
record_classification(metrics.as_deref(), duration, outcome);
if let Some(analytics) = analytics {
analytics.track_guardian_v2_event(GuardianV2Event {
thread_id: thread_id.clone(),
turn_id: turn_id.clone(),
item_id: Some(call_id),
model: parent_model.as_ref().map(|model| model.slug.clone()),
occurred_at_ms: codex_analytics::now_unix_millis(),
kind: GuardianV2EventKind::Classification {
outcome,
risk_level: classification_risk,
duration_ms: u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
},
});
}
if matches!(result, Ok(ClassificationOutcome::Scored)) {
truncations.emit(metrics.as_deref());
}