mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Add Guardian V2 analytics events (#42144)
## What changed - Emit `codex_guardian_v2_classification` events with the outcome, risk level, duration, model, and thread attribution. - Emit `codex_guardian_v2_fast_decision` events when Guardian V2 approves an action without a full review. - Add `guardian_v2_enabled` to turn analytics, accounting for review scope and model requirements. ## Testing - Extend the Guardian V2 app-server coverage to verify event payloads, attribution, enablement state, and scoped event emission. GitOrigin-RevId: 0805975f8e8f9d954aa1058ea8c60545a090ae74
This commit is contained in:
1 parent
a876a9e494
commit
86b7d12742
17 files changed
+272
-20
No files matched your search
@@ -13,6 +13,7 @@ doctest = false
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
codex-analytics = { workspace = true }
|
||||
codex-api = { workspace = true }
|
||||
codex-core = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
|
||||
@@ -7,6 +7,9 @@ use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use codex_analytics::AnalyticsEventsClient;
|
||||
use codex_analytics::GuardianV2Event;
|
||||
use codex_analytics::GuardianV2EventKind;
|
||||
use codex_core::ThreadManager;
|
||||
use codex_core::config::Config;
|
||||
use codex_core::context::GuardianReviewEvidence;
|
||||
@@ -18,6 +21,7 @@ use codex_extension_api::ExtensionFuture;
|
||||
use codex_extension_api::ExtensionMetrics;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::ExtensionWarning;
|
||||
use codex_extension_api::GuardianV2Enabled;
|
||||
use codex_extension_api::ResponseItem;
|
||||
use codex_extension_api::SkillInvocationContributor;
|
||||
use codex_extension_api::SkillInvocationInput;
|
||||
@@ -103,8 +107,6 @@ pub enum StrictReviewReason {
|
||||
StaleScore,
|
||||
}
|
||||
|
||||
struct GuardianV2Enabled;
|
||||
|
||||
enum ClassificationOutcome {
|
||||
Scored,
|
||||
Superseded,
|
||||
@@ -196,6 +198,10 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
|
||||
let sampler = input
|
||||
.thread_store
|
||||
.get_or_init(|| LunaSampler::new(sampler_config));
|
||||
let guardian_v2_enabled = GuardianV2Enabled {
|
||||
computer_use_only: guardian_config.review_scope
|
||||
== GuardianV2ReviewScope::ComputerUseOnly,
|
||||
};
|
||||
input.thread_store.insert(guardian_config);
|
||||
input.thread_store.insert(GuardianV2ScoreProgress {
|
||||
metrics: input.extension_metrics.clone(),
|
||||
@@ -205,7 +211,7 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
|
||||
input
|
||||
.thread_store
|
||||
.insert(TrustedSkillRoots::from_config(input.config));
|
||||
input.thread_store.insert(GuardianV2Enabled);
|
||||
input.thread_store.insert(guardian_v2_enabled);
|
||||
|
||||
tokio::spawn(async move {
|
||||
sampler.prewarm().await;
|
||||
@@ -431,6 +437,7 @@ impl GuardianV2Extension {
|
||||
.fetch_add(/*val*/ 1, Ordering::Relaxed);
|
||||
}
|
||||
let metrics = score_progress.metrics.clone();
|
||||
let analytics = input.session_store.get::<AnalyticsEventsClient>();
|
||||
let sampled_at = SystemTime::now();
|
||||
let tool_call_index = score_progress
|
||||
.latest_tool_call
|
||||
@@ -673,6 +680,7 @@ impl GuardianV2Extension {
|
||||
format!("{planned_action}\n"),
|
||||
">>> APPROVAL REQUEST END\n".to_owned(),
|
||||
]);
|
||||
let mut classification_risk = None;
|
||||
let mut classification_finished_at = None;
|
||||
let result: Result<ClassificationOutcome, String> = async {
|
||||
let review_model_messages = if config.guardian_policy_config.is_none() {
|
||||
@@ -722,11 +730,12 @@ impl GuardianV2Extension {
|
||||
}
|
||||
Err(error) => return Err(error.to_string()),
|
||||
};
|
||||
let action_risk = match output.as_str() {
|
||||
"high" => 1.0,
|
||||
"low" => 0.0,
|
||||
let (action_risk, risk_level) = match output.as_str() {
|
||||
"high" => (1.0, "high"),
|
||||
"low" => (0.0, "low"),
|
||||
_ => return Err("invalid Guardian V2 classification".to_owned()),
|
||||
};
|
||||
classification_risk = Some(risk_level);
|
||||
let score = SecurityRiskScore {
|
||||
scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]),
|
||||
call_id: Some(call_id.clone()),
|
||||
@@ -794,19 +803,29 @@ impl GuardianV2Extension {
|
||||
if result.is_err() {
|
||||
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
|
||||
}
|
||||
record_classification(
|
||||
metrics.as_deref(),
|
||||
classification_finished_at
|
||||
.map(|finished_at: Instant| {
|
||||
finished_at.duration_since(classification_started_at)
|
||||
})
|
||||
.unwrap_or_else(|| classification_started_at.elapsed()),
|
||||
match &result {
|
||||
Ok(ClassificationOutcome::Scored) => "success",
|
||||
Ok(ClassificationOutcome::Superseded) => "superseded",
|
||||
Err(_) => "failure",
|
||||
},
|
||||
);
|
||||
let duration = classification_finished_at
|
||||
.map(|finished_at: Instant| finished_at.duration_since(classification_started_at))
|
||||
.unwrap_or_else(|| classification_started_at.elapsed());
|
||||
let outcome = match &result {
|
||||
Ok(ClassificationOutcome::Scored) => "success",
|
||||
Ok(ClassificationOutcome::Superseded) => "superseded",
|
||||
Err(_) => "failure",
|
||||
};
|
||||
record_classification(metrics.as_deref(), duration, outcome);
|
||||
if let Some(analytics) = analytics {
|
||||
analytics.track_guardian_v2_event(GuardianV2Event {
|
||||
thread_id: thread_id.clone(),
|
||||
turn_id: turn_id.clone(),
|
||||
item_id: Some(call_id),
|
||||
model: parent_model.as_ref().map(|model| model.slug.clone()),
|
||||
occurred_at_ms: codex_analytics::now_unix_millis(),
|
||||
kind: GuardianV2EventKind::Classification {
|
||||
outcome,
|
||||
risk_level: classification_risk,
|
||||
duration_ms: u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
|
||||
},
|
||||
});
|
||||
}
|
||||
if matches!(result, Ok(ClassificationOutcome::Scored)) {
|
||||
truncations.emit(metrics.as_deref());
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user