From 86b7d127428573c8088ef7ed478e59f7bb28a9db Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 1 Sep 2026 21:48:34 +0000 Subject: [PATCH] Add Guardian V2 analytics events (#42144) ## What changed - Emit `codex_guardian_v2_classification` events with the outcome, risk level, duration, model, and thread attribution. - Emit `codex_guardian_v2_fast_decision` events when Guardian V2 approves an action without a full review. - Add `guardian_v2_enabled` to turn analytics, accounting for review scope and model requirements. ## Testing - Extend the Guardian V2 app-server coverage to verify event payloads, attribution, enablement state, and scoped event emission. GitOrigin-RevId: 0805975f8e8f9d954aa1058ea8c60545a090ae74 --- codex-rs/Cargo.lock | 1 + .../analytics/src/analytics_client_tests.rs | 4 + codex-rs/analytics/src/client.rs | 7 ++ codex-rs/analytics/src/events.rs | 3 + codex-rs/analytics/src/facts.rs | 3 + codex-rs/analytics/src/guardian_v2.rs | 49 +++++++++++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 39 +++++++++ .../app-server/tests/suite/v2/analytics.rs | 1 + .../app-server/tests/suite/v2/guardian_v2.rs | 81 ++++++++++++++++++- codex-rs/core/src/guardian/review.rs | 23 ++++++ codex-rs/core/src/session/turn.rs | 12 +++ .../ext/extension-api/src/contributors.rs | 1 + .../src/contributors/approval_review.rs | 6 ++ codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/guardian-v2/Cargo.toml | 1 + .../guardian-v2/src/async_scorer/extension.rs | 57 ++++++++----- 17 files changed, 272 insertions(+), 20 deletions(-) create mode 100644 codex-rs/analytics/src/guardian_v2.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f5cbc1b787..30063d3c3d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3382,6 +3382,7 @@ name = "codex-guardian-v2" version = "0.0.0" dependencies = [ "anyhow", + "codex-analytics", "codex-api", "codex-core", "codex-extension-api", diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 9c19c7cad0..0f5661c54d 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -480,6 +480,7 @@ fn sample_turn_resolved_config(thread_id: &str, turn_id: &str) -> TurnResolvedCo service_tier: None, approval_policy: AskForApproval::OnRequest, approvals_reviewer: ApprovalsReviewer::AutoReview, + guardian_v2_enabled: false, sandbox_network_access: true, collaboration_mode: ModeKind::Plan, personality: None, @@ -4674,6 +4675,7 @@ fn turn_event_serializes_expected_shape() { service_tier: "flex".to_string(), approval_policy: "on-request".to_string(), approvals_reviewer: "auto_review".to_string(), + guardian_v2_enabled: true, sandbox_network_access: true, collaboration_mode: Some("plan"), personality: Some("pragmatic".to_string()), @@ -4761,6 +4763,7 @@ fn turn_event_serializes_expected_shape() { "service_tier": "flex", "approval_policy": "on-request", "approvals_reviewer": "auto_review", + "guardian_v2_enabled": true, "sandbox_network_access": true, "collaboration_mode": "plan", "personality": "pragmatic", @@ -5114,6 +5117,7 @@ async fn turn_lifecycle_emits_turn_event() { }) ); assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["guardian_v2_enabled"], json!(false)); assert_eq!(payload["event_params"]["ephemeral"], json!(false)); assert_eq!(payload["event_params"]["workspace_kind"], json!(null)); assert_eq!(payload["event_params"]["num_input_images"], json!(1)); diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 7b5056acc8..96dce42750 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -33,6 +33,7 @@ use crate::facts::TurnCodexErrorFact; use crate::facts::TurnProfileFact; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnTokenUsageFact; +use crate::guardian_v2::GuardianV2Event; use crate::now_unix_millis; use crate::reducer::AnalyticsReducer; use crate::reducer::MAX_PLUGIN_MEASUREMENTS_PER_BATCH; @@ -492,6 +493,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_guardian_v2_event(&self, event: GuardianV2Event) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianV2( + Box::new(event), + ))); + } + pub fn track_goal_event(&self, event: CodexGoalEvent) { self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new( event, diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index dbc99f7ca4..0342ea79c9 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -25,6 +25,7 @@ use crate::facts::TurnStatus; use crate::facts::TurnSteerRejectionReason; use crate::facts::TurnSteerResult; use crate::facts::TurnSubmissionType; +use crate::guardian_v2::GuardianV2EventRequest; use crate::now_unix_millis; use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::CommandExecutionSource; @@ -69,6 +70,7 @@ pub(crate) enum TrackEventRequest { ThreadInitialized(ThreadInitializedEvent), ThreadArchive(ThreadArchiveEvent), GuardianReview(Box), + GuardianV2(Box), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), HookRun(CodexHookRunEventRequest), @@ -1032,6 +1034,7 @@ pub(crate) struct CodexTurnEventParams { pub(crate) service_tier: String, pub(crate) approval_policy: String, pub(crate) approvals_reviewer: String, + pub(crate) guardian_v2_enabled: bool, pub(crate) sandbox_network_access: bool, pub(crate) collaboration_mode: Option<&'static str>, pub(crate) personality: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index ef730e611a..4fb8bbca3d 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -1,6 +1,7 @@ use crate::events::AppServerRpcTransport; use crate::events::CodexRuntimeMetadata; use crate::events::GuardianReviewEventParams; +use crate::guardian_v2::GuardianV2Event; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponsePayload; use codex_app_server_protocol::InitializeParams; @@ -199,6 +200,7 @@ pub struct TurnResolvedConfigFact { pub service_tier: Option, pub approval_policy: AskForApproval, pub approvals_reviewer: ApprovalsReviewer, + pub guardian_v2_enabled: bool, pub sandbox_network_access: bool, pub collaboration_mode: ModeKind, pub personality: Option, @@ -565,6 +567,7 @@ pub(crate) enum CustomAnalyticsFact { Compaction(Box), Goal(Box), GuardianReview(Box), + GuardianV2(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), TurnProfile(Box), diff --git a/codex-rs/analytics/src/guardian_v2.rs b/codex-rs/analytics/src/guardian_v2.rs new file mode 100644 index 0000000000..731ce80019 --- /dev/null +++ b/codex-rs/analytics/src/guardian_v2.rs @@ -0,0 +1,49 @@ +//! Guardian V2 classifier and fast-decision facts, enriched by the existing reducer. +//! Payloads contain attribution and bounded outcomes, never prompts or tool arguments. + +use crate::events::CodexAppServerClientMetadata; +use crate::events::CodexRuntimeMetadata; +use codex_protocol::protocol::ThreadSource; +use serde::Serialize; + +#[derive(Serialize)] +pub struct GuardianV2Event { + pub thread_id: String, + pub turn_id: String, + pub item_id: Option, + pub model: Option, + pub occurred_at_ms: u64, + #[serde(flatten)] + pub kind: GuardianV2EventKind, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum GuardianV2EventKind { + Classification { + outcome: &'static str, + risk_level: Option<&'static str>, + duration_ms: u64, + }, + FastDecision { + decision: &'static str, + }, +} + +#[derive(Serialize)] +pub(crate) struct GuardianV2EventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: GuardianV2EventParams, +} + +#[derive(Serialize)] +pub(crate) struct GuardianV2EventParams { + pub(crate) session_id: String, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) thread_source: Option, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, + #[serde(flatten)] + pub(crate) guardian_v2: GuardianV2Event, +} diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 49a87ab845..7d2f86b979 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -4,6 +4,7 @@ mod analytics_capture; mod client; mod events; mod facts; +mod guardian_v2; mod reducer; use std::time::SystemTime; @@ -71,6 +72,8 @@ pub use facts::TurnSteerRequestError; pub use facts::TurnSteerResult; pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; +pub use guardian_v2::GuardianV2Event; +pub use guardian_v2::GuardianV2EventKind; #[cfg(test)] mod analytics_client_tests; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index a655bcfb33..be5b8f24cb 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -113,6 +113,9 @@ use crate::facts::TurnStatus; use crate::facts::TurnSteerRejectionReason; use crate::facts::TurnSteerResult; use crate::facts::TurnTokenUsageFact; +use crate::guardian_v2::GuardianV2EventKind; +use crate::guardian_v2::GuardianV2EventParams; +use crate::guardian_v2::GuardianV2EventRequest; use crate::now_unix_millis; use crate::now_unix_seconds; use crate::option_i64_to_u64; @@ -618,6 +621,40 @@ impl AnalyticsReducer { CustomAnalyticsFact::Goal(input) => { self.ingest_goal(*input, out); } + CustomAnalyticsFact::GuardianV2(input) => { + let event_type = match &input.kind { + GuardianV2EventKind::Classification { .. } => { + "codex_guardian_v2_classification" + } + GuardianV2EventKind::FastDecision { .. } => { + "codex_guardian_v2_fast_decision" + } + }; + if let Some((connection, thread, metadata)) = + self.thread_context_or_warn(AnalyticsDropSite { + event_name: event_type, + thread_id: &input.thread_id, + turn_id: Some(&input.turn_id), + review_id: None, + item_id: input.item_id.as_deref(), + }) + { + out.push(TrackEventRequest::GuardianV2(Box::new( + GuardianV2EventRequest { + event_type, + event_params: GuardianV2EventParams { + session_id: metadata.session_id.clone(), + app_server_client: thread.app_server_client(connection), + runtime: connection.runtime.clone(), + thread_source: metadata.thread_source.clone(), + subagent_source: metadata.subagent_source.clone(), + parent_thread_id: metadata.parent_thread_id.clone(), + guardian_v2: *input, + }, + }, + ))); + } + } CustomAnalyticsFact::GuardianReview(input) => { self.ingest_guardian_review(*input, out); } @@ -3403,6 +3440,7 @@ fn codex_turn_event_params( service_tier, approval_policy, approvals_reviewer, + guardian_v2_enabled, sandbox_network_access, collaboration_mode, personality, @@ -3449,6 +3487,7 @@ fn codex_turn_event_params( .unwrap_or_else(|| "default".to_string()), approval_policy: approval_policy.to_string(), approvals_reviewer: approvals_reviewer.to_string(), + guardian_v2_enabled, sandbox_network_access, collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), personality: personality_mode(personality), diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index 12de353176..6ab5ee10b2 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -198,6 +198,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> { .expect("turn completion time"); assert!(started_at > 0 && completed_at >= started_at); assert!(params["duration_ms"].is_u64()); + assert_eq!(params["guardian_v2_enabled"], false); json!([ params["turn_id"], params["status"], diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs index f90299a0b0..990b7e6019 100644 --- a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs +++ b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs @@ -56,6 +56,9 @@ use tokio::net::TcpListener; use tokio::sync::Notify; use tokio::time::timeout; +use super::analytics::captured_analytics_events; +use super::analytics::mount_analytics_capture; +use super::analytics::wait_for_matching_analytics_event; use super::mcp_tool::TEST_SERVER_NAME; use super::mcp_tool::TEST_TOOL_NAME; use super::mcp_tool::start_mcp_server; @@ -633,6 +636,8 @@ async fn guardian_v2_routes_scoped_tool_approvals( let (mcp_server_url, mcp_server_handle) = start_mcp_server(sensitive_action).await?; let codex_home = TempDir::new()?; + let analytics_server = responses::start_mock_server().await; + mount_analytics_capture(&analytics_server, codex_home.path()).await?; let root_skill = if matches!(lifecycle, ThreadLifecycle::RootTrustedSkill) { let path = codex_home.path().join("skills/root-trusted/SKILL.md"); std::fs::create_dir_all(path.parent().expect("root skill parent"))?; @@ -700,7 +705,10 @@ async fn guardian_v2_routes_scoped_tool_approvals( .with_model(MODEL) .with_provider_config("supports_websockets = false") .with_approval_policy("on-request") - .with_root_config(reviewer_config) + .with_root_config(&format!( + "{reviewer_config}\nchatgpt_base_url = \"{}\"", + analytics_server.uri(), + )) .with_extra_config(&format!( "[mcp_servers.{server_name}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"{tool_approval_mode}\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}{guardian_scope_config}" )) @@ -1358,6 +1366,77 @@ async fn guardian_v2_routes_scoped_tool_approvals( timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?; } + if matches!(lifecycle, ThreadLifecycle::New) + && matches!(scope, GuardianToolScope::AllTools) + && sensitive_action.is_none() + { + if classifier_in_scope { + wait_for_matching_analytics_event(&analytics_server, TIMEOUT, |event| { + event["event_type"] == "codex_guardian_v2_classification" + && event["event_params"]["item_id"] == "guardian-action-0" + }) + .await?; + } + timeout(TIMEOUT, app_server.shutdown_gracefully()).await??; + let events = captured_analytics_events(&analytics_server).await; + let turn = &events + .iter() + .find(|event| { + event["event_type"] == "codex_turn_event" + && event["event_params"]["thread_id"] == reviewed_thread_id + }) + .expect("parent turn analytics")["event_params"]; + assert_eq!(turn["guardian_v2_enabled"], classifier_in_scope); + let classification = events.iter().find(|event| { + event["event_type"] == "codex_guardian_v2_classification" + && event["event_params"]["item_id"] == "guardian-action-0" + }); + assert_eq!(classification.is_some(), classifier_in_scope); + if let Some(event) = classification { + assert_eq!( + json!([ + event["event_params"]["outcome"], + event["event_params"]["risk_level"] + ]), + match risk { + GuardianRisk::Low => json!(["success", "low"]), + GuardianRisk::Threshold | GuardianRisk::High => json!(["success", "high"]), + GuardianRisk::InvalidResponse => json!(["failure", null]), + } + ); + } + let approvals = events + .iter() + .filter(|event| event["event_type"] == "codex_guardian_v2_fast_decision") + .collect::>(); + assert_eq!( + approvals.len(), + usize::from(classifier_in_scope && matches!(risk, GuardianRisk::Low)) + ); + for event in classification.into_iter().chain(approvals) { + let params = &event["event_params"]; + for key in [ + "session_id", + "thread_id", + "turn_id", + "model", + "app_server_client", + "runtime", + "thread_source", + "subagent_source", + "parent_thread_id", + ] { + assert_eq!(params[key], turn[key], "{key}"); + } + if event["event_type"] == "codex_guardian_v2_fast_decision" { + assert_eq!( + json!([params["item_id"], params["decision"]]), + json!(["guardian-action-1", "approved"]) + ); + } + } + } + mcp_server_handle.abort(); responses_server.abort(); Ok(()) diff --git a/codex-rs/core/src/guardian/review.rs b/codex-rs/core/src/guardian/review.rs index 6f299c62a9..18df6421a1 100644 --- a/codex-rs/core/src/guardian/review.rs +++ b/codex-rs/core/src/guardian/review.rs @@ -5,8 +5,11 @@ use codex_analytics::GuardianReviewFailureReason; use codex_analytics::GuardianReviewTerminalStatus; use codex_analytics::GuardianReviewTrackContext; use codex_analytics::GuardianReviewedAction; +use codex_analytics::GuardianV2Event; +use codex_analytics::GuardianV2EventKind; use codex_async_utils::THREAD_STACK_SIZE_BYTES; use codex_core_plugins::PluginCommandAttribution; +use codex_extension_api::GuardianV2Enabled; use codex_extension_api::ThreadIdleCause; use codex_features::Feature; use codex_protocol::config_types::ApprovalsReviewer; @@ -362,6 +365,26 @@ async fn run_guardian_review( .await { if decision == ReviewDecision::Approved { + if session + .services + .thread_extension_data + .get::() + .is_some() + { + session + .services + .analytics_events_client + .track_guardian_v2_event(GuardianV2Event { + thread_id: session.thread_id.to_string(), + turn_id: guardian_request_turn_id(&request, &turn.sub_id).to_owned(), + item_id: guardian_request_target_item_id(&request).map(str::to_owned), + model: Some(turn.model_info().slug.clone()), + occurred_at_ms: codex_analytics::now_unix_millis(), + kind: GuardianV2EventKind::FastDecision { + decision: "approved", + }, + }); + } record_guardian_non_denial(&session, guardian_request_turn_id(&request, &turn.sub_id)) .await; } diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 96ba5ffe03..c5f6eb2597 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -1029,6 +1029,18 @@ async fn track_turn_resolved_config_analytics( .and_then(ServiceTier::from_request_value), approval_policy: turn_context.approval_policy(), approvals_reviewer: turn_context.config.approvals_reviewer, + guardian_v2_enabled: sess + .services + .thread_extension_data + .get::() + .is_some_and(|state| { + state.computer_use_only + || !turn_context + .config + .config_layer_stack + .requirements() + .auto_review_required_for_model(&turn_context.model_info().slug) + }), sandbox_network_access: turn_context.network_sandbox_policy().is_enabled(), collaboration_mode: turn_context.mode(), personality: turn_context.personality(), diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 218b5eef79..ebb180755e 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -26,6 +26,7 @@ mod world_state; pub use approval_review::ApprovalAssessment; pub use approval_review::ApprovalReviewError; pub use approval_review::ApprovalReviewInput; +pub use approval_review::GuardianV2Enabled; pub use context::TurnContextContributionInput; pub use mcp::McpServerContribution; pub use mcp::McpServerContributionContext; diff --git a/codex-rs/ext/extension-api/src/contributors/approval_review.rs b/codex-rs/ext/extension-api/src/contributors/approval_review.rs index 5669ada823..cabd93293c 100644 --- a/codex-rs/ext/extension-api/src/contributors/approval_review.rs +++ b/codex-rs/ext/extension-api/src/contributors/approval_review.rs @@ -10,6 +10,12 @@ use serde_json::Value; use crate::ConversationHistorySnapshot; use crate::ExtensionData; +/// Thread-local state installed only after Guardian V2's async classifier initializes. +pub struct GuardianV2Enabled { + /// Computer-use classification remains active for models requiring synchronous review. + pub computer_use_only: bool, +} + /// Classification returned by an approval reviewer. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ApprovalAssessment { diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index d927ab65a1..a8ba8f1f20 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -47,6 +47,7 @@ pub use contributors::ApprovalReviewInput; pub use contributors::ConfigContributor; pub use contributors::ContextContributor; pub use contributors::ExtensionFuture; +pub use contributors::GuardianV2Enabled; pub use contributors::McpServerContribution; pub use contributors::McpServerContributionContext; pub use contributors::McpServerContributor; diff --git a/codex-rs/ext/guardian-v2/Cargo.toml b/codex-rs/ext/guardian-v2/Cargo.toml index 242102a99c..00d381dfc5 100644 --- a/codex-rs/ext/guardian-v2/Cargo.toml +++ b/codex-rs/ext/guardian-v2/Cargo.toml @@ -13,6 +13,7 @@ doctest = false workspace = true [dependencies] +codex-analytics = { workspace = true } codex-api = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs index ed7ea62eb7..97342a76d2 100644 --- a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs +++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs @@ -7,6 +7,9 @@ use std::sync::atomic::Ordering; use std::time::Instant; use std::time::SystemTime; +use codex_analytics::AnalyticsEventsClient; +use codex_analytics::GuardianV2Event; +use codex_analytics::GuardianV2EventKind; use codex_core::ThreadManager; use codex_core::config::Config; use codex_core::context::GuardianReviewEvidence; @@ -18,6 +21,7 @@ use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionMetrics; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ExtensionWarning; +use codex_extension_api::GuardianV2Enabled; use codex_extension_api::ResponseItem; use codex_extension_api::SkillInvocationContributor; use codex_extension_api::SkillInvocationInput; @@ -103,8 +107,6 @@ pub enum StrictReviewReason { StaleScore, } -struct GuardianV2Enabled; - enum ClassificationOutcome { Scored, Superseded, @@ -196,6 +198,10 @@ impl ThreadLifecycleContributor for GuardianV2Extension { let sampler = input .thread_store .get_or_init(|| LunaSampler::new(sampler_config)); + let guardian_v2_enabled = GuardianV2Enabled { + computer_use_only: guardian_config.review_scope + == GuardianV2ReviewScope::ComputerUseOnly, + }; input.thread_store.insert(guardian_config); input.thread_store.insert(GuardianV2ScoreProgress { metrics: input.extension_metrics.clone(), @@ -205,7 +211,7 @@ impl ThreadLifecycleContributor for GuardianV2Extension { input .thread_store .insert(TrustedSkillRoots::from_config(input.config)); - input.thread_store.insert(GuardianV2Enabled); + input.thread_store.insert(guardian_v2_enabled); tokio::spawn(async move { sampler.prewarm().await; @@ -431,6 +437,7 @@ impl GuardianV2Extension { .fetch_add(/*val*/ 1, Ordering::Relaxed); } let metrics = score_progress.metrics.clone(); + let analytics = input.session_store.get::(); let sampled_at = SystemTime::now(); let tool_call_index = score_progress .latest_tool_call @@ -673,6 +680,7 @@ impl GuardianV2Extension { format!("{planned_action}\n"), ">>> APPROVAL REQUEST END\n".to_owned(), ]); + let mut classification_risk = None; let mut classification_finished_at = None; let result: Result = async { let review_model_messages = if config.guardian_policy_config.is_none() { @@ -722,11 +730,12 @@ impl GuardianV2Extension { } Err(error) => return Err(error.to_string()), }; - let action_risk = match output.as_str() { - "high" => 1.0, - "low" => 0.0, + let (action_risk, risk_level) = match output.as_str() { + "high" => (1.0, "high"), + "low" => (0.0, "low"), _ => return Err("invalid Guardian V2 classification".to_owned()), }; + classification_risk = Some(risk_level); let score = SecurityRiskScore { scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]), call_id: Some(call_id.clone()), @@ -794,19 +803,29 @@ impl GuardianV2Extension { if result.is_err() { Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at); } - record_classification( - metrics.as_deref(), - classification_finished_at - .map(|finished_at: Instant| { - finished_at.duration_since(classification_started_at) - }) - .unwrap_or_else(|| classification_started_at.elapsed()), - match &result { - Ok(ClassificationOutcome::Scored) => "success", - Ok(ClassificationOutcome::Superseded) => "superseded", - Err(_) => "failure", - }, - ); + let duration = classification_finished_at + .map(|finished_at: Instant| finished_at.duration_since(classification_started_at)) + .unwrap_or_else(|| classification_started_at.elapsed()); + let outcome = match &result { + Ok(ClassificationOutcome::Scored) => "success", + Ok(ClassificationOutcome::Superseded) => "superseded", + Err(_) => "failure", + }; + record_classification(metrics.as_deref(), duration, outcome); + if let Some(analytics) = analytics { + analytics.track_guardian_v2_event(GuardianV2Event { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item_id: Some(call_id), + model: parent_model.as_ref().map(|model| model.slug.clone()), + occurred_at_ms: codex_analytics::now_unix_millis(), + kind: GuardianV2EventKind::Classification { + outcome, + risk_level: classification_risk, + duration_ms: u64::try_from(duration.as_millis()).unwrap_or(u64::MAX), + }, + }); + } if matches!(result, Ok(ClassificationOutcome::Scored)) { truncations.emit(metrics.as_deref()); }