mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Add Guardian V2 analytics events (#42144)
## What changed - Emit `codex_guardian_v2_classification` events with the outcome, risk level, duration, model, and thread attribution. - Emit `codex_guardian_v2_fast_decision` events when Guardian V2 approves an action without a full review. - Add `guardian_v2_enabled` to turn analytics, accounting for review scope and model requirements. ## Testing - Extend the Guardian V2 app-server coverage to verify event payloads, attribution, enablement state, and scoped event emission. GitOrigin-RevId: 0805975f8e8f9d954aa1058ea8c60545a090ae74
This commit is contained in:
1 parent
a876a9e494
commit
86b7d12742
17 files changed
+272
-20
No files matched your search
Generated
+1
@@ -3382,6 +3382,7 @@ name = "codex-guardian-v2"
|
|||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"codex-analytics",
|
||||||
"codex-api",
|
"codex-api",
|
||||||
"codex-core",
|
"codex-core",
|
||||||
"codex-extension-api",
|
"codex-extension-api",
|
||||||
|
|||||||
@@ -480,6 +480,7 @@ fn sample_turn_resolved_config(thread_id: &str, turn_id: &str) -> TurnResolvedCo
|
|||||||
service_tier: None,
|
service_tier: None,
|
||||||
approval_policy: AskForApproval::OnRequest,
|
approval_policy: AskForApproval::OnRequest,
|
||||||
approvals_reviewer: ApprovalsReviewer::AutoReview,
|
approvals_reviewer: ApprovalsReviewer::AutoReview,
|
||||||
|
guardian_v2_enabled: false,
|
||||||
sandbox_network_access: true,
|
sandbox_network_access: true,
|
||||||
collaboration_mode: ModeKind::Plan,
|
collaboration_mode: ModeKind::Plan,
|
||||||
personality: None,
|
personality: None,
|
||||||
@@ -4674,6 +4675,7 @@ fn turn_event_serializes_expected_shape() {
|
|||||||
service_tier: "flex".to_string(),
|
service_tier: "flex".to_string(),
|
||||||
approval_policy: "on-request".to_string(),
|
approval_policy: "on-request".to_string(),
|
||||||
approvals_reviewer: "auto_review".to_string(),
|
approvals_reviewer: "auto_review".to_string(),
|
||||||
|
guardian_v2_enabled: true,
|
||||||
sandbox_network_access: true,
|
sandbox_network_access: true,
|
||||||
collaboration_mode: Some("plan"),
|
collaboration_mode: Some("plan"),
|
||||||
personality: Some("pragmatic".to_string()),
|
personality: Some("pragmatic".to_string()),
|
||||||
@@ -4761,6 +4763,7 @@ fn turn_event_serializes_expected_shape() {
|
|||||||
"service_tier": "flex",
|
"service_tier": "flex",
|
||||||
"approval_policy": "on-request",
|
"approval_policy": "on-request",
|
||||||
"approvals_reviewer": "auto_review",
|
"approvals_reviewer": "auto_review",
|
||||||
|
"guardian_v2_enabled": true,
|
||||||
"sandbox_network_access": true,
|
"sandbox_network_access": true,
|
||||||
"collaboration_mode": "plan",
|
"collaboration_mode": "plan",
|
||||||
"personality": "pragmatic",
|
"personality": "pragmatic",
|
||||||
@@ -5114,6 +5117,7 @@ async fn turn_lifecycle_emits_turn_event() {
|
|||||||
})
|
})
|
||||||
);
|
);
|
||||||
assert!(payload["event_params"].get("product_client_id").is_none());
|
assert!(payload["event_params"].get("product_client_id").is_none());
|
||||||
|
assert_eq!(payload["event_params"]["guardian_v2_enabled"], json!(false));
|
||||||
assert_eq!(payload["event_params"]["ephemeral"], json!(false));
|
assert_eq!(payload["event_params"]["ephemeral"], json!(false));
|
||||||
assert_eq!(payload["event_params"]["workspace_kind"], json!(null));
|
assert_eq!(payload["event_params"]["workspace_kind"], json!(null));
|
||||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ use crate::facts::TurnCodexErrorFact;
|
|||||||
use crate::facts::TurnProfileFact;
|
use crate::facts::TurnProfileFact;
|
||||||
use crate::facts::TurnResolvedConfigFact;
|
use crate::facts::TurnResolvedConfigFact;
|
||||||
use crate::facts::TurnTokenUsageFact;
|
use crate::facts::TurnTokenUsageFact;
|
||||||
|
use crate::guardian_v2::GuardianV2Event;
|
||||||
use crate::now_unix_millis;
|
use crate::now_unix_millis;
|
||||||
use crate::reducer::AnalyticsReducer;
|
use crate::reducer::AnalyticsReducer;
|
||||||
use crate::reducer::MAX_PLUGIN_MEASUREMENTS_PER_BATCH;
|
use crate::reducer::MAX_PLUGIN_MEASUREMENTS_PER_BATCH;
|
||||||
@@ -492,6 +493,12 @@ impl AnalyticsEventsClient {
|
|||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn track_guardian_v2_event(&self, event: GuardianV2Event) {
|
||||||
|
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianV2(
|
||||||
|
Box::new(event),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
pub fn track_goal_event(&self, event: CodexGoalEvent) {
|
pub fn track_goal_event(&self, event: CodexGoalEvent) {
|
||||||
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(
|
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(
|
||||||
event,
|
event,
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ use crate::facts::TurnStatus;
|
|||||||
use crate::facts::TurnSteerRejectionReason;
|
use crate::facts::TurnSteerRejectionReason;
|
||||||
use crate::facts::TurnSteerResult;
|
use crate::facts::TurnSteerResult;
|
||||||
use crate::facts::TurnSubmissionType;
|
use crate::facts::TurnSubmissionType;
|
||||||
|
use crate::guardian_v2::GuardianV2EventRequest;
|
||||||
use crate::now_unix_millis;
|
use crate::now_unix_millis;
|
||||||
use codex_app_server_protocol::CodexErrorInfo;
|
use codex_app_server_protocol::CodexErrorInfo;
|
||||||
use codex_app_server_protocol::CommandExecutionSource;
|
use codex_app_server_protocol::CommandExecutionSource;
|
||||||
@@ -69,6 +70,7 @@ pub(crate) enum TrackEventRequest {
|
|||||||
ThreadInitialized(ThreadInitializedEvent),
|
ThreadInitialized(ThreadInitializedEvent),
|
||||||
ThreadArchive(ThreadArchiveEvent),
|
ThreadArchive(ThreadArchiveEvent),
|
||||||
GuardianReview(Box<GuardianReviewEventRequest>),
|
GuardianReview(Box<GuardianReviewEventRequest>),
|
||||||
|
GuardianV2(Box<GuardianV2EventRequest>),
|
||||||
AppMentioned(CodexAppMentionedEventRequest),
|
AppMentioned(CodexAppMentionedEventRequest),
|
||||||
AppUsed(CodexAppUsedEventRequest),
|
AppUsed(CodexAppUsedEventRequest),
|
||||||
HookRun(CodexHookRunEventRequest),
|
HookRun(CodexHookRunEventRequest),
|
||||||
@@ -1032,6 +1034,7 @@ pub(crate) struct CodexTurnEventParams {
|
|||||||
pub(crate) service_tier: String,
|
pub(crate) service_tier: String,
|
||||||
pub(crate) approval_policy: String,
|
pub(crate) approval_policy: String,
|
||||||
pub(crate) approvals_reviewer: String,
|
pub(crate) approvals_reviewer: String,
|
||||||
|
pub(crate) guardian_v2_enabled: bool,
|
||||||
pub(crate) sandbox_network_access: bool,
|
pub(crate) sandbox_network_access: bool,
|
||||||
pub(crate) collaboration_mode: Option<&'static str>,
|
pub(crate) collaboration_mode: Option<&'static str>,
|
||||||
pub(crate) personality: Option<String>,
|
pub(crate) personality: Option<String>,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use crate::events::AppServerRpcTransport;
|
use crate::events::AppServerRpcTransport;
|
||||||
use crate::events::CodexRuntimeMetadata;
|
use crate::events::CodexRuntimeMetadata;
|
||||||
use crate::events::GuardianReviewEventParams;
|
use crate::events::GuardianReviewEventParams;
|
||||||
|
use crate::guardian_v2::GuardianV2Event;
|
||||||
use codex_app_server_protocol::ClientRequest;
|
use codex_app_server_protocol::ClientRequest;
|
||||||
use codex_app_server_protocol::ClientResponsePayload;
|
use codex_app_server_protocol::ClientResponsePayload;
|
||||||
use codex_app_server_protocol::InitializeParams;
|
use codex_app_server_protocol::InitializeParams;
|
||||||
@@ -199,6 +200,7 @@ pub struct TurnResolvedConfigFact {
|
|||||||
pub service_tier: Option<ServiceTier>,
|
pub service_tier: Option<ServiceTier>,
|
||||||
pub approval_policy: AskForApproval,
|
pub approval_policy: AskForApproval,
|
||||||
pub approvals_reviewer: ApprovalsReviewer,
|
pub approvals_reviewer: ApprovalsReviewer,
|
||||||
|
pub guardian_v2_enabled: bool,
|
||||||
pub sandbox_network_access: bool,
|
pub sandbox_network_access: bool,
|
||||||
pub collaboration_mode: ModeKind,
|
pub collaboration_mode: ModeKind,
|
||||||
pub personality: Option<Personality>,
|
pub personality: Option<Personality>,
|
||||||
@@ -565,6 +567,7 @@ pub(crate) enum CustomAnalyticsFact {
|
|||||||
Compaction(Box<CodexCompactionEvent>),
|
Compaction(Box<CodexCompactionEvent>),
|
||||||
Goal(Box<CodexGoalEvent>),
|
Goal(Box<CodexGoalEvent>),
|
||||||
GuardianReview(Box<GuardianReviewEventParams>),
|
GuardianReview(Box<GuardianReviewEventParams>),
|
||||||
|
GuardianV2(Box<GuardianV2Event>),
|
||||||
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
||||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||||
TurnProfile(Box<TurnProfileFact>),
|
TurnProfile(Box<TurnProfileFact>),
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
//! Guardian V2 classifier and fast-decision facts, enriched by the existing reducer.
|
||||||
|
//! Payloads contain attribution and bounded outcomes, never prompts or tool arguments.
|
||||||
|
|
||||||
|
use crate::events::CodexAppServerClientMetadata;
|
||||||
|
use crate::events::CodexRuntimeMetadata;
|
||||||
|
use codex_protocol::protocol::ThreadSource;
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct GuardianV2Event {
|
||||||
|
pub thread_id: String,
|
||||||
|
pub turn_id: String,
|
||||||
|
pub item_id: Option<String>,
|
||||||
|
pub model: Option<String>,
|
||||||
|
pub occurred_at_ms: u64,
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub kind: GuardianV2EventKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum GuardianV2EventKind {
|
||||||
|
Classification {
|
||||||
|
outcome: &'static str,
|
||||||
|
risk_level: Option<&'static str>,
|
||||||
|
duration_ms: u64,
|
||||||
|
},
|
||||||
|
FastDecision {
|
||||||
|
decision: &'static str,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(crate) struct GuardianV2EventRequest {
|
||||||
|
pub(crate) event_type: &'static str,
|
||||||
|
pub(crate) event_params: GuardianV2EventParams,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(crate) struct GuardianV2EventParams {
|
||||||
|
pub(crate) session_id: String,
|
||||||
|
pub(crate) app_server_client: CodexAppServerClientMetadata,
|
||||||
|
pub(crate) runtime: CodexRuntimeMetadata,
|
||||||
|
pub(crate) thread_source: Option<ThreadSource>,
|
||||||
|
pub(crate) subagent_source: Option<String>,
|
||||||
|
pub(crate) parent_thread_id: Option<String>,
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub(crate) guardian_v2: GuardianV2Event,
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ mod analytics_capture;
|
|||||||
mod client;
|
mod client;
|
||||||
mod events;
|
mod events;
|
||||||
mod facts;
|
mod facts;
|
||||||
|
mod guardian_v2;
|
||||||
mod reducer;
|
mod reducer;
|
||||||
|
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
@@ -71,6 +72,8 @@ pub use facts::TurnSteerRequestError;
|
|||||||
pub use facts::TurnSteerResult;
|
pub use facts::TurnSteerResult;
|
||||||
pub use facts::TurnTokenUsageFact;
|
pub use facts::TurnTokenUsageFact;
|
||||||
pub use facts::build_track_events_context;
|
pub use facts::build_track_events_context;
|
||||||
|
pub use guardian_v2::GuardianV2Event;
|
||||||
|
pub use guardian_v2::GuardianV2EventKind;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod analytics_client_tests;
|
mod analytics_client_tests;
|
||||||
|
|||||||
@@ -113,6 +113,9 @@ use crate::facts::TurnStatus;
|
|||||||
use crate::facts::TurnSteerRejectionReason;
|
use crate::facts::TurnSteerRejectionReason;
|
||||||
use crate::facts::TurnSteerResult;
|
use crate::facts::TurnSteerResult;
|
||||||
use crate::facts::TurnTokenUsageFact;
|
use crate::facts::TurnTokenUsageFact;
|
||||||
|
use crate::guardian_v2::GuardianV2EventKind;
|
||||||
|
use crate::guardian_v2::GuardianV2EventParams;
|
||||||
|
use crate::guardian_v2::GuardianV2EventRequest;
|
||||||
use crate::now_unix_millis;
|
use crate::now_unix_millis;
|
||||||
use crate::now_unix_seconds;
|
use crate::now_unix_seconds;
|
||||||
use crate::option_i64_to_u64;
|
use crate::option_i64_to_u64;
|
||||||
@@ -618,6 +621,40 @@ impl AnalyticsReducer {
|
|||||||
CustomAnalyticsFact::Goal(input) => {
|
CustomAnalyticsFact::Goal(input) => {
|
||||||
self.ingest_goal(*input, out);
|
self.ingest_goal(*input, out);
|
||||||
}
|
}
|
||||||
|
CustomAnalyticsFact::GuardianV2(input) => {
|
||||||
|
let event_type = match &input.kind {
|
||||||
|
GuardianV2EventKind::Classification { .. } => {
|
||||||
|
"codex_guardian_v2_classification"
|
||||||
|
}
|
||||||
|
GuardianV2EventKind::FastDecision { .. } => {
|
||||||
|
"codex_guardian_v2_fast_decision"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some((connection, thread, metadata)) =
|
||||||
|
self.thread_context_or_warn(AnalyticsDropSite {
|
||||||
|
event_name: event_type,
|
||||||
|
thread_id: &input.thread_id,
|
||||||
|
turn_id: Some(&input.turn_id),
|
||||||
|
review_id: None,
|
||||||
|
item_id: input.item_id.as_deref(),
|
||||||
|
})
|
||||||
|
{
|
||||||
|
out.push(TrackEventRequest::GuardianV2(Box::new(
|
||||||
|
GuardianV2EventRequest {
|
||||||
|
event_type,
|
||||||
|
event_params: GuardianV2EventParams {
|
||||||
|
session_id: metadata.session_id.clone(),
|
||||||
|
app_server_client: thread.app_server_client(connection),
|
||||||
|
runtime: connection.runtime.clone(),
|
||||||
|
thread_source: metadata.thread_source.clone(),
|
||||||
|
subagent_source: metadata.subagent_source.clone(),
|
||||||
|
parent_thread_id: metadata.parent_thread_id.clone(),
|
||||||
|
guardian_v2: *input,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
CustomAnalyticsFact::GuardianReview(input) => {
|
CustomAnalyticsFact::GuardianReview(input) => {
|
||||||
self.ingest_guardian_review(*input, out);
|
self.ingest_guardian_review(*input, out);
|
||||||
}
|
}
|
||||||
@@ -3403,6 +3440,7 @@ fn codex_turn_event_params(
|
|||||||
service_tier,
|
service_tier,
|
||||||
approval_policy,
|
approval_policy,
|
||||||
approvals_reviewer,
|
approvals_reviewer,
|
||||||
|
guardian_v2_enabled,
|
||||||
sandbox_network_access,
|
sandbox_network_access,
|
||||||
collaboration_mode,
|
collaboration_mode,
|
||||||
personality,
|
personality,
|
||||||
@@ -3449,6 +3487,7 @@ fn codex_turn_event_params(
|
|||||||
.unwrap_or_else(|| "default".to_string()),
|
.unwrap_or_else(|| "default".to_string()),
|
||||||
approval_policy: approval_policy.to_string(),
|
approval_policy: approval_policy.to_string(),
|
||||||
approvals_reviewer: approvals_reviewer.to_string(),
|
approvals_reviewer: approvals_reviewer.to_string(),
|
||||||
|
guardian_v2_enabled,
|
||||||
sandbox_network_access,
|
sandbox_network_access,
|
||||||
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
|
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
|
||||||
personality: personality_mode(personality),
|
personality: personality_mode(personality),
|
||||||
|
|||||||
@@ -198,6 +198,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
|
|||||||
.expect("turn completion time");
|
.expect("turn completion time");
|
||||||
assert!(started_at > 0 && completed_at >= started_at);
|
assert!(started_at > 0 && completed_at >= started_at);
|
||||||
assert!(params["duration_ms"].is_u64());
|
assert!(params["duration_ms"].is_u64());
|
||||||
|
assert_eq!(params["guardian_v2_enabled"], false);
|
||||||
json!([
|
json!([
|
||||||
params["turn_id"],
|
params["turn_id"],
|
||||||
params["status"],
|
params["status"],
|
||||||
|
|||||||
@@ -56,6 +56,9 @@ use tokio::net::TcpListener;
|
|||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
|
|
||||||
|
use super::analytics::captured_analytics_events;
|
||||||
|
use super::analytics::mount_analytics_capture;
|
||||||
|
use super::analytics::wait_for_matching_analytics_event;
|
||||||
use super::mcp_tool::TEST_SERVER_NAME;
|
use super::mcp_tool::TEST_SERVER_NAME;
|
||||||
use super::mcp_tool::TEST_TOOL_NAME;
|
use super::mcp_tool::TEST_TOOL_NAME;
|
||||||
use super::mcp_tool::start_mcp_server;
|
use super::mcp_tool::start_mcp_server;
|
||||||
@@ -633,6 +636,8 @@ async fn guardian_v2_routes_scoped_tool_approvals(
|
|||||||
let (mcp_server_url, mcp_server_handle) = start_mcp_server(sensitive_action).await?;
|
let (mcp_server_url, mcp_server_handle) = start_mcp_server(sensitive_action).await?;
|
||||||
|
|
||||||
let codex_home = TempDir::new()?;
|
let codex_home = TempDir::new()?;
|
||||||
|
let analytics_server = responses::start_mock_server().await;
|
||||||
|
mount_analytics_capture(&analytics_server, codex_home.path()).await?;
|
||||||
let root_skill = if matches!(lifecycle, ThreadLifecycle::RootTrustedSkill) {
|
let root_skill = if matches!(lifecycle, ThreadLifecycle::RootTrustedSkill) {
|
||||||
let path = codex_home.path().join("skills/root-trusted/SKILL.md");
|
let path = codex_home.path().join("skills/root-trusted/SKILL.md");
|
||||||
std::fs::create_dir_all(path.parent().expect("root skill parent"))?;
|
std::fs::create_dir_all(path.parent().expect("root skill parent"))?;
|
||||||
@@ -700,7 +705,10 @@ async fn guardian_v2_routes_scoped_tool_approvals(
|
|||||||
.with_model(MODEL)
|
.with_model(MODEL)
|
||||||
.with_provider_config("supports_websockets = false")
|
.with_provider_config("supports_websockets = false")
|
||||||
.with_approval_policy("on-request")
|
.with_approval_policy("on-request")
|
||||||
.with_root_config(reviewer_config)
|
.with_root_config(&format!(
|
||||||
|
"{reviewer_config}\nchatgpt_base_url = \"{}\"",
|
||||||
|
analytics_server.uri(),
|
||||||
|
))
|
||||||
.with_extra_config(&format!(
|
.with_extra_config(&format!(
|
||||||
"[mcp_servers.{server_name}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"{tool_approval_mode}\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}{guardian_scope_config}"
|
"[mcp_servers.{server_name}]\nurl = \"{mcp_server_url}/mcp\"\ndefault_tools_approval_mode = \"{tool_approval_mode}\"\n\n[analytics]\nenabled = true\n\n[otel]\nmetrics_exporter = {{ otlp-http = {{ endpoint = \"{responses_url}/metrics\", protocol = \"json\" }} }}{guardian_scope_config}"
|
||||||
))
|
))
|
||||||
@@ -1358,6 +1366,77 @@ async fn guardian_v2_routes_scoped_tool_approvals(
|
|||||||
timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?;
|
timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if matches!(lifecycle, ThreadLifecycle::New)
|
||||||
|
&& matches!(scope, GuardianToolScope::AllTools)
|
||||||
|
&& sensitive_action.is_none()
|
||||||
|
{
|
||||||
|
if classifier_in_scope {
|
||||||
|
wait_for_matching_analytics_event(&analytics_server, TIMEOUT, |event| {
|
||||||
|
event["event_type"] == "codex_guardian_v2_classification"
|
||||||
|
&& event["event_params"]["item_id"] == "guardian-action-0"
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
timeout(TIMEOUT, app_server.shutdown_gracefully()).await??;
|
||||||
|
let events = captured_analytics_events(&analytics_server).await;
|
||||||
|
let turn = &events
|
||||||
|
.iter()
|
||||||
|
.find(|event| {
|
||||||
|
event["event_type"] == "codex_turn_event"
|
||||||
|
&& event["event_params"]["thread_id"] == reviewed_thread_id
|
||||||
|
})
|
||||||
|
.expect("parent turn analytics")["event_params"];
|
||||||
|
assert_eq!(turn["guardian_v2_enabled"], classifier_in_scope);
|
||||||
|
let classification = events.iter().find(|event| {
|
||||||
|
event["event_type"] == "codex_guardian_v2_classification"
|
||||||
|
&& event["event_params"]["item_id"] == "guardian-action-0"
|
||||||
|
});
|
||||||
|
assert_eq!(classification.is_some(), classifier_in_scope);
|
||||||
|
if let Some(event) = classification {
|
||||||
|
assert_eq!(
|
||||||
|
json!([
|
||||||
|
event["event_params"]["outcome"],
|
||||||
|
event["event_params"]["risk_level"]
|
||||||
|
]),
|
||||||
|
match risk {
|
||||||
|
GuardianRisk::Low => json!(["success", "low"]),
|
||||||
|
GuardianRisk::Threshold | GuardianRisk::High => json!(["success", "high"]),
|
||||||
|
GuardianRisk::InvalidResponse => json!(["failure", null]),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let approvals = events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| event["event_type"] == "codex_guardian_v2_fast_decision")
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
approvals.len(),
|
||||||
|
usize::from(classifier_in_scope && matches!(risk, GuardianRisk::Low))
|
||||||
|
);
|
||||||
|
for event in classification.into_iter().chain(approvals) {
|
||||||
|
let params = &event["event_params"];
|
||||||
|
for key in [
|
||||||
|
"session_id",
|
||||||
|
"thread_id",
|
||||||
|
"turn_id",
|
||||||
|
"model",
|
||||||
|
"app_server_client",
|
||||||
|
"runtime",
|
||||||
|
"thread_source",
|
||||||
|
"subagent_source",
|
||||||
|
"parent_thread_id",
|
||||||
|
] {
|
||||||
|
assert_eq!(params[key], turn[key], "{key}");
|
||||||
|
}
|
||||||
|
if event["event_type"] == "codex_guardian_v2_fast_decision" {
|
||||||
|
assert_eq!(
|
||||||
|
json!([params["item_id"], params["decision"]]),
|
||||||
|
json!(["guardian-action-1", "approved"])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
mcp_server_handle.abort();
|
mcp_server_handle.abort();
|
||||||
responses_server.abort();
|
responses_server.abort();
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -5,8 +5,11 @@ use codex_analytics::GuardianReviewFailureReason;
|
|||||||
use codex_analytics::GuardianReviewTerminalStatus;
|
use codex_analytics::GuardianReviewTerminalStatus;
|
||||||
use codex_analytics::GuardianReviewTrackContext;
|
use codex_analytics::GuardianReviewTrackContext;
|
||||||
use codex_analytics::GuardianReviewedAction;
|
use codex_analytics::GuardianReviewedAction;
|
||||||
|
use codex_analytics::GuardianV2Event;
|
||||||
|
use codex_analytics::GuardianV2EventKind;
|
||||||
use codex_async_utils::THREAD_STACK_SIZE_BYTES;
|
use codex_async_utils::THREAD_STACK_SIZE_BYTES;
|
||||||
use codex_core_plugins::PluginCommandAttribution;
|
use codex_core_plugins::PluginCommandAttribution;
|
||||||
|
use codex_extension_api::GuardianV2Enabled;
|
||||||
use codex_extension_api::ThreadIdleCause;
|
use codex_extension_api::ThreadIdleCause;
|
||||||
use codex_features::Feature;
|
use codex_features::Feature;
|
||||||
use codex_protocol::config_types::ApprovalsReviewer;
|
use codex_protocol::config_types::ApprovalsReviewer;
|
||||||
@@ -362,6 +365,26 @@ async fn run_guardian_review(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
if decision == ReviewDecision::Approved {
|
if decision == ReviewDecision::Approved {
|
||||||
|
if session
|
||||||
|
.services
|
||||||
|
.thread_extension_data
|
||||||
|
.get::<GuardianV2Enabled>()
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
session
|
||||||
|
.services
|
||||||
|
.analytics_events_client
|
||||||
|
.track_guardian_v2_event(GuardianV2Event {
|
||||||
|
thread_id: session.thread_id.to_string(),
|
||||||
|
turn_id: guardian_request_turn_id(&request, &turn.sub_id).to_owned(),
|
||||||
|
item_id: guardian_request_target_item_id(&request).map(str::to_owned),
|
||||||
|
model: Some(turn.model_info().slug.clone()),
|
||||||
|
occurred_at_ms: codex_analytics::now_unix_millis(),
|
||||||
|
kind: GuardianV2EventKind::FastDecision {
|
||||||
|
decision: "approved",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
record_guardian_non_denial(&session, guardian_request_turn_id(&request, &turn.sub_id))
|
record_guardian_non_denial(&session, guardian_request_turn_id(&request, &turn.sub_id))
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1029,6 +1029,18 @@ async fn track_turn_resolved_config_analytics(
|
|||||||
.and_then(ServiceTier::from_request_value),
|
.and_then(ServiceTier::from_request_value),
|
||||||
approval_policy: turn_context.approval_policy(),
|
approval_policy: turn_context.approval_policy(),
|
||||||
approvals_reviewer: turn_context.config.approvals_reviewer,
|
approvals_reviewer: turn_context.config.approvals_reviewer,
|
||||||
|
guardian_v2_enabled: sess
|
||||||
|
.services
|
||||||
|
.thread_extension_data
|
||||||
|
.get::<codex_extension_api::GuardianV2Enabled>()
|
||||||
|
.is_some_and(|state| {
|
||||||
|
state.computer_use_only
|
||||||
|
|| !turn_context
|
||||||
|
.config
|
||||||
|
.config_layer_stack
|
||||||
|
.requirements()
|
||||||
|
.auto_review_required_for_model(&turn_context.model_info().slug)
|
||||||
|
}),
|
||||||
sandbox_network_access: turn_context.network_sandbox_policy().is_enabled(),
|
sandbox_network_access: turn_context.network_sandbox_policy().is_enabled(),
|
||||||
collaboration_mode: turn_context.mode(),
|
collaboration_mode: turn_context.mode(),
|
||||||
personality: turn_context.personality(),
|
personality: turn_context.personality(),
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ mod world_state;
|
|||||||
pub use approval_review::ApprovalAssessment;
|
pub use approval_review::ApprovalAssessment;
|
||||||
pub use approval_review::ApprovalReviewError;
|
pub use approval_review::ApprovalReviewError;
|
||||||
pub use approval_review::ApprovalReviewInput;
|
pub use approval_review::ApprovalReviewInput;
|
||||||
|
pub use approval_review::GuardianV2Enabled;
|
||||||
pub use context::TurnContextContributionInput;
|
pub use context::TurnContextContributionInput;
|
||||||
pub use mcp::McpServerContribution;
|
pub use mcp::McpServerContribution;
|
||||||
pub use mcp::McpServerContributionContext;
|
pub use mcp::McpServerContributionContext;
|
||||||
|
|||||||
@@ -10,6 +10,12 @@ use serde_json::Value;
|
|||||||
use crate::ConversationHistorySnapshot;
|
use crate::ConversationHistorySnapshot;
|
||||||
use crate::ExtensionData;
|
use crate::ExtensionData;
|
||||||
|
|
||||||
|
/// Thread-local state installed only after Guardian V2's async classifier initializes.
|
||||||
|
pub struct GuardianV2Enabled {
|
||||||
|
/// Computer-use classification remains active for models requiring synchronous review.
|
||||||
|
pub computer_use_only: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Classification returned by an approval reviewer.
|
/// Classification returned by an approval reviewer.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct ApprovalAssessment {
|
pub struct ApprovalAssessment {
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ pub use contributors::ApprovalReviewInput;
|
|||||||
pub use contributors::ConfigContributor;
|
pub use contributors::ConfigContributor;
|
||||||
pub use contributors::ContextContributor;
|
pub use contributors::ContextContributor;
|
||||||
pub use contributors::ExtensionFuture;
|
pub use contributors::ExtensionFuture;
|
||||||
|
pub use contributors::GuardianV2Enabled;
|
||||||
pub use contributors::McpServerContribution;
|
pub use contributors::McpServerContribution;
|
||||||
pub use contributors::McpServerContributionContext;
|
pub use contributors::McpServerContributionContext;
|
||||||
pub use contributors::McpServerContributor;
|
pub use contributors::McpServerContributor;
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ doctest = false
|
|||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
codex-analytics = { workspace = true }
|
||||||
codex-api = { workspace = true }
|
codex-api = { workspace = true }
|
||||||
codex-core = { workspace = true }
|
codex-core = { workspace = true }
|
||||||
codex-extension-api = { workspace = true }
|
codex-extension-api = { workspace = true }
|
||||||
|
|||||||
@@ -7,6 +7,9 @@ use std::sync::atomic::Ordering;
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
use codex_analytics::AnalyticsEventsClient;
|
||||||
|
use codex_analytics::GuardianV2Event;
|
||||||
|
use codex_analytics::GuardianV2EventKind;
|
||||||
use codex_core::ThreadManager;
|
use codex_core::ThreadManager;
|
||||||
use codex_core::config::Config;
|
use codex_core::config::Config;
|
||||||
use codex_core::context::GuardianReviewEvidence;
|
use codex_core::context::GuardianReviewEvidence;
|
||||||
@@ -18,6 +21,7 @@ use codex_extension_api::ExtensionFuture;
|
|||||||
use codex_extension_api::ExtensionMetrics;
|
use codex_extension_api::ExtensionMetrics;
|
||||||
use codex_extension_api::ExtensionRegistryBuilder;
|
use codex_extension_api::ExtensionRegistryBuilder;
|
||||||
use codex_extension_api::ExtensionWarning;
|
use codex_extension_api::ExtensionWarning;
|
||||||
|
use codex_extension_api::GuardianV2Enabled;
|
||||||
use codex_extension_api::ResponseItem;
|
use codex_extension_api::ResponseItem;
|
||||||
use codex_extension_api::SkillInvocationContributor;
|
use codex_extension_api::SkillInvocationContributor;
|
||||||
use codex_extension_api::SkillInvocationInput;
|
use codex_extension_api::SkillInvocationInput;
|
||||||
@@ -103,8 +107,6 @@ pub enum StrictReviewReason {
|
|||||||
StaleScore,
|
StaleScore,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct GuardianV2Enabled;
|
|
||||||
|
|
||||||
enum ClassificationOutcome {
|
enum ClassificationOutcome {
|
||||||
Scored,
|
Scored,
|
||||||
Superseded,
|
Superseded,
|
||||||
@@ -196,6 +198,10 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
|
|||||||
let sampler = input
|
let sampler = input
|
||||||
.thread_store
|
.thread_store
|
||||||
.get_or_init(|| LunaSampler::new(sampler_config));
|
.get_or_init(|| LunaSampler::new(sampler_config));
|
||||||
|
let guardian_v2_enabled = GuardianV2Enabled {
|
||||||
|
computer_use_only: guardian_config.review_scope
|
||||||
|
== GuardianV2ReviewScope::ComputerUseOnly,
|
||||||
|
};
|
||||||
input.thread_store.insert(guardian_config);
|
input.thread_store.insert(guardian_config);
|
||||||
input.thread_store.insert(GuardianV2ScoreProgress {
|
input.thread_store.insert(GuardianV2ScoreProgress {
|
||||||
metrics: input.extension_metrics.clone(),
|
metrics: input.extension_metrics.clone(),
|
||||||
@@ -205,7 +211,7 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
|
|||||||
input
|
input
|
||||||
.thread_store
|
.thread_store
|
||||||
.insert(TrustedSkillRoots::from_config(input.config));
|
.insert(TrustedSkillRoots::from_config(input.config));
|
||||||
input.thread_store.insert(GuardianV2Enabled);
|
input.thread_store.insert(guardian_v2_enabled);
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
sampler.prewarm().await;
|
sampler.prewarm().await;
|
||||||
@@ -431,6 +437,7 @@ impl GuardianV2Extension {
|
|||||||
.fetch_add(/*val*/ 1, Ordering::Relaxed);
|
.fetch_add(/*val*/ 1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
let metrics = score_progress.metrics.clone();
|
let metrics = score_progress.metrics.clone();
|
||||||
|
let analytics = input.session_store.get::<AnalyticsEventsClient>();
|
||||||
let sampled_at = SystemTime::now();
|
let sampled_at = SystemTime::now();
|
||||||
let tool_call_index = score_progress
|
let tool_call_index = score_progress
|
||||||
.latest_tool_call
|
.latest_tool_call
|
||||||
@@ -673,6 +680,7 @@ impl GuardianV2Extension {
|
|||||||
format!("{planned_action}\n"),
|
format!("{planned_action}\n"),
|
||||||
">>> APPROVAL REQUEST END\n".to_owned(),
|
">>> APPROVAL REQUEST END\n".to_owned(),
|
||||||
]);
|
]);
|
||||||
|
let mut classification_risk = None;
|
||||||
let mut classification_finished_at = None;
|
let mut classification_finished_at = None;
|
||||||
let result: Result<ClassificationOutcome, String> = async {
|
let result: Result<ClassificationOutcome, String> = async {
|
||||||
let review_model_messages = if config.guardian_policy_config.is_none() {
|
let review_model_messages = if config.guardian_policy_config.is_none() {
|
||||||
@@ -722,11 +730,12 @@ impl GuardianV2Extension {
|
|||||||
}
|
}
|
||||||
Err(error) => return Err(error.to_string()),
|
Err(error) => return Err(error.to_string()),
|
||||||
};
|
};
|
||||||
let action_risk = match output.as_str() {
|
let (action_risk, risk_level) = match output.as_str() {
|
||||||
"high" => 1.0,
|
"high" => (1.0, "high"),
|
||||||
"low" => 0.0,
|
"low" => (0.0, "low"),
|
||||||
_ => return Err("invalid Guardian V2 classification".to_owned()),
|
_ => return Err("invalid Guardian V2 classification".to_owned()),
|
||||||
};
|
};
|
||||||
|
classification_risk = Some(risk_level);
|
||||||
let score = SecurityRiskScore {
|
let score = SecurityRiskScore {
|
||||||
scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]),
|
scores: BTreeMap::from([("action_risk".to_owned(), action_risk)]),
|
||||||
call_id: Some(call_id.clone()),
|
call_id: Some(call_id.clone()),
|
||||||
@@ -794,19 +803,29 @@ impl GuardianV2Extension {
|
|||||||
if result.is_err() {
|
if result.is_err() {
|
||||||
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
|
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
|
||||||
}
|
}
|
||||||
record_classification(
|
let duration = classification_finished_at
|
||||||
metrics.as_deref(),
|
.map(|finished_at: Instant| finished_at.duration_since(classification_started_at))
|
||||||
classification_finished_at
|
.unwrap_or_else(|| classification_started_at.elapsed());
|
||||||
.map(|finished_at: Instant| {
|
let outcome = match &result {
|
||||||
finished_at.duration_since(classification_started_at)
|
Ok(ClassificationOutcome::Scored) => "success",
|
||||||
})
|
Ok(ClassificationOutcome::Superseded) => "superseded",
|
||||||
.unwrap_or_else(|| classification_started_at.elapsed()),
|
Err(_) => "failure",
|
||||||
match &result {
|
};
|
||||||
Ok(ClassificationOutcome::Scored) => "success",
|
record_classification(metrics.as_deref(), duration, outcome);
|
||||||
Ok(ClassificationOutcome::Superseded) => "superseded",
|
if let Some(analytics) = analytics {
|
||||||
Err(_) => "failure",
|
analytics.track_guardian_v2_event(GuardianV2Event {
|
||||||
},
|
thread_id: thread_id.clone(),
|
||||||
);
|
turn_id: turn_id.clone(),
|
||||||
|
item_id: Some(call_id),
|
||||||
|
model: parent_model.as_ref().map(|model| model.slug.clone()),
|
||||||
|
occurred_at_ms: codex_analytics::now_unix_millis(),
|
||||||
|
kind: GuardianV2EventKind::Classification {
|
||||||
|
outcome,
|
||||||
|
risk_level: classification_risk,
|
||||||
|
duration_ms: u64::try_from(duration.as_millis()).unwrap_or(u64::MAX),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
if matches!(result, Ok(ClassificationOutcome::Scored)) {
|
if matches!(result, Ok(ClassificationOutcome::Scored)) {
|
||||||
truncations.emit(metrics.as_deref());
|
truncations.emit(metrics.as_deref());
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user