mirror of
https://github.com/openai/codex.git
synced 2026-11-10 00:42:33 +00:00
Log Guardian V2 classification results (#40028)
## What changed - Emit a structured log event for each completed Guardian V2 classification with its thread, turn, tool call, risk score, review threshold, sample time, and whether the score was accepted or superseded. - Verify that accepted async classification events appear in the reviewed thread's log export with the expected context and decision fields. GitOrigin-RevId: a027d5991b12a1968b9327da57e2993f0a529ca1
This commit is contained in:
1 parent
9445ef227e
commit
50ea8fd411
4 files changed
+60
-6
No files matched your search
Generated
+1
@@ -3320,6 +3320,7 @@ dependencies = [
|
|||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ use codex_app_server_protocol::TurnStartParams;
|
|||||||
use codex_app_server_protocol::TurnStartResponse;
|
use codex_app_server_protocol::TurnStartResponse;
|
||||||
use codex_app_server_protocol::UserInput;
|
use codex_app_server_protocol::UserInput;
|
||||||
use codex_features::Feature;
|
use codex_features::Feature;
|
||||||
|
use codex_state::StateRuntime;
|
||||||
|
use codex_utils_absolute_path::test_support::PathExt;
|
||||||
use core_test_support::responses;
|
use core_test_support::responses;
|
||||||
use core_test_support::skip_if_no_network;
|
use core_test_support::skip_if_no_network;
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
@@ -623,6 +625,43 @@ async fn guardian_v2_routes_tool_approvals(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if matches!(requirement, ModelReviewRequirement::Optional) {
|
||||||
|
let state_db = StateRuntime::init(
|
||||||
|
codex_state::SqliteConfig::new_for_testing(codex_home.path().abs()),
|
||||||
|
"mock_provider".to_owned(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// Exercise the same log export used by feedback/upload, including async
|
||||||
|
// classifier events that cannot rely on inheriting a thread tracing span.
|
||||||
|
let logs = timeout(TIMEOUT, async {
|
||||||
|
loop {
|
||||||
|
let logs = String::from_utf8(
|
||||||
|
state_db
|
||||||
|
.query_feedback_logs_for_threads(&[&reviewed_thread_id])
|
||||||
|
.await?,
|
||||||
|
)?;
|
||||||
|
if logs.contains("Guardian V2 classification result") {
|
||||||
|
return anyhow::Ok(logs);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
|
let expected = [
|
||||||
|
"Guardian V2 classification result".to_owned(),
|
||||||
|
"call_id=guardian-action-0".into(),
|
||||||
|
format!("thread_id={reviewed_thread_id}"),
|
||||||
|
format!("action_risk={luna_score}"),
|
||||||
|
"review_threshold=0.5".into(),
|
||||||
|
"accepted=true".into(),
|
||||||
|
];
|
||||||
|
assert!(
|
||||||
|
logs.lines()
|
||||||
|
.any(|line| expected.iter().all(|field| line.contains(field))),
|
||||||
|
"missing feedback log with fields: {expected:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if matches!(
|
if matches!(
|
||||||
lifecycle,
|
lifecycle,
|
||||||
ThreadLifecycle::RootRollback
|
ThreadLifecycle::RootRollback
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ http = { workspace = true }
|
|||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["time"] }
|
tokio = { workspace = true, features = ["time"] }
|
||||||
|
tracing = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
|
|||||||
@@ -566,6 +566,7 @@ impl GuardianV2Extension {
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let call_id = input.call_id.to_owned();
|
||||||
let action = GuardianAction {
|
let action = GuardianAction {
|
||||||
tool_name: input.tool_name.clone(),
|
tool_name: input.tool_name.clone(),
|
||||||
payload: input.payload.clone(),
|
payload: input.payload.clone(),
|
||||||
@@ -741,12 +742,24 @@ impl GuardianV2Extension {
|
|||||||
scores,
|
scores,
|
||||||
sampled_at: Some(sampled_at.into()),
|
sampled_at: Some(sampled_at.into()),
|
||||||
};
|
};
|
||||||
if !thread
|
let accepted =
|
||||||
.thread_extension_data()
|
thread
|
||||||
.insert_if(score.clone(), |previous| {
|
.thread_extension_data()
|
||||||
previous.is_none_or(|previous| previous.sampled_at < score.sampled_at)
|
.insert_if(score.clone(), |previous| {
|
||||||
})
|
previous.is_none_or(|previous| previous.sampled_at < score.sampled_at)
|
||||||
{
|
});
|
||||||
|
tracing::info!(
|
||||||
|
%thread_id,
|
||||||
|
%turn_id,
|
||||||
|
%call_id,
|
||||||
|
tool_call_index,
|
||||||
|
action_risk = score.scores.get("action_risk").copied(),
|
||||||
|
review_threshold = guardian_config.review_threshold,
|
||||||
|
sampled_at = ?score.sampled_at,
|
||||||
|
accepted,
|
||||||
|
"Guardian V2 classification result"
|
||||||
|
);
|
||||||
|
if !accepted {
|
||||||
return Ok("superseded");
|
return Ok("superseded");
|
||||||
}
|
}
|
||||||
score_progress
|
score_progress
|
||||||
|
|||||||
Reference in new issue
Block a user