mirror of
https://github.com/openai/codex.git
synced 2026-11-13 06:45:07 +00:00
Attach failed Guardian reviews to diagnostic reports (#41936)
## What changed - Retain bounded, process-local records for failed Guardian reviews, including the reviewed action, decision, status, model, instructions, and reviewer history. - Add recent records from the reported thread and its descendants to log-enabled report uploads as `auto-review-failures.jsonl`. - Preserve the action and decision while omitting oversized reviewer context, and exclude successful, ephemeral, and capture-disabled reviews. ## Testing - Cover denied and invalid decisions, cleaned-up ephemeral reviewers, descendant-thread selection, record count and byte limits, and oversized-context fallback. GitOrigin-RevId: 5f12f7605041c3c84e826e6bac1cf06b1518ab24
This commit is contained in:
1 parent
981d6b652b
commit
13d75cd1c3
12 files changed
+439
-13
No files matched your search
@@ -27,8 +27,10 @@ use codex_login::CodexAuth;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::config_types::WindowsSandboxLevel;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::PermissionProfileSnapshot;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::AutoReviewMessages;
|
||||
use codex_protocol::openai_models::MODEL_SPECIALTY_CYBER;
|
||||
use codex_protocol::openai_models::ModelsResponse;
|
||||
@@ -1324,6 +1326,15 @@ async fn guardian_denial_rejects_tool_call_with_rationale(
|
||||
),
|
||||
ev_completed("resp-parent-tool-denied"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-guardian-inspect"),
|
||||
ev_function_call(
|
||||
"exec-guardian-inspect",
|
||||
"exec_command",
|
||||
&json!({"cmd": "printf guardian-inspection-evidence"}).to_string(),
|
||||
),
|
||||
ev_completed("resp-guardian-inspect"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-guardian-denied"),
|
||||
ev_assistant_message(
|
||||
@@ -1374,6 +1385,67 @@ async fn guardian_denial_rejects_tool_call_with_rationale(
|
||||
assert!(guardian_request.body_contains_text(&command));
|
||||
assert_eq!(guardian_request.body_json()["model"], "gpt-5.6-luna");
|
||||
|
||||
let feedback =
|
||||
codex_feedback::guardian_review_failures_attachment(&[test.session_configured.thread_id])
|
||||
.expect("failed Guardian review");
|
||||
let record: serde_json::Value = serde_json::from_slice(&feedback.buffer)?;
|
||||
assert!(
|
||||
guardian_request.body_contains_text(record["action"].as_str().expect("reviewed action"))
|
||||
);
|
||||
let recorded_history: Vec<ResponseItem> = serde_json::from_value(record["history"].clone())?;
|
||||
let inspection_request = requests
|
||||
.iter()
|
||||
.find(|request| {
|
||||
request
|
||||
.function_call_output_text("exec-guardian-inspect")
|
||||
.is_some()
|
||||
})
|
||||
.expect("Guardian request following its inspection tool");
|
||||
assert!(
|
||||
inspection_request
|
||||
.function_call_output_text("exec-guardian-inspect")
|
||||
.expect("inspection output")
|
||||
.contains("guardian-inspection-evidence")
|
||||
);
|
||||
let inspection_output = inspection_request
|
||||
.input()
|
||||
.into_iter()
|
||||
.find(|item| {
|
||||
item["type"] == "function_call_output" && item["call_id"] == "exec-guardian-inspect"
|
||||
})
|
||||
.expect("inspection response item");
|
||||
assert!(recorded_history.contains(&serde_json::from_value(inspection_output)?));
|
||||
assert!(recorded_history.iter().any(|item| {
|
||||
matches!(item, ResponseItem::Message { role, content, .. }
|
||||
if role == "assistant" && content.iter().any(|part| {
|
||||
matches!(part, ContentItem::OutputText { text }
|
||||
if Some(text.as_str()) == record["decision"].as_str())
|
||||
}))
|
||||
}));
|
||||
assert_eq!(
|
||||
json!({
|
||||
"reviewed_thread_id": record["reviewed_thread_id"],
|
||||
"reviewer_thread_id": record["reviewer_thread_id"],
|
||||
"status": record["status"],
|
||||
"decision": serde_json::from_str::<serde_json::Value>(
|
||||
record["decision"].as_str().expect("raw Guardian decision"),
|
||||
)?,
|
||||
"context_omitted": record["context_omitted"],
|
||||
}),
|
||||
json!({
|
||||
"reviewed_thread_id": test.session_configured.thread_id,
|
||||
"reviewer_thread_id": guardian_request.body_json()["client_metadata"]["thread_id"],
|
||||
"status": "denied",
|
||||
"decision": {
|
||||
"risk_level": "high",
|
||||
"user_authorization": "low",
|
||||
"outcome": "deny",
|
||||
"rationale": "The requested write has unacceptable test risk.",
|
||||
},
|
||||
"context_omitted": false,
|
||||
})
|
||||
);
|
||||
|
||||
let tool_output = requests
|
||||
.iter()
|
||||
.find_map(|request| request.function_call_output_text("exec-call-denied"))
|
||||
|
||||
@@ -323,5 +323,34 @@ async fn guardian_subagent_review_preserves_late_root_user_authorization() -> Re
|
||||
assert!(guardian_transcript.contains("assistant: Agent message from /root"));
|
||||
assert!(guardian_transcript.contains(FORWARDED_AGENT_MESSAGE));
|
||||
|
||||
let feedback_thread_ids = test
|
||||
.thread_manager
|
||||
.list_agent_subtree_thread_ids(root_thread_id)
|
||||
.await?;
|
||||
let feedback = codex_feedback::guardian_review_failures_attachment(&feedback_thread_ids)
|
||||
.expect("failed worker review");
|
||||
let record: Value = serde_json::from_slice(&feedback.buffer)?;
|
||||
assert_eq!(
|
||||
json!({
|
||||
"reviewed_thread_id": record["reviewed_thread_id"],
|
||||
"reviewer_thread_id": record["reviewer_thread_id"],
|
||||
"status": record["status"],
|
||||
"decision": serde_json::from_str::<Value>(
|
||||
record["decision"].as_str().expect("raw Guardian decision"),
|
||||
)?,
|
||||
}),
|
||||
json!({
|
||||
"reviewed_thread_id": worker_thread_id,
|
||||
"reviewer_thread_id": guardian_review.single_request().body_json()["client_metadata"]["thread_id"],
|
||||
"status": "denied",
|
||||
"decision": {
|
||||
"risk_level": "high",
|
||||
"user_authorization": "high",
|
||||
"outcome": "deny",
|
||||
"rationale": "The agent message requests a different action.",
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user