Attach failed Guardian reviews to diagnostic reports (#41936)

## What changed

- Retain bounded, process-local records for failed Guardian reviews, including the reviewed action, decision, status, model, instructions, and reviewer history.
- Add recent records from the reported thread and its descendants to log-enabled report uploads as `auto-review-failures.jsonl`.
- Preserve the action and decision while omitting oversized reviewer context, and exclude successful, ephemeral, and capture-disabled reviews.

## Testing

- Cover denied and invalid decisions, cleaned-up ephemeral reviewers, descendant-thread selection, record count and byte limits, and oversized-context fallback.

GitOrigin-RevId: 5f12f7605041c3c84e826e6bac1cf06b1518ab24
This commit is contained in:
jif authored and copyberry committed 2026-08-31 23:44:41 +00:00
1 parent 981d6b652b
commit 13d75cd1c3
12 files changed
+439 -13

No files matched your search

+119
View File
@@ -0,0 +1,119 @@
//! Captures bounded failed-review evidence for opt-in feedback uploads.
//! Records outlive replaced and short-lived reviewers without writing extra history files.
use super::GuardianAssessmentOutcome;
use super::approval_request::format_guardian_action_pretty;
use super::prompt::parse_guardian_assessment;
use super::review_session::GuardianReviewSessionOutcome;
use super::review_session::GuardianReviewSessionParams;
use crate::session::session::Session;
use codex_feedback::record_guardian_review_failure;
use codex_protocol::ThreadId;
use codex_protocol::models::ResponseItem;
use serde::Serialize;
use std::io;
use std::io::Write;
const MAX_RECORD_BYTES: usize = 4 * 1024 * 1024;
#[derive(Serialize)]
struct ReviewFeedbackRecord<'a> {
reviewed_thread_id: ThreadId,
reviewed_turn_id: &'a str,
reviewer_thread_id: ThreadId,
model: &'a str,
status: &'a str,
decision: Option<&'a str>,
action: &'a str,
action_truncated: bool,
instructions: Option<&'a str>,
history: Vec<&'a ResponseItem>,
context_omitted: bool,
}
pub(super) async fn record_failed_review(
reviewer: &Session,
params: &GuardianReviewSessionParams,
outcome: &GuardianReviewSessionOutcome,
) {
if !params.spawn_config.feedback_enabled || params.spawn_config.ephemeral {
return;
}
let (status, decision) = match outcome {
GuardianReviewSessionOutcome::Completed(Ok(Some(decision))) => {
match parse_guardian_assessment(Some(decision)) {
Ok(assessment) if assessment.outcome == GuardianAssessmentOutcome::Allow => {
return;
}
Ok(_) => ("denied", Some(decision.as_str())),
Err(_) => ("invalid_decision", Some(decision.as_str())),
}
}
GuardianReviewSessionOutcome::Completed(Ok(None)) => ("missing_decision", None),
GuardianReviewSessionOutcome::Completed(Err(_))
| GuardianReviewSessionOutcome::PromptBuildFailed(_)
| GuardianReviewSessionOutcome::SessionFailed { .. } => ("failed", None),
GuardianReviewSessionOutcome::TimedOut => ("timed_out", None),
GuardianReviewSessionOutcome::Aborted => ("aborted", None),
};
let Ok(action) = format_guardian_action_pretty(&params.request) else {
tracing::warn!("Could not serialize Guardian feedback action");
return;
};
let instructions = reviewer.get_prompt_base_instructions().await;
let history = reviewer.clone_history().await;
ReviewFeedbackRecord {
reviewed_thread_id: params.parent_session.thread_id(),
reviewed_turn_id: &params.parent_context.turn().sub_id,
reviewer_thread_id: reviewer.thread_id(),
model: &params.model,
status,
decision,
action: &action.text,
action_truncated: action.truncated,
instructions: Some(&instructions.text),
history: history.raw_items().collect(),
context_omitted: false,
}
.store();
}
impl ReviewFeedbackRecord<'_> {
fn store(mut self) {
let mut buffer = BoundedBuffer(Vec::new());
if serde_json::to_writer(&mut buffer, &self).is_err() {
// Preserve the action and decision even when optional reviewer history is too large.
self.instructions = None;
self.history.clear();
self.context_omitted = true;
buffer = BoundedBuffer(Vec::new());
if serde_json::to_writer(&mut buffer, &self).is_err() {
tracing::warn!("Guardian feedback action and decision exceed the record limit");
return;
}
}
record_guardian_review_failure(self.reviewed_thread_id, buffer.0);
}
}
struct BoundedBuffer(Vec<u8>);
impl Write for BoundedBuffer {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
if bytes.len() > MAX_RECORD_BYTES.saturating_sub(self.0.len()) {
return Err(io::Error::other(
"Guardian feedback record exceeds its size limit",
));
}
self.0.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[cfg(test)]
#[path = "feedback_tests.rs"]
mod tests;
@@ -0,0 +1,45 @@
use super::*;
use pretty_assertions::assert_eq;
use serde_json::json;
#[test]
fn oversized_context_keeps_the_action_and_decision() -> anyhow::Result<()> {
let thread_id = ThreadId::new();
let reviewer_thread_id = ThreadId::new();
let decision = r#"{"outcome":"deny","rationale":"Missing approval."}"#;
let action = r#"{"command":"git push"}"#;
ReviewFeedbackRecord {
reviewed_thread_id: thread_id,
reviewed_turn_id: "parent-turn",
reviewer_thread_id,
model: "review-model",
status: "denied",
decision: Some(decision),
action,
action_truncated: false,
instructions: Some(&"x".repeat(MAX_RECORD_BYTES)),
history: Vec::new(),
context_omitted: false,
}
.store();
let contents = codex_feedback::guardian_review_failures_attachment(&[thread_id])
.expect("failed-review record")
.buffer;
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&contents)?,
json!({
"reviewed_thread_id": thread_id,
"reviewed_turn_id": "parent-turn",
"reviewer_thread_id": reviewer_thread_id,
"model": "review-model",
"status": "denied",
"decision": decision,
"action": action,
"action_truncated": false,
"instructions": null,
"history": [],
"context_omitted": true,
})
);
Ok(())
}
+1
View File
@@ -12,6 +12,7 @@
//! 4. Apply the guardian's explicit allow/deny outcome.
mod approval_request;
mod feedback;
mod metrics;
mod prompt;
mod review;
@@ -79,6 +79,7 @@ use super::ApprovalRequestReasons;
use super::GUARDIAN_REVIEWER_NAME;
use super::GuardianApprovalRequest;
use super::GuardianReviewContext;
use super::feedback::record_failed_review;
#[cfg(test)]
use super::prompt::BUNDLED_GUARDIAN_POLICY;
use super::prompt::BUNDLED_GUARDIAN_POLICY_TEMPLATE;
@@ -652,6 +653,7 @@ impl GuardianReviewSessionManager {
deadline,
))
.await;
record_failed_review(&trunk.session, &params, &outcome).await;
if keep_review_session && matches!(outcome, GuardianReviewSessionOutcome::Completed(_)) {
trunk.refresh_last_committed_fork_snapshot().await;
}
@@ -826,6 +828,7 @@ impl GuardianReviewSessionManager {
deadline,
))
.await;
record_failed_review(&review_session.session, &params, &outcome).await;
if let Some(review_session) = self.take_active_ephemeral(&review_session).await {
cleanup.disarm();
review_session.shutdown_in_background();
+33 -11
View File
@@ -3349,15 +3349,14 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
{
const TEST_STACK_SIZE_BYTES: usize = 4 * 1024 * 1024;
let handle =
std::thread::Builder::new()
.name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string())
.stack_size(TEST_STACK_SIZE_BYTES)
.spawn(|| -> anyhow::Result<()> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime.block_on(Box::pin(async {
let handle = std::thread::Builder::new()
.name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string())
.stack_size(TEST_STACK_SIZE_BYTES)
.spawn(|| -> anyhow::Result<()> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime.block_on(Box::pin(async {
let first_assessment = serde_json::json!({
"risk_level": "low",
"user_authorization": "high",
@@ -3421,7 +3420,11 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
])
.await;
let (session, turn) = guardian_test_session_and_turn_with_base_url(server.uri()).await;
let (mut session, turn) = guardian_test_session_and_turn_with_base_url(server.uri()).await;
// Isolate feedback from other tests using the fixed parent session ID.
Arc::get_mut(&mut session)
.expect("session should be uniquely owned")
.thread_id = ThreadId::new();
turn.turn_metadata_state
.set_parent_turn_id("upstream-parent-turn".to_string());
turn.turn_metadata_state
@@ -3616,11 +3619,30 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
.send(())
.expect("second guardian review gate should still be open");
assert_eq!(second_review.await?, ReviewDecision::Approved);
let feedback = codex_feedback::guardian_review_failures_attachment(&[session.thread_id()])
.expect("failed ephemeral review survives cleanup and subsequent allowed reviews");
let record: serde_json::Value = serde_json::from_slice(&feedback.buffer)?;
assert_eq!(
serde_json::json!({
"reviewer_thread_id": record["reviewer_thread_id"],
"status": record["status"],
"decision": record["decision"],
"command": serde_json::from_str::<serde_json::Value>(
record["action"].as_str().expect("reviewed action")
)?["command"],
}),
serde_json::json!({
"reviewer_thread_id": failed_ephemeral_request_body["client_metadata"]["thread_id"],
"status": "invalid_decision",
"decision": "not valid guardian json",
"command": ["git", "push"],
})
);
server.shutdown().await;
Ok(())
}))
})?;
})?;
match handle.join() {
Ok(result) => result,