mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Attach failed Guardian reviews to diagnostic reports (#41936)
## What changed - Retain bounded, process-local records for failed Guardian reviews, including the reviewed action, decision, status, model, instructions, and reviewer history. - Add recent records from the reported thread and its descendants to log-enabled report uploads as `auto-review-failures.jsonl`. - Preserve the action and decision while omitting oversized reviewer context, and exclude successful, ephemeral, and capture-disabled reviews. ## Testing - Cover denied and invalid decisions, cleaned-up ephemeral reviewers, descendant-thread selection, record count and byte limits, and oversized-context fallback. GitOrigin-RevId: 5f12f7605041c3c84e826e6bac1cf06b1518ab24
This commit is contained in:
1 parent
981d6b652b
commit
13d75cd1c3
12 files changed
+439
-13
No files matched your search
@@ -0,0 +1,119 @@
|
||||
//! Captures bounded failed-review evidence for opt-in feedback uploads.
|
||||
//! Records outlive replaced and short-lived reviewers without writing extra history files.
|
||||
|
||||
use super::GuardianAssessmentOutcome;
|
||||
use super::approval_request::format_guardian_action_pretty;
|
||||
use super::prompt::parse_guardian_assessment;
|
||||
use super::review_session::GuardianReviewSessionOutcome;
|
||||
use super::review_session::GuardianReviewSessionParams;
|
||||
use crate::session::session::Session;
|
||||
use codex_feedback::record_guardian_review_failure;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use serde::Serialize;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
|
||||
const MAX_RECORD_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ReviewFeedbackRecord<'a> {
|
||||
reviewed_thread_id: ThreadId,
|
||||
reviewed_turn_id: &'a str,
|
||||
reviewer_thread_id: ThreadId,
|
||||
model: &'a str,
|
||||
status: &'a str,
|
||||
decision: Option<&'a str>,
|
||||
action: &'a str,
|
||||
action_truncated: bool,
|
||||
instructions: Option<&'a str>,
|
||||
history: Vec<&'a ResponseItem>,
|
||||
context_omitted: bool,
|
||||
}
|
||||
|
||||
pub(super) async fn record_failed_review(
|
||||
reviewer: &Session,
|
||||
params: &GuardianReviewSessionParams,
|
||||
outcome: &GuardianReviewSessionOutcome,
|
||||
) {
|
||||
if !params.spawn_config.feedback_enabled || params.spawn_config.ephemeral {
|
||||
return;
|
||||
}
|
||||
let (status, decision) = match outcome {
|
||||
GuardianReviewSessionOutcome::Completed(Ok(Some(decision))) => {
|
||||
match parse_guardian_assessment(Some(decision)) {
|
||||
Ok(assessment) if assessment.outcome == GuardianAssessmentOutcome::Allow => {
|
||||
return;
|
||||
}
|
||||
Ok(_) => ("denied", Some(decision.as_str())),
|
||||
Err(_) => ("invalid_decision", Some(decision.as_str())),
|
||||
}
|
||||
}
|
||||
GuardianReviewSessionOutcome::Completed(Ok(None)) => ("missing_decision", None),
|
||||
GuardianReviewSessionOutcome::Completed(Err(_))
|
||||
| GuardianReviewSessionOutcome::PromptBuildFailed(_)
|
||||
| GuardianReviewSessionOutcome::SessionFailed { .. } => ("failed", None),
|
||||
GuardianReviewSessionOutcome::TimedOut => ("timed_out", None),
|
||||
GuardianReviewSessionOutcome::Aborted => ("aborted", None),
|
||||
};
|
||||
let Ok(action) = format_guardian_action_pretty(¶ms.request) else {
|
||||
tracing::warn!("Could not serialize Guardian feedback action");
|
||||
return;
|
||||
};
|
||||
let instructions = reviewer.get_prompt_base_instructions().await;
|
||||
let history = reviewer.clone_history().await;
|
||||
ReviewFeedbackRecord {
|
||||
reviewed_thread_id: params.parent_session.thread_id(),
|
||||
reviewed_turn_id: ¶ms.parent_context.turn().sub_id,
|
||||
reviewer_thread_id: reviewer.thread_id(),
|
||||
model: ¶ms.model,
|
||||
status,
|
||||
decision,
|
||||
action: &action.text,
|
||||
action_truncated: action.truncated,
|
||||
instructions: Some(&instructions.text),
|
||||
history: history.raw_items().collect(),
|
||||
context_omitted: false,
|
||||
}
|
||||
.store();
|
||||
}
|
||||
|
||||
impl ReviewFeedbackRecord<'_> {
|
||||
fn store(mut self) {
|
||||
let mut buffer = BoundedBuffer(Vec::new());
|
||||
if serde_json::to_writer(&mut buffer, &self).is_err() {
|
||||
// Preserve the action and decision even when optional reviewer history is too large.
|
||||
self.instructions = None;
|
||||
self.history.clear();
|
||||
self.context_omitted = true;
|
||||
buffer = BoundedBuffer(Vec::new());
|
||||
if serde_json::to_writer(&mut buffer, &self).is_err() {
|
||||
tracing::warn!("Guardian feedback action and decision exceed the record limit");
|
||||
return;
|
||||
}
|
||||
}
|
||||
record_guardian_review_failure(self.reviewed_thread_id, buffer.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct BoundedBuffer(Vec<u8>);
|
||||
|
||||
impl Write for BoundedBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.len() > MAX_RECORD_BYTES.saturating_sub(self.0.len()) {
|
||||
return Err(io::Error::other(
|
||||
"Guardian feedback record exceeds its size limit",
|
||||
));
|
||||
}
|
||||
self.0.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "feedback_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,45 @@
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn oversized_context_keeps_the_action_and_decision() -> anyhow::Result<()> {
|
||||
let thread_id = ThreadId::new();
|
||||
let reviewer_thread_id = ThreadId::new();
|
||||
let decision = r#"{"outcome":"deny","rationale":"Missing approval."}"#;
|
||||
let action = r#"{"command":"git push"}"#;
|
||||
ReviewFeedbackRecord {
|
||||
reviewed_thread_id: thread_id,
|
||||
reviewed_turn_id: "parent-turn",
|
||||
reviewer_thread_id,
|
||||
model: "review-model",
|
||||
status: "denied",
|
||||
decision: Some(decision),
|
||||
action,
|
||||
action_truncated: false,
|
||||
instructions: Some(&"x".repeat(MAX_RECORD_BYTES)),
|
||||
history: Vec::new(),
|
||||
context_omitted: false,
|
||||
}
|
||||
.store();
|
||||
let contents = codex_feedback::guardian_review_failures_attachment(&[thread_id])
|
||||
.expect("failed-review record")
|
||||
.buffer;
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&contents)?,
|
||||
json!({
|
||||
"reviewed_thread_id": thread_id,
|
||||
"reviewed_turn_id": "parent-turn",
|
||||
"reviewer_thread_id": reviewer_thread_id,
|
||||
"model": "review-model",
|
||||
"status": "denied",
|
||||
"decision": decision,
|
||||
"action": action,
|
||||
"action_truncated": false,
|
||||
"instructions": null,
|
||||
"history": [],
|
||||
"context_omitted": true,
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
//! 4. Apply the guardian's explicit allow/deny outcome.
|
||||
|
||||
mod approval_request;
|
||||
mod feedback;
|
||||
mod metrics;
|
||||
mod prompt;
|
||||
mod review;
|
||||
|
||||
@@ -79,6 +79,7 @@ use super::ApprovalRequestReasons;
|
||||
use super::GUARDIAN_REVIEWER_NAME;
|
||||
use super::GuardianApprovalRequest;
|
||||
use super::GuardianReviewContext;
|
||||
use super::feedback::record_failed_review;
|
||||
#[cfg(test)]
|
||||
use super::prompt::BUNDLED_GUARDIAN_POLICY;
|
||||
use super::prompt::BUNDLED_GUARDIAN_POLICY_TEMPLATE;
|
||||
@@ -652,6 +653,7 @@ impl GuardianReviewSessionManager {
|
||||
deadline,
|
||||
))
|
||||
.await;
|
||||
record_failed_review(&trunk.session, ¶ms, &outcome).await;
|
||||
if keep_review_session && matches!(outcome, GuardianReviewSessionOutcome::Completed(_)) {
|
||||
trunk.refresh_last_committed_fork_snapshot().await;
|
||||
}
|
||||
@@ -826,6 +828,7 @@ impl GuardianReviewSessionManager {
|
||||
deadline,
|
||||
))
|
||||
.await;
|
||||
record_failed_review(&review_session.session, ¶ms, &outcome).await;
|
||||
if let Some(review_session) = self.take_active_ephemeral(&review_session).await {
|
||||
cleanup.disarm();
|
||||
review_session.shutdown_in_background();
|
||||
|
||||
@@ -3349,15 +3349,14 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
|
||||
{
|
||||
const TEST_STACK_SIZE_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
let handle =
|
||||
std::thread::Builder::new()
|
||||
.name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string())
|
||||
.stack_size(TEST_STACK_SIZE_BYTES)
|
||||
.spawn(|| -> anyhow::Result<()> {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
runtime.block_on(Box::pin(async {
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string())
|
||||
.stack_size(TEST_STACK_SIZE_BYTES)
|
||||
.spawn(|| -> anyhow::Result<()> {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
runtime.block_on(Box::pin(async {
|
||||
let first_assessment = serde_json::json!({
|
||||
"risk_level": "low",
|
||||
"user_authorization": "high",
|
||||
@@ -3421,7 +3420,11 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
|
||||
])
|
||||
.await;
|
||||
|
||||
let (session, turn) = guardian_test_session_and_turn_with_base_url(server.uri()).await;
|
||||
let (mut session, turn) = guardian_test_session_and_turn_with_base_url(server.uri()).await;
|
||||
// Isolate feedback from other tests using the fixed parent session ID.
|
||||
Arc::get_mut(&mut session)
|
||||
.expect("session should be uniquely owned")
|
||||
.thread_id = ThreadId::new();
|
||||
turn.turn_metadata_state
|
||||
.set_parent_turn_id("upstream-parent-turn".to_string());
|
||||
turn.turn_metadata_state
|
||||
@@ -3616,11 +3619,30 @@ async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() ->
|
||||
.send(())
|
||||
.expect("second guardian review gate should still be open");
|
||||
assert_eq!(second_review.await?, ReviewDecision::Approved);
|
||||
let feedback = codex_feedback::guardian_review_failures_attachment(&[session.thread_id()])
|
||||
.expect("failed ephemeral review survives cleanup and subsequent allowed reviews");
|
||||
let record: serde_json::Value = serde_json::from_slice(&feedback.buffer)?;
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"reviewer_thread_id": record["reviewer_thread_id"],
|
||||
"status": record["status"],
|
||||
"decision": record["decision"],
|
||||
"command": serde_json::from_str::<serde_json::Value>(
|
||||
record["action"].as_str().expect("reviewed action")
|
||||
)?["command"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"reviewer_thread_id": failed_ephemeral_request_body["client_metadata"]["thread_id"],
|
||||
"status": "invalid_decision",
|
||||
"decision": "not valid guardian json",
|
||||
"command": ["git", "push"],
|
||||
})
|
||||
);
|
||||
server.shutdown().await;
|
||||
|
||||
Ok(())
|
||||
}))
|
||||
})?;
|
||||
})?;
|
||||
|
||||
match handle.join() {
|
||||
Ok(result) => result,
|
||||
|
||||
Reference in new issue
Block a user