Include Node REPL results in Guardian reviews (#38397)

## What changed

- Capture accepted, successful `node_repl` results from Code Mode and include them as bounded, untrusted evidence in Guardian review prompts.
- Enable the enhanced transcript through `guardian_enhanced_node_repl_transcripts` or when the selected model requires automatic Node REPL review.
- Keep this evidence out of the parent model history, exclude encrypted and failed results, and avoid resending admitted evidence when a Guardian session is reused.
- Clear retained evidence and invalidate review sessions when a thread is rolled back.

## Testing

- Add unit coverage for evidence ordering, escaping, truncation, and empty responses.
- Add an integration test covering feature-disabled, feature-enabled, and model-required behavior, including filtering and review-session reuse.

GitOrigin-RevId: edc3dca5d5b88d472a492f35531ec46889a89d72
This commit is contained in:
Ankush Gupta authored and copyberry committed 2026-08-13 15:50:48 +00:00
1 parent 779e9114ae
commit 053dda6b89
14 files changed
+757 -39

No files matched your search

+2 -2
View File
@@ -34,6 +34,7 @@ pub(crate) use approval_request::GuardianMcpAnnotations;
pub(crate) use approval_request::GuardianNetworkAccessTrigger;
#[cfg(test)]
pub(crate) use approval_request::guardian_approval_request_to_json;
pub(crate) use prompt::guardian_truncate_text;
pub(crate) use review::GuardianReviewOptions;
pub(crate) use review::guardian_timeout_message;
pub(crate) use review::is_guardian_reviewer_source;
@@ -61,6 +62,7 @@ const GUARDIAN_MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
const GUARDIAN_MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
const GUARDIAN_MAX_MESSAGE_ENTRY_TOKENS: usize = 2_000;
const GUARDIAN_MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
pub(crate) const GUARDIAN_MAX_NODE_REPL_TOOL_RESULT_TOKENS: usize = 6_000;
const GUARDIAN_MAX_ACTION_STRING_TOKENS: usize = 16_000;
const GUARDIAN_RECENT_ENTRY_LIMIT: usize = 40;
const TRUNCATION_TAG: &str = "truncated";
@@ -221,8 +223,6 @@ use prompt::collect_guardian_transcript_entries;
#[cfg(test)]
use prompt::guardian_output_schema;
#[cfg(test)]
use prompt::guardian_truncate_text;
#[cfg(test)]
use prompt::parse_guardian_assessment;
#[cfg(test)]
use prompt::render_guardian_transcript_entries;
+78 -15
View File
@@ -1,5 +1,6 @@
use std::collections::HashMap;
use codex_features::Feature;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::plaintext_agent_message_content;
use codex_protocol::protocol::GuardianRiskLevel;
@@ -9,6 +10,8 @@ use serde::Deserialize;
use serde_json::Value;
use crate::compact::content_items_to_text;
use crate::context::ContextualUserFragment;
use crate::context::NodeReplReviewEvidence;
use crate::event_mapping::is_contextual_user_message_content;
use crate::session::session::Session;
use crate::session::turn_context::TurnEnvironment;
@@ -22,6 +25,7 @@ use super::AUTO_REVIEW_DENIED_ACTION_APPROVAL_DEVELOPER_PREFIX;
use super::ApprovalRequestReasons;
use super::GUARDIAN_MAX_MESSAGE_ENTRY_TOKENS;
use super::GUARDIAN_MAX_MESSAGE_TRANSCRIPT_TOKENS;
use super::GUARDIAN_MAX_NODE_REPL_TOOL_RESULT_TOKENS;
use super::GUARDIAN_MAX_TOOL_ENTRY_TOKENS;
use super::GUARDIAN_MAX_TOOL_TRANSCRIPT_TOKENS;
use super::GUARDIAN_RECENT_ENTRY_LIMIT;
@@ -46,6 +50,7 @@ pub(crate) enum GuardianTranscriptEntryKind {
User,
Assistant,
Tool(String),
NodeReplToolResult(String),
}
impl GuardianTranscriptEntryKind {
@@ -54,7 +59,7 @@ impl GuardianTranscriptEntryKind {
Self::Developer => "developer",
Self::User => "user",
Self::Assistant => "assistant",
Self::Tool(role) => role.as_str(),
Self::Tool(role) | Self::NodeReplToolResult(role) => role.as_str(),
}
}
@@ -63,13 +68,14 @@ impl GuardianTranscriptEntryKind {
}
fn is_tool(&self) -> bool {
matches!(self, Self::Tool(_))
matches!(self, Self::Tool(_) | Self::NodeReplToolResult(_))
}
}
pub(crate) struct GuardianPromptItems {
pub(crate) items: Vec<UserInput>,
pub(crate) transcript_cursor: GuardianTranscriptCursor,
pub(crate) node_repl_evidence_sequence: u64,
pub(crate) reviewed_action_truncated: bool,
}
@@ -110,6 +116,7 @@ pub(crate) async fn build_guardian_prompt_items(
},
request,
mode,
/*reviewed_node_repl_evidence_sequence*/ 0,
)
.await
}
@@ -120,7 +127,21 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
reasons: ApprovalRequestReasons,
request: GuardianApprovalRequest,
mode: GuardianPromptMode,
reviewed_node_repl_evidence_sequence: u64,
) -> serde_json::Result<GuardianPromptItems> {
let node_repl_transcripts_enabled = parent_context.is_some_and(|context| {
context.turn().model_info.node_repl_auto_review_required
|| context
.turn()
.config
.features
.enabled(Feature::GuardianEnhancedNodeReplTranscripts)
});
let node_repl_result_token_limit = if node_repl_transcripts_enabled {
GUARDIAN_MAX_NODE_REPL_TOOL_RESULT_TOKENS
} else {
GUARDIAN_MAX_TOOL_ENTRY_TOKENS
};
let history = session.clone_history().await;
let transcript_entries = collect_guardian_transcript_entries(history.raw_items());
let transcript_cursor = GuardianTranscriptCursor {
@@ -146,7 +167,12 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
let (transcript_entries, omission_note, headings) = match prompt_shape {
GuardianPromptShape::Full => {
let (transcript_entries, omission_note) =
render_guardian_transcript_entries(transcript_entries.as_slice());
render_guardian_transcript_entries_with_offset(
transcript_entries.as_slice(),
/*entry_number_offset*/ 0,
"<no retained transcript entries>",
node_repl_result_token_limit,
);
(
transcript_entries,
omission_note,
@@ -166,6 +192,7 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
&transcript_entries[already_seen_entry_count..],
already_seen_entry_count,
"<no retained transcript delta entries>",
node_repl_result_token_limit,
);
(
transcript_entries,
@@ -206,6 +233,17 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
push_text(denied_reads_context);
push_text(">>> PARENT TURN PERMISSION CONTEXT END\n".to_string());
}
let mut node_repl_evidence_sequence = reviewed_node_repl_evidence_sequence;
if node_repl_transcripts_enabled
&& let Some(fragment) = session
.services
.thread_extension_data
.get::<NodeReplReviewEvidence>()
.and_then(|evidence| evidence.snapshot_since(reviewed_node_repl_evidence_sequence))
{
node_repl_evidence_sequence = fragment.sequence;
push_text(fragment.render());
}
match &request {
GuardianApprovalRequest::NetworkAccess { trigger, .. } => {
push_text(">>> APPROVAL REQUEST START\n".to_string());
@@ -250,6 +288,7 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
Ok(GuardianPromptItems {
items,
transcript_cursor,
node_repl_evidence_sequence,
reviewed_action_truncated: planned_action_json.truncated,
})
}
@@ -314,6 +353,7 @@ struct GuardianPromptHeadings {
///
/// Returns the rendered transcript plus an omission note when some entries were
/// skipped.
#[cfg(test)]
pub(crate) fn render_guardian_transcript_entries(
entries: &[GuardianTranscriptEntry],
) -> (Vec<String>, Option<String>) {
@@ -321,6 +361,7 @@ pub(crate) fn render_guardian_transcript_entries(
entries,
/*entry_number_offset*/ 0,
"<no retained transcript entries>",
GUARDIAN_MAX_TOOL_ENTRY_TOKENS,
)
}
@@ -328,6 +369,7 @@ fn render_guardian_transcript_entries_with_offset(
entries: &[GuardianTranscriptEntry],
entry_number_offset: usize,
empty_placeholder: &str,
node_repl_result_token_limit: usize,
) -> (Vec<String>, Option<String>) {
if entries.is_empty() {
return (vec![empty_placeholder.to_string()], None);
@@ -337,7 +379,12 @@ fn render_guardian_transcript_entries_with_offset(
.iter()
.enumerate()
.map(|(index, entry)| {
let token_cap = if entry.kind.is_tool() {
let token_cap = if matches!(
entry.kind,
GuardianTranscriptEntryKind::NodeReplToolResult(_)
) {
node_repl_result_token_limit
} else if entry.kind.is_tool() {
GUARDIAN_MAX_TOOL_ENTRY_TOKENS
} else {
GUARDIAN_MAX_MESSAGE_ENTRY_TOKENS
@@ -486,10 +533,12 @@ pub(crate) fn collect_guardian_transcript_entries<'a>(
ResponseItem::FunctionCall {
call_id,
name,
namespace,
arguments,
..
} => {
tool_names_by_call_id.insert(call_id.clone(), name.clone());
tool_names_by_call_id
.insert(call_id.as_str(), (name.as_str(), namespace.as_deref()));
(!arguments.trim().is_empty()).then(|| GuardianTranscriptEntry {
kind: GuardianTranscriptEntryKind::Tool(format!("tool {name} call")),
text: arguments.clone(),
@@ -498,10 +547,12 @@ pub(crate) fn collect_guardian_transcript_entries<'a>(
ResponseItem::CustomToolCall {
call_id,
name,
namespace,
input,
..
} => {
tool_names_by_call_id.insert(call_id.clone(), name.clone());
tool_names_by_call_id
.insert(call_id.as_str(), (name.as_str(), namespace.as_deref()));
(!input.trim().is_empty()).then(|| GuardianTranscriptEntry {
kind: GuardianTranscriptEntryKind::Tool(format!("tool {name} call")),
text: input.clone(),
@@ -519,15 +570,27 @@ pub(crate) fn collect_guardian_transcript_entries<'a>(
| ResponseItem::CustomToolCallOutput {
call_id, output, ..
} => output.body.to_text().and_then(|text| {
non_empty_entry(
GuardianTranscriptEntryKind::Tool(
tool_names_by_call_id.get(call_id).map_or_else(
|| "tool result".to_string(),
|name| format!("tool {name} result"),
),
),
text,
)
let kind = match tool_names_by_call_id.get(call_id.as_str()) {
Some((name, namespace))
if matches!(
namespace,
Some(
"mcp__node_repl" | "mcp__node_repl__" | "node_repl" | "node_repl__"
)
) || namespace.is_none()
&& (name.starts_with("mcp__node_repl__")
|| name.starts_with("node_repl__")) =>
{
GuardianTranscriptEntryKind::NodeReplToolResult(format!(
"tool {name} result"
))
}
Some((name, _)) => {
GuardianTranscriptEntryKind::Tool(format!("tool {name} result"))
}
None => GuardianTranscriptEntryKind::Tool("tool result".to_string()),
};
non_empty_entry(kind, text)
}),
_ => None,
};
+101 -21
View File
@@ -15,6 +15,7 @@ use codex_protocol::ThreadId;
use codex_protocol::config_types::AutoCompactTokenLimitScope;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
use codex_protocol::items::TurnItem;
use codex_protocol::models::BaseInstructionsProvenance;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
@@ -127,9 +128,16 @@ struct GuardianReviewSession {
struct GuardianReviewState {
prior_review_count: usize,
last_reviewed_transcript_cursor: Option<GuardianTranscriptCursor>,
last_admitted_node_repl_response_sequence: u64,
pending_node_repl_evidence_admission: Option<PendingNodeReplEvidenceAdmission>,
last_committed_fork_snapshot: Option<GuardianReviewForkSnapshot>,
}
struct PendingNodeReplEvidenceAdmission {
turn_id: String,
response_sequence: u64,
}
fn had_prior_review_context(prompt_mode: &GuardianPromptMode) -> bool {
matches!(prompt_mode, GuardianPromptMode::Delta { .. })
}
@@ -158,6 +166,7 @@ struct GuardianReviewForkSnapshot {
initial_history: InitialHistory,
prior_review_count: usize,
last_reviewed_transcript_cursor: Option<GuardianTranscriptCursor>,
last_admitted_node_repl_response_sequence: u64,
}
#[derive(Debug, Clone, PartialEq)]
@@ -301,10 +310,13 @@ impl GuardianReviewSession {
let mut state = self.state.lock().await;
let prior_review_count = state.prior_review_count;
let last_reviewed_transcript_cursor = state.last_reviewed_transcript_cursor;
let last_admitted_node_repl_response_sequence =
state.last_admitted_node_repl_response_sequence;
state.last_committed_fork_snapshot = Some(GuardianReviewForkSnapshot {
initial_history: InitialHistory::Forked(items),
prior_review_count,
last_reviewed_transcript_cursor,
last_admitted_node_repl_response_sequence,
});
}
Ok(Some(_)) => {}
@@ -314,6 +326,29 @@ impl GuardianReviewSession {
}
}
}
async fn admit_node_repl_evidence(&self, event: &Event) {
let EventMsg::ItemCompleted(completed) = &event.msg else {
return;
};
let TurnItem::UserMessage(_) = &completed.item else {
return;
};
let mut state = self.state.lock().await;
let Some(pending) = state.pending_node_repl_evidence_admission.as_ref() else {
return;
};
if completed.thread_id == self.session.thread_id()
&& event.id == pending.turn_id
&& completed.turn_id == pending.turn_id
{
state.last_admitted_node_repl_response_sequence = state
.last_admitted_node_repl_response_sequence
.max(pending.response_sequence);
state.pending_node_repl_evidence_admission = None;
}
}
}
impl EphemeralReviewCleanup {
@@ -418,6 +453,10 @@ impl GuardianReviewSessionManager {
pub(crate) async fn shutdown(&self) {
self.cancellation_token.cancel();
self.invalidate_for_node_repl_evidence().await;
}
pub(crate) async fn invalidate_for_node_repl_evidence(&self) {
let (review_session, ephemeral_reviews) = {
let mut state = self.state.lock().await;
(
@@ -425,11 +464,13 @@ impl GuardianReviewSessionManager {
std::mem::take(&mut state.ephemeral_reviews),
)
};
if let Some(review_session) = review_session {
review_session.shutdown().await;
}
for review_session in ephemeral_reviews {
review_session.shutdown().await;
for review_session in review_session.into_iter().chain(ephemeral_reviews) {
if self.cancellation_token.is_cancelled() {
review_session.shutdown().await;
} else {
review_session.cancel_token.cancel();
review_session.shutdown_in_background();
}
}
}
@@ -596,6 +637,8 @@ impl GuardianReviewSessionManager {
state: Mutex::new(GuardianReviewState {
prior_review_count: 0,
last_reviewed_transcript_cursor: None,
last_admitted_node_repl_response_sequence: 0,
pending_node_repl_evidence_admission: None,
last_committed_fork_snapshot: None,
}),
}));
@@ -621,6 +664,8 @@ impl GuardianReviewSessionManager {
state: Mutex::new(GuardianReviewState {
prior_review_count: 0,
last_reviewed_transcript_cursor: None,
last_admitted_node_repl_response_sequence: 0,
pending_node_repl_evidence_admission: None,
last_committed_fork_snapshot: None,
}),
}));
@@ -752,17 +797,24 @@ async fn spawn_guardian_review_session(
parent_compaction: Option<ResponseItem>,
fork_snapshot: Option<GuardianReviewForkSnapshot>,
) -> anyhow::Result<GuardianReviewSession> {
let (initial_history, prior_review_count, initial_transcript_cursor) = match fork_snapshot {
let (
initial_history,
prior_review_count,
initial_transcript_cursor,
last_admitted_node_repl_response_sequence,
) = match fork_snapshot {
Some(fork_snapshot) => (
Some(fork_snapshot.initial_history),
fork_snapshot.prior_review_count,
fork_snapshot.last_reviewed_transcript_cursor,
fork_snapshot.last_admitted_node_repl_response_sequence,
),
None => (
parent_compaction
.map(|item| InitialHistory::Forked(vec![RolloutItem::ResponseItem(item.into())])),
0,
None,
0,
),
};
let (session, io) = Box::pin(run_codex_thread_interactive(
@@ -789,6 +841,8 @@ async fn spawn_guardian_review_session(
state: Mutex::new(GuardianReviewState {
prior_review_count,
last_reviewed_transcript_cursor: initial_transcript_cursor,
last_admitted_node_repl_response_sequence,
pending_node_repl_evidence_admission: None,
last_committed_fork_snapshot: None,
}),
})
@@ -804,8 +858,9 @@ async fn run_review_on_session(
bool,
GuardianReviewAnalyticsResult,
) {
let (send_followup_reminder, prompt_mode) = {
let state = review_session.state.lock().await;
let (send_followup_reminder, prompt_mode, last_admitted_node_repl_response_sequence) = {
let mut state = review_session.state.lock().await;
state.pending_node_repl_evidence_admission = None;
let send_followup_reminder = state.prior_review_count == 1;
let prompt_mode = if state.prior_review_count == 0 {
@@ -816,7 +871,11 @@ async fn run_review_on_session(
GuardianPromptMode::Full
};
(send_followup_reminder, prompt_mode)
(
send_followup_reminder,
prompt_mode,
state.last_admitted_node_repl_response_sequence,
)
};
let model_info = params
.parent_session
@@ -865,6 +924,7 @@ async fn run_review_on_session(
params.reasons.clone(),
params.request.clone(),
prompt_mode,
last_admitted_node_repl_response_sequence,
)
.await
}),
@@ -886,6 +946,9 @@ async fn run_review_on_session(
};
let reviewed_action_truncated = prompt_items.reviewed_action_truncated;
let transcript_cursor = prompt_items.transcript_cursor;
let node_repl_evidence_admission = (prompt_items.node_repl_evidence_sequence
> last_admitted_node_repl_response_sequence)
.then_some(prompt_items.node_repl_evidence_sequence);
let token_usage_at_review_start = review_session
.session
.total_token_usage()
@@ -962,6 +1025,13 @@ async fn run_review_on_session(
}
Err(outcome) => return (outcome, false, analytics_result),
};
if let Some(response_sequence) = node_repl_evidence_admission {
let mut state = review_session.state.lock().await;
state.pending_node_repl_evidence_admission = Some(PendingNodeReplEvidenceAdmission {
turn_id: child_turn_id.clone(),
response_sequence,
});
}
analytics_result.reviewed_action_truncated = reviewed_action_truncated;
let outcome = wait_for_guardian_review(
@@ -1023,7 +1093,7 @@ async fn wait_for_guardian_review(
tokio::select! {
_ = &mut timeout => {
let keep_review_session = interrupt_and_drain_turn(
&review_session.io,
review_session,
expected_turn_id,
)
.await
@@ -1038,7 +1108,7 @@ async fn wait_for_guardian_review(
}
} => {
let keep_review_session = interrupt_and_drain_turn(
&review_session.io,
review_session,
expected_turn_id,
)
.await
@@ -1048,6 +1118,9 @@ async fn wait_for_guardian_review(
event = review_session.io.next_event() => {
match event {
Ok(event) if !event_matches_turn(&event, expected_turn_id) => {}
Ok(event) if matches!(&event.msg, EventMsg::ItemCompleted(_)) => {
review_session.admit_node_repl_evidence(&event).await;
}
Ok(event) => match event.msg {
EventMsg::TurnComplete(turn_complete) => {
analytics_result.time_to_first_token_ms = turn_complete
@@ -1229,18 +1302,23 @@ async fn run_before_review_deadline_with_cancel<T>(
result
}
async fn interrupt_and_drain_turn(io: &SessionIo, expected_turn_id: &str) -> anyhow::Result<()> {
let _ = io.submit(Op::Interrupt).await;
async fn interrupt_and_drain_turn(
review_session: &GuardianReviewSession,
expected_turn_id: &str,
) -> anyhow::Result<()> {
let _ = review_session.io.submit(Op::Interrupt).await;
tokio::time::timeout(GUARDIAN_INTERRUPT_DRAIN_TIMEOUT, async {
loop {
let event = io.next_event().await?;
if event_matches_turn(&event, expected_turn_id)
&& matches!(
event.msg,
EventMsg::TurnAborted(_) | EventMsg::TurnComplete(_)
)
{
let event = review_session.io.next_event().await?;
if !event_matches_turn(&event, expected_turn_id) {
continue;
}
review_session.admit_node_repl_evidence(&event).await;
if matches!(
event.msg,
EventMsg::TurnAborted(_) | EventMsg::TurnComplete(_)
) {
return Ok::<(), anyhow::Error>(());
}
}
@@ -1293,6 +1371,8 @@ mod tests {
state: Mutex::new(GuardianReviewState {
prior_review_count: 0,
last_reviewed_transcript_cursor: None,
last_admitted_node_repl_response_sequence: 0,
pending_node_repl_evidence_admission: None,
last_committed_fork_snapshot: None,
}),
},
@@ -2207,7 +2287,7 @@ mod tests {
.await
.expect("queue current turn abort");
interrupt_and_drain_turn(&review_session.io, "current-turn")
interrupt_and_drain_turn(&review_session, "current-turn")
.await
.expect("drain current turn");
+11 -1
View File
@@ -506,6 +506,7 @@ async fn build_guardian_prompt_prefers_retry_reason_over_approval_reason() -> an
justification: None,
},
GuardianPromptMode::Full,
/*reviewed_node_repl_evidence_sequence*/ 0,
)
.await?;
@@ -543,6 +544,7 @@ async fn build_guardian_prompt_truncates_oversized_approval_reason() -> anyhow::
justification: None,
},
GuardianPromptMode::Full,
/*reviewed_node_repl_evidence_sequence*/ 0,
)
.await?;
@@ -639,6 +641,7 @@ async fn build_guardian_prompt_includes_parent_turn_denied_reads() -> anyhow::Re
justification: Some("Need to inspect the secret file.".to_string()),
},
GuardianPromptMode::Full,
/*reviewed_node_repl_evidence_sequence*/ 0,
)
.await?;
@@ -945,7 +948,7 @@ fn collect_guardian_transcript_entries_keeps_manual_approval_developer_message()
#[test]
fn collect_guardian_transcript_entries_includes_recent_tool_calls_and_output() {
let items = vec![
let mut items = vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -1000,6 +1003,13 @@ fn collect_guardian_transcript_entries_includes_recent_tool_calls_and_output() {
text: "repo is public".to_string(),
}
);
if let ResponseItem::FunctionCall { namespace, .. } = &mut items[1] {
*namespace = Some("mcp__node_repl__".to_string());
}
assert!(matches!(
collect_guardian_transcript_entries(&items)[2].kind,
GuardianTranscriptEntryKind::NodeReplToolResult(_)
));
}
#[test]