mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
Include node_repl images in Guardian review evidence (#38454)
## What changed - Preserve valid image content from completed `node_repl` Code Mode responses alongside text evidence for Guardian reviews. - Enable multimodal evidence for models that require automatic `node_repl` review, or when both `guardian_enhanced_node_repl_transcripts` and `guardian_node_repl_transcript_images` are enabled. - Bound retained and rendered evidence, deduplicate images already present in review history, and fall back to text-only evidence when the reviewer lacks image support or prompt capacity. - Keep review evidence out of the parent turn input. ## Testing - Cover feature gating, mixed text and image ordering, truncation and eviction, unsupported reviewers, prompt pressure, image detail normalization, and reused review sessions. GitOrigin-RevId: a3e482402f7b3484401b231dadc662e6e3504fdf
This commit is contained in:
1 parent
1b4ea8b3be
commit
4d9f3021c8
14 files changed
+713
-87
No files matched your search
@@ -1,6 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::models::plaintext_agent_message_content;
|
||||
use codex_protocol::protocol::GuardianRiskLevel;
|
||||
@@ -10,8 +9,9 @@ use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::compact::content_items_to_text;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context::NodeReplReviewEvidence;
|
||||
use crate::context::NodeReplReviewEvidenceMode;
|
||||
use crate::context::node_repl_review_evidence_mode;
|
||||
use crate::event_mapping::is_contextual_user_message_content;
|
||||
use crate::session::session::Session;
|
||||
use crate::session::turn_context::TurnEnvironment;
|
||||
@@ -129,14 +129,10 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
|
||||
mode: GuardianPromptMode,
|
||||
reviewed_node_repl_evidence_sequence: u64,
|
||||
) -> serde_json::Result<GuardianPromptItems> {
|
||||
let node_repl_transcripts_enabled = parent_context.is_some_and(|context| {
|
||||
context.turn().model_info.node_repl_auto_review_required
|
||||
|| context
|
||||
.turn()
|
||||
.config
|
||||
.features
|
||||
.enabled(Feature::GuardianEnhancedNodeReplTranscripts)
|
||||
});
|
||||
let evidence_mode = parent_context
|
||||
.map(|context| node_repl_review_evidence_mode(context.turn()))
|
||||
.unwrap_or(NodeReplReviewEvidenceMode::Disabled);
|
||||
let node_repl_transcripts_enabled = evidence_mode != NodeReplReviewEvidenceMode::Disabled;
|
||||
let node_repl_result_token_limit = if node_repl_transcripts_enabled {
|
||||
GUARDIAN_MAX_NODE_REPL_TOOL_RESULT_TOKENS
|
||||
} else {
|
||||
@@ -242,8 +238,14 @@ pub(crate) async fn build_guardian_prompt_items_with_parent_turn(
|
||||
.and_then(|evidence| evidence.snapshot_since(reviewed_node_repl_evidence_sequence))
|
||||
{
|
||||
node_repl_evidence_sequence = fragment.sequence;
|
||||
push_text(fragment.render());
|
||||
items.extend(fragment.into_inputs(evidence_mode));
|
||||
}
|
||||
let mut push_text = |text: String| {
|
||||
items.push(UserInput::Text {
|
||||
text,
|
||||
text_elements: Vec::new(),
|
||||
});
|
||||
};
|
||||
match &request {
|
||||
GuardianApprovalRequest::NetworkAccess { trigger, .. } => {
|
||||
push_text(">>> APPROVAL REQUEST START\n".to_string());
|
||||
|
||||
@@ -699,18 +699,10 @@ pub(crate) fn spawn_approval_request_review(
|
||||
) -> oneshot::Receiver<ReviewDecision> {
|
||||
let context = context.into();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let runtime = session.services.runtime_handle.clone();
|
||||
let spawn_result = std::thread::Builder::new()
|
||||
.name("codex-approval-review".to_string())
|
||||
.spawn(move || {
|
||||
let Ok(runtime) = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
else {
|
||||
let _ = tx.send(ReviewDecision::denied(
|
||||
"automatic approval review could not complete",
|
||||
));
|
||||
return;
|
||||
};
|
||||
let decision = runtime.block_on(review_approval_request_with_cancel(
|
||||
&session,
|
||||
context,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
@@ -17,8 +18,12 @@ use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::BaseInstructionsProvenance;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::openai_models::ModelMessages;
|
||||
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
@@ -46,6 +51,10 @@ use crate::config::Permissions;
|
||||
use crate::context::ContextualUserFragment;
|
||||
use crate::context::GuardianFollowupReviewReminder;
|
||||
use crate::environment_selection::TurnEnvironmentSnapshot;
|
||||
use crate::image_preparation::ImagePreparationMode;
|
||||
use crate::image_preparation::ImageResizeNoticeMode;
|
||||
use crate::image_preparation::prepare_response_items;
|
||||
use crate::image_preparation::unified_image_budget_enabled;
|
||||
use crate::session::GitEnrichmentPolicy;
|
||||
use crate::session::SessionIo;
|
||||
use crate::session::session::Session;
|
||||
@@ -57,7 +66,9 @@ use codex_protocol::turn_input::TurnInputMode;
|
||||
use codex_protocol::turn_input::TurnInputRequest;
|
||||
use codex_protocol::turn_input::TurnInputSubmission;
|
||||
use codex_protocol::turn_input::TurnStartOptions;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_thread_store::PersistContext;
|
||||
use codex_tools::normalize_output_image_detail;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
|
||||
use super::ApprovalRequestReasons;
|
||||
@@ -73,6 +84,7 @@ use super::prompt::guardian_policy_prompt_with_config_and_template;
|
||||
use super::review::guardian_review_session_config;
|
||||
|
||||
const GUARDIAN_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const GUARDIAN_MAX_IMAGE_ITEM_TOKENS: i64 = 10_000;
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum GuardianReviewSessionOutcome {
|
||||
Completed(anyhow::Result<Option<String>>),
|
||||
@@ -906,7 +918,6 @@ async fn run_review_on_session(
|
||||
if send_followup_reminder {
|
||||
append_guardian_followup_reminder(review_session).await;
|
||||
}
|
||||
|
||||
let prompt_items = run_before_review_deadline(
|
||||
deadline,
|
||||
params.external_cancel.as_ref(),
|
||||
@@ -918,7 +929,7 @@ async fn run_review_on_session(
|
||||
.sync_session_approved_hosts_to(&review_session.session.services.network_approval)
|
||||
.await;
|
||||
|
||||
build_guardian_prompt_items_with_parent_turn(
|
||||
let mut prompt_items = build_guardian_prompt_items_with_parent_turn(
|
||||
params.parent_session.as_ref(),
|
||||
Some(¶ms.parent_context),
|
||||
params.reasons.clone(),
|
||||
@@ -926,7 +937,91 @@ async fn run_review_on_session(
|
||||
prompt_mode,
|
||||
last_admitted_node_repl_response_sequence,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
if prompt_items
|
||||
.items
|
||||
.iter()
|
||||
.any(|item| matches!(item, UserInput::Image { .. }))
|
||||
{
|
||||
let reviewer_history = review_session.session.clone_history().await;
|
||||
let reviewer_image_urls = reviewer_history
|
||||
.raw_items()
|
||||
.flat_map(|item| match item {
|
||||
ResponseItem::Message { content, .. } => content.as_slice(),
|
||||
_ => &[],
|
||||
})
|
||||
.filter_map(|item| match item {
|
||||
ContentItem::InputImage { image_url, .. } => Some(image_url.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
let context_window = model_info.resolved_context_window().map(|supported| {
|
||||
params
|
||||
.spawn_config
|
||||
.model_context_window
|
||||
.unwrap_or(supported)
|
||||
.min(supported)
|
||||
.saturating_mul(model_info.effective_context_window_percent.clamp(0, 100))
|
||||
/ 100
|
||||
});
|
||||
let admit_images = if let Some(context_window) = context_window.filter(|limit| {
|
||||
*limit > 0
|
||||
&& !model_info.used_fallback_model_metadata
|
||||
&& model_info.input_modalities.contains(&InputModality::Image)
|
||||
}) {
|
||||
let features = ¶ms.spawn_config.features;
|
||||
let mode = if unified_image_budget_enabled(features, &model_info) {
|
||||
ImagePreparationMode::UnifiedBudget
|
||||
} else {
|
||||
ImagePreparationMode::DetailBased
|
||||
};
|
||||
prompt_items.items.retain_mut(|item| {
|
||||
let UserInput::Image { detail, .. } = item else {
|
||||
return true;
|
||||
};
|
||||
*detail = match normalize_output_image_detail(&model_info, *detail) {
|
||||
_ if mode == ImagePreparationMode::UnifiedBudget => {
|
||||
Some(ImageDetail::Original)
|
||||
}
|
||||
Some(ImageDetail::Low) => Some(ImageDetail::High),
|
||||
detail => detail,
|
||||
};
|
||||
let mut prepared = vec![ResponseInputItem::from(vec![item.clone()]).into()];
|
||||
prepare_response_items(
|
||||
&mut prepared,
|
||||
mode,
|
||||
ImageResizeNoticeMode::Disabled,
|
||||
);
|
||||
let Some(ResponseItem::Message { content, .. }) = prepared.first() else {
|
||||
return false;
|
||||
};
|
||||
content.iter().any(|item| {
|
||||
matches!(item, ContentItem::InputImage { image_url, .. }
|
||||
if !reviewer_image_urls.contains(image_url.as_str()))
|
||||
})
|
||||
});
|
||||
let prompt: ResponseItem =
|
||||
ResponseInputItem::from(prompt_items.items.clone()).into();
|
||||
let prompt_tokens = crate::context_manager::estimate_item_token_count(&prompt);
|
||||
let base_instructions = review_session.session.get_base_instructions().await;
|
||||
let history_tokens = reviewer_history
|
||||
.estimate_token_count_with_base_instructions(&base_instructions)
|
||||
.unwrap_or(i64::MAX)
|
||||
.max(review_session.session.get_total_token_usage().await);
|
||||
prompt_tokens <= GUARDIAN_MAX_IMAGE_ITEM_TOKENS
|
||||
&& prompt_tokens.saturating_add(history_tokens) <= context_window
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if !admit_images {
|
||||
prompt_items
|
||||
.items
|
||||
.retain(|item| !matches!(item, UserInput::Image { .. }));
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, anyhow::Error>(prompt_items)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -938,7 +1033,7 @@ async fn run_review_on_session(
|
||||
Ok(prompt_items) => prompt_items,
|
||||
Err(err) => {
|
||||
return (
|
||||
GuardianReviewSessionOutcome::PromptBuildFailed(err.into()),
|
||||
GuardianReviewSessionOutcome::PromptBuildFailed(err),
|
||||
false,
|
||||
analytics_result,
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user