Move Guardian image selection into shared context sections (#43601)

## What changed

Move bounded transcript image collection into `codex-guardian-context` and expose it through an optional `TranscriptImages` section. Update the async scorer to consume this section, preserving source filtering, image detail, omission accounting, and the existing limits of four images and 8 MiB of image URL data.

## Testing

Add coverage for source order, tool-output filtering, disabled image collection, oldest-image eviction, oversized images, and omitted-byte accounting.

GitOrigin-RevId: 097799ea1be6d73b347088f959de1fd23f02ec2c
This commit is contained in:
felixxia-oai authored and copyberry committed 2026-09-07 21:46:23 +00:00
1 parent f5331dc237
commit b4373e53ab
10 files changed
+316 -99

No files matched your search

@@ -73,6 +73,8 @@ use super::wrapper_lag::WrapperLag;
use codex_core::context::GuardianReviewEvidenceFragment;
use codex_guardian_context::PreviousReviews;
use codex_guardian_context::ReviewEvidence;
use codex_guardian_context::TranscriptImageInput;
use codex_guardian_context::TranscriptImages;
use codex_guardian_context::render_review_evidence;
enum ClassificationOutcome {
@@ -479,9 +481,6 @@ impl GuardianV2Extension {
} else {
Vec::new()
};
let rendered_images = guardian_config
.transcript
.images(input.conversation_history.review_items(), node_repl_images);
// Capture root evidence before background metadata resolution or model I/O.
// Later root changes invalidate this sample through its captured authorization version.
let root_snapshot = if context_mode == GuardianContextMode::ThreadOwned {
@@ -581,6 +580,14 @@ impl GuardianV2Extension {
previous_reviews: Some(&reviews),
trusted_tool: trusted_tool_context.as_ref(),
trusted_skill_paths: &trusted_skill_paths,
images: Some(TranscriptImageInput {
enabled: guardian_config.transcript.include_images,
include_tool_outputs: guardian_config
.transcript
.sources
.contains(&super::transcript::TranscriptSource::ToolOutputs),
node_repl_images: &node_repl_images,
}),
})
});
let transcript = match transcript {
@@ -602,19 +609,16 @@ impl GuardianV2Extension {
}
};
drop(history);
drop(node_repl_images);
truncations.extend(transcript.truncations);
truncations.record(
"transcript_image",
rendered_images.omitted_bytes,
/*retained_bytes*/ 0,
);
let images = rendered_images.images;
let mut classification_input = Vec::new();
let mut trusted_review_evidence = None;
let mut trusted_tool_context = None;
let mut trusted_skills = None;
let mut rendered_images = TranscriptImages::default();
for section in transcript.sections {
match section {
ContextSection::TranscriptImages(images) => rendered_images = images,
ContextSection::TrustedSkills(skills) => trusted_skills = Some(skills),
ContextSection::TrustedTool(tool) => trusted_tool_context = Some(tool),
ContextSection::PreviousReviews(reviews) => {
@@ -636,6 +640,12 @@ impl GuardianV2Extension {
}
}
}
truncations.record(
"transcript_image",
rendered_images.omitted_bytes,
/*retained_bytes*/ 0,
);
let images = rendered_images.images;
let mut failure_reason = "invalid_output";
let mut classification_risk = None;
let mut classification_finished_at = None;
@@ -1,5 +1,3 @@
use std::collections::VecDeque;
use codex_extension_api::ConversationHistorySnapshot;
use codex_extension_api::ResponseItem;
pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;
@@ -18,13 +16,11 @@ use codex_guardian_context::SectionError;
use codex_guardian_context::SectionHistory;
use codex_guardian_context::SectionInput;
use codex_guardian_context::TranscriptEntryLimits;
use codex_guardian_context::TranscriptImageInput;
use codex_guardian_context::TranscriptRetentionConfig;
use codex_guardian_context::TrustedTool;
use codex_guardian_context::default_registry;
pub(crate) use codex_guardian_context::truncate_text as truncate_entry;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::protocol::TruncationPolicy;
use self::window::TranscriptWindow;
@@ -37,8 +33,6 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;
const MAX_TRANSCRIPT_IMAGES: usize = 4;
const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TranscriptEntryKind {
User,
@@ -65,6 +59,7 @@ pub(crate) struct ContextInput<'a> {
pub(crate) previous_reviews: Option<&'a PreviousReviews>,
pub(crate) trusted_tool: Option<&'a TrustedTool>,
pub(crate) trusted_skill_paths: &'a [String],
pub(crate) images: Option<TranscriptImageInput<'a>>,
}
pub(crate) struct RenderedContext {
@@ -72,11 +67,6 @@ pub(crate) struct RenderedContext {
pub(crate) truncations: Vec<TruncationObservation>,
}
pub(crate) struct RenderedImages {
pub(crate) images: Vec<ContentItem>,
pub(crate) omitted_bytes: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct TranscriptConfig {
pub(crate) sources: Vec<TranscriptSource>,
@@ -103,84 +93,6 @@ impl Default for TranscriptConfig {
}
impl TranscriptConfig {
pub(crate) fn images<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
node_repl_images: impl IntoIterator<Item = ContentItem>,
) -> RenderedImages {
if !self.include_images {
return RenderedImages {
images: Vec::new(),
omitted_bytes: 0,
};
}
let mut images = VecDeque::new();
let mut image_bytes = 0usize;
let mut omitted_bytes = 0usize;
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
return;
}
while images.len() >= MAX_TRANSCRIPT_IMAGES
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
{
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
break;
};
image_bytes -= image_url.len();
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
}
image_bytes += image_url.len();
images.push_back(ContentItem::InputImage {
image_url: image_url.to_owned(),
detail,
});
};
for item in items {
match item {
ResponseItem::Message { role, content, .. }
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
if let ContentItem::InputImage { image_url, detail } = item {
include_image(image_url, *detail);
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. }
if self.sources.contains(&TranscriptSource::ToolOutputs) =>
{
if let Some(content) = output.content_items() {
for item in content {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
item
{
include_image(image_url, *detail);
}
}
}
}
_ => {}
}
}
if self.sources.contains(&TranscriptSource::ToolOutputs) {
for image in node_repl_images {
if let ContentItem::InputImage { image_url, detail } = image {
include_image(&image_url, detail);
}
}
}
RenderedImages {
images: images.into_iter().collect(),
omitted_bytes,
}
}
pub(crate) fn build_context(
&self,
input: ContextInput<'_>,
@@ -194,6 +106,7 @@ impl TranscriptConfig {
previous_reviews,
trusted_tool,
trusted_skill_paths,
images,
} = input;
let history = SnapshotHistory(history);
let retention = TranscriptRetentionConfig {
@@ -224,6 +137,7 @@ impl TranscriptConfig {
previous_reviews,
trusted_tool,
trusted_skill_paths,
images,
})?;
let mut truncations = Vec::new();
let sections = context
@@ -246,6 +160,9 @@ impl TranscriptConfig {
ContextSection::PermissionContext { items } => {
ContextSection::PermissionContext { items }
}
ContextSection::TranscriptImages(images) => {
ContextSection::TranscriptImages(images)
}
ContextSection::TrustedSkills(skills) => ContextSection::TrustedSkills(skills),
ContextSection::TrustedTool(tool) => ContextSection::TrustedTool(tool),
ContextSection::PreviousReviews(reviews) => {
@@ -110,6 +110,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -136,6 +137,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("compose authorization and transcript");
assert_eq!(
@@ -168,6 +170,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -195,6 +198,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -247,6 +251,7 @@ fn transcript_truncates_oversized_entries_without_splitting_characters() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript");
let transcript = rendered.transcript_entries();
@@ -314,6 +319,7 @@ fn transcript_preserves_first_and_latest_user_messages_and_recent_history() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -379,6 +385,7 @@ fn transcript_preserves_user_restrictions_before_final_assistant_messages() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -418,6 +425,7 @@ fn transcript_preserves_recent_tool_evidence_when_protected_messages_fill_entry_
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript");
@@ -483,6 +491,7 @@ fn transcript_reserves_five_recent_tool_entries_from_protected_messages() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -538,6 +547,7 @@ fn rejected_commentary_does_not_evict_retained_message_evidence() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -578,6 +588,7 @@ fn transcript_evicts_protected_messages_in_cacheable_chunks() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries()
@@ -648,6 +659,7 @@ fn transcript_preserves_latest_final_when_reserved_tools_fill_entry_window() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -720,6 +732,7 @@ fn transcript_does_not_protect_legacy_inter_agent_instructions() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -766,6 +779,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -827,6 +841,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -890,6 +905,7 @@ fn transcript_preserves_newest_manual_approval_when_message_budget_overflows() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -949,6 +965,7 @@ fn rejected_message_does_not_evict_retained_tool_entries() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -993,6 +1010,7 @@ fn transcript_evicts_non_user_entries_in_cacheable_chunks() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries()
@@ -1065,6 +1083,7 @@ fn transcript_truncates_tool_results_using_standard_budget() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -1119,6 +1138,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries(),
@@ -1147,6 +1167,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries(),
@@ -1193,6 +1214,7 @@ fn configured_reasoning_counts_against_message_budget() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -1261,6 +1283,7 @@ fn transcript_keeps_only_manual_approval_developer_messages() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -1347,6 +1370,7 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();
@@ -1415,6 +1439,7 @@ fn transcript_omits_encrypted_messages_arguments_and_tool_outputs() {
previous_reviews: None,
trusted_tool: None,
trusted_skill_paths: &[],
images: None,
})
.expect("collect transcript")
.transcript_entries();