mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
Move Guardian image selection into shared context sections (#43601)
## What changed Move bounded transcript image collection into `codex-guardian-context` and expose it through an optional `TranscriptImages` section. Update the async scorer to consume this section, preserving source filtering, image detail, omission accounting, and the existing limits of four images and 8 MiB of image URL data. ## Testing Add coverage for source order, tool-output filtering, disabled image collection, oldest-image eviction, oversized images, and omitted-byte accounting. GitOrigin-RevId: 097799ea1be6d73b347088f959de1fd23f02ec2c
This commit is contained in:
1 parent
f5331dc237
commit
b4373e53ab
10 files changed
+316
-99
No files matched your search
@@ -73,6 +73,8 @@ use super::wrapper_lag::WrapperLag;
|
||||
use codex_core::context::GuardianReviewEvidenceFragment;
|
||||
use codex_guardian_context::PreviousReviews;
|
||||
use codex_guardian_context::ReviewEvidence;
|
||||
use codex_guardian_context::TranscriptImageInput;
|
||||
use codex_guardian_context::TranscriptImages;
|
||||
use codex_guardian_context::render_review_evidence;
|
||||
|
||||
enum ClassificationOutcome {
|
||||
@@ -479,9 +481,6 @@ impl GuardianV2Extension {
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let rendered_images = guardian_config
|
||||
.transcript
|
||||
.images(input.conversation_history.review_items(), node_repl_images);
|
||||
// Capture root evidence before background metadata resolution or model I/O.
|
||||
// Later root changes invalidate this sample through its captured authorization version.
|
||||
let root_snapshot = if context_mode == GuardianContextMode::ThreadOwned {
|
||||
@@ -581,6 +580,14 @@ impl GuardianV2Extension {
|
||||
previous_reviews: Some(&reviews),
|
||||
trusted_tool: trusted_tool_context.as_ref(),
|
||||
trusted_skill_paths: &trusted_skill_paths,
|
||||
images: Some(TranscriptImageInput {
|
||||
enabled: guardian_config.transcript.include_images,
|
||||
include_tool_outputs: guardian_config
|
||||
.transcript
|
||||
.sources
|
||||
.contains(&super::transcript::TranscriptSource::ToolOutputs),
|
||||
node_repl_images: &node_repl_images,
|
||||
}),
|
||||
})
|
||||
});
|
||||
let transcript = match transcript {
|
||||
@@ -602,19 +609,16 @@ impl GuardianV2Extension {
|
||||
}
|
||||
};
|
||||
drop(history);
|
||||
drop(node_repl_images);
|
||||
truncations.extend(transcript.truncations);
|
||||
truncations.record(
|
||||
"transcript_image",
|
||||
rendered_images.omitted_bytes,
|
||||
/*retained_bytes*/ 0,
|
||||
);
|
||||
let images = rendered_images.images;
|
||||
let mut classification_input = Vec::new();
|
||||
let mut trusted_review_evidence = None;
|
||||
let mut trusted_tool_context = None;
|
||||
let mut trusted_skills = None;
|
||||
let mut rendered_images = TranscriptImages::default();
|
||||
for section in transcript.sections {
|
||||
match section {
|
||||
ContextSection::TranscriptImages(images) => rendered_images = images,
|
||||
ContextSection::TrustedSkills(skills) => trusted_skills = Some(skills),
|
||||
ContextSection::TrustedTool(tool) => trusted_tool_context = Some(tool),
|
||||
ContextSection::PreviousReviews(reviews) => {
|
||||
@@ -636,6 +640,12 @@ impl GuardianV2Extension {
|
||||
}
|
||||
}
|
||||
}
|
||||
truncations.record(
|
||||
"transcript_image",
|
||||
rendered_images.omitted_bytes,
|
||||
/*retained_bytes*/ 0,
|
||||
);
|
||||
let images = rendered_images.images;
|
||||
let mut failure_reason = "invalid_output";
|
||||
let mut classification_risk = None;
|
||||
let mut classification_finished_at = None;
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use codex_extension_api::ConversationHistorySnapshot;
|
||||
use codex_extension_api::ResponseItem;
|
||||
pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;
|
||||
@@ -18,13 +16,11 @@ use codex_guardian_context::SectionError;
|
||||
use codex_guardian_context::SectionHistory;
|
||||
use codex_guardian_context::SectionInput;
|
||||
use codex_guardian_context::TranscriptEntryLimits;
|
||||
use codex_guardian_context::TranscriptImageInput;
|
||||
use codex_guardian_context::TranscriptRetentionConfig;
|
||||
use codex_guardian_context::TrustedTool;
|
||||
use codex_guardian_context::default_registry;
|
||||
pub(crate) use codex_guardian_context::truncate_text as truncate_entry;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
|
||||
use self::window::TranscriptWindow;
|
||||
@@ -37,8 +33,6 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
|
||||
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
|
||||
pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
|
||||
pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;
|
||||
const MAX_TRANSCRIPT_IMAGES: usize = 4;
|
||||
const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TranscriptEntryKind {
|
||||
User,
|
||||
@@ -65,6 +59,7 @@ pub(crate) struct ContextInput<'a> {
|
||||
pub(crate) previous_reviews: Option<&'a PreviousReviews>,
|
||||
pub(crate) trusted_tool: Option<&'a TrustedTool>,
|
||||
pub(crate) trusted_skill_paths: &'a [String],
|
||||
pub(crate) images: Option<TranscriptImageInput<'a>>,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedContext {
|
||||
@@ -72,11 +67,6 @@ pub(crate) struct RenderedContext {
|
||||
pub(crate) truncations: Vec<TruncationObservation>,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedImages {
|
||||
pub(crate) images: Vec<ContentItem>,
|
||||
pub(crate) omitted_bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct TranscriptConfig {
|
||||
pub(crate) sources: Vec<TranscriptSource>,
|
||||
@@ -103,84 +93,6 @@ impl Default for TranscriptConfig {
|
||||
}
|
||||
|
||||
impl TranscriptConfig {
|
||||
pub(crate) fn images<'a>(
|
||||
&self,
|
||||
items: impl IntoIterator<Item = &'a ResponseItem>,
|
||||
node_repl_images: impl IntoIterator<Item = ContentItem>,
|
||||
) -> RenderedImages {
|
||||
if !self.include_images {
|
||||
return RenderedImages {
|
||||
images: Vec::new(),
|
||||
omitted_bytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let mut images = VecDeque::new();
|
||||
let mut image_bytes = 0usize;
|
||||
let mut omitted_bytes = 0usize;
|
||||
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
return;
|
||||
}
|
||||
while images.len() >= MAX_TRANSCRIPT_IMAGES
|
||||
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
|
||||
{
|
||||
let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
|
||||
break;
|
||||
};
|
||||
image_bytes -= image_url.len();
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
}
|
||||
image_bytes += image_url.len();
|
||||
images.push_back(ContentItem::InputImage {
|
||||
image_url: image_url.to_owned(),
|
||||
detail,
|
||||
});
|
||||
};
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Message { role, content, .. }
|
||||
if matches!(role.as_str(), "user" | "assistant") =>
|
||||
{
|
||||
for item in content {
|
||||
if let ContentItem::InputImage { image_url, detail } = item {
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. }
|
||||
if self.sources.contains(&TranscriptSource::ToolOutputs) =>
|
||||
{
|
||||
if let Some(content) = output.content_items() {
|
||||
for item in content {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
|
||||
item
|
||||
{
|
||||
include_image(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if self.sources.contains(&TranscriptSource::ToolOutputs) {
|
||||
for image in node_repl_images {
|
||||
if let ContentItem::InputImage { image_url, detail } = image {
|
||||
include_image(&image_url, detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RenderedImages {
|
||||
images: images.into_iter().collect(),
|
||||
omitted_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_context(
|
||||
&self,
|
||||
input: ContextInput<'_>,
|
||||
@@ -194,6 +106,7 @@ impl TranscriptConfig {
|
||||
previous_reviews,
|
||||
trusted_tool,
|
||||
trusted_skill_paths,
|
||||
images,
|
||||
} = input;
|
||||
let history = SnapshotHistory(history);
|
||||
let retention = TranscriptRetentionConfig {
|
||||
@@ -224,6 +137,7 @@ impl TranscriptConfig {
|
||||
previous_reviews,
|
||||
trusted_tool,
|
||||
trusted_skill_paths,
|
||||
images,
|
||||
})?;
|
||||
let mut truncations = Vec::new();
|
||||
let sections = context
|
||||
@@ -246,6 +160,9 @@ impl TranscriptConfig {
|
||||
ContextSection::PermissionContext { items } => {
|
||||
ContextSection::PermissionContext { items }
|
||||
}
|
||||
ContextSection::TranscriptImages(images) => {
|
||||
ContextSection::TranscriptImages(images)
|
||||
}
|
||||
ContextSection::TrustedSkills(skills) => ContextSection::TrustedSkills(skills),
|
||||
ContextSection::TrustedTool(tool) => ContextSection::TrustedTool(tool),
|
||||
ContextSection::PreviousReviews(reviews) => {
|
||||
|
||||
@@ -110,6 +110,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -136,6 +137,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("compose authorization and transcript");
|
||||
assert_eq!(
|
||||
@@ -168,6 +170,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -195,6 +198,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -247,6 +251,7 @@ fn transcript_truncates_oversized_entries_without_splitting_characters() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript");
|
||||
let transcript = rendered.transcript_entries();
|
||||
@@ -314,6 +319,7 @@ fn transcript_preserves_first_and_latest_user_messages_and_recent_history() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -379,6 +385,7 @@ fn transcript_preserves_user_restrictions_before_final_assistant_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -418,6 +425,7 @@ fn transcript_preserves_recent_tool_evidence_when_protected_messages_fill_entry_
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript");
|
||||
|
||||
@@ -483,6 +491,7 @@ fn transcript_reserves_five_recent_tool_entries_from_protected_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -538,6 +547,7 @@ fn rejected_commentary_does_not_evict_retained_message_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -578,6 +588,7 @@ fn transcript_evicts_protected_messages_in_cacheable_chunks() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries()
|
||||
@@ -648,6 +659,7 @@ fn transcript_preserves_latest_final_when_reserved_tools_fill_entry_window() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -720,6 +732,7 @@ fn transcript_does_not_protect_legacy_inter_agent_instructions() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -766,6 +779,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -827,6 +841,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -890,6 +905,7 @@ fn transcript_preserves_newest_manual_approval_when_message_budget_overflows() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -949,6 +965,7 @@ fn rejected_message_does_not_evict_retained_tool_entries() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -993,6 +1010,7 @@ fn transcript_evicts_non_user_entries_in_cacheable_chunks() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries()
|
||||
@@ -1065,6 +1083,7 @@ fn transcript_truncates_tool_results_using_standard_budget() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1119,6 +1138,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries(),
|
||||
@@ -1147,6 +1167,7 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries(),
|
||||
@@ -1193,6 +1214,7 @@ fn configured_reasoning_counts_against_message_budget() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1261,6 +1283,7 @@ fn transcript_keeps_only_manual_approval_developer_messages() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1347,6 +1370,7 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
@@ -1415,6 +1439,7 @@ fn transcript_omits_encrypted_messages_arguments_and_tool_outputs() {
|
||||
previous_reviews: None,
|
||||
trusted_tool: None,
|
||||
trusted_skill_paths: &[],
|
||||
images: None,
|
||||
})
|
||||
.expect("collect transcript")
|
||||
.transcript_entries();
|
||||
|
||||
Reference in new issue
Block a user