diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 371b031ad8..2addb50e8e 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1276,6 +1276,10 @@ "additionalProperties": false, "description": "Bounds and optional sources for the Guardian v2 conversation transcript.", "properties": { + "include_images": { + "description": "Include recent screenshots from messages and configured tool outputs.", + "type": "boolean" + }, "max_message_entry_tokens": { "format": "uint", "maximum": 100000.0, diff --git a/codex-rs/ext/guardian-v2/src/config.rs b/codex-rs/ext/guardian-v2/src/config.rs index 506d2e48bb..0dc35a8758 100644 --- a/codex-rs/ext/guardian-v2/src/config.rs +++ b/codex-rs/ext/guardian-v2/src/config.rs @@ -144,6 +144,9 @@ impl GuardianV2Config { .unwrap_or_else(|| { vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs] }), + include_images: transcript_config + .and_then(|transcript| transcript.include_images) + .unwrap_or(false), max_message_entry_tokens, max_tool_entry_tokens, max_message_transcript_tokens, diff --git a/codex-rs/ext/guardian-v2/src/extension.rs b/codex-rs/ext/guardian-v2/src/extension.rs index a38a47528a..2834fbac4d 100644 --- a/codex-rs/ext/guardian-v2/src/extension.rs +++ b/codex-rs/ext/guardian-v2/src/extension.rs @@ -350,6 +350,9 @@ impl ToolLifecycleContributor for GuardianV2Extension { let transcript = guardian_config .transcript .build(conversation_history.items()); + let images = guardian_config + .transcript + .images(conversation_history.items()); drop(conversation_history); let planned_action = match action.render(guardian_config.max_action_tokens) { Ok(planned_action) => planned_action, @@ -418,6 +421,7 @@ impl ToolLifecycleContributor for GuardianV2Extension { .sample(LunaSamplingRequest { instructions, input: classification_input, + images, parent_compaction, parent_compaction_hash, output_schema: json!({ diff --git a/codex-rs/ext/guardian-v2/src/extension_tests.rs b/codex-rs/ext/guardian-v2/src/extension_tests.rs index b5963933d2..d04ca355e7 100644 --- a/codex-rs/ext/guardian-v2/src/extension_tests.rs +++ b/codex-rs/ext/guardian-v2/src/extension_tests.rs @@ -21,7 +21,9 @@ use codex_login::CodexAuth; use codex_model_provider_info::ModelProviderInfo; use codex_protocol::ResponseItemId; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageDetail; use codex_protocol::models::InternalChatMessageMetadataPassthrough; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::protocol::ReviewDecision; @@ -459,6 +461,76 @@ max_recent_non_user_entries = 8 Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn contributor_includes_configured_transcript_images() -> Result<()> { + skip_if_no_network!(Ok(())); + + let history = vec![ + ResponseItem::Message { + id: None, + role: "user".to_owned(), + content: vec![ + ContentItem::InputText { + text: "Review what is shown on screen.".to_owned(), + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,user-screenshot".to_owned(), + detail: Some(ImageDetail::High), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }, + ResponseItem::FunctionCallOutput { + id: None, + call_id: "previous-call".to_owned(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "Screenshot captured.".to_owned(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,tool-screenshot".to_owned(), + detail: Some(ImageDetail::Low), + }, + ]), + internal_chat_message_metadata_passthrough: None, + }, + ]; + let configuration = r#" +[features.guardianv2] +enabled = true + +[features.guardianv2.transcript] +include_images = true +"#; + let (request, _test, _registry) = sample_configured_conversation_history( + history, + r#"{"path":"README.md"}"#, + Some(TEST_GUARDIAN_POLICY), + configuration, + ) + .await?; + let content = request["input"][2]["content"] + .as_array() + .expect("Luna user content should be an array"); + + assert_eq!( + content[content.len() - 2..], + [ + json!({ + "type": "input_image", + "image_url": "data:image/png;base64,user-screenshot", + }), + json!({ + "type": "input_image", + "image_url": "data:image/png;base64,tool-screenshot", + }), + ] + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/ext/guardian-v2/src/sampler.rs b/codex-rs/ext/guardian-v2/src/sampler.rs index a02acccb70..358d0a9724 100644 --- a/codex-rs/ext/guardian-v2/src/sampler.rs +++ b/codex-rs/ext/guardian-v2/src/sampler.rs @@ -74,6 +74,8 @@ pub struct LunaSamplingRequest { pub instructions: String, /// Ordered untrusted input entries that the model should classify. pub input: Vec, + /// Optional bounded screenshots accompanying the transcript. + pub images: Vec, /// Opaque parent compaction to reuse only for compatible model configurations. pub parent_compaction: Option, /// Current parent model's encrypted-compaction compatibility hash. @@ -317,6 +319,12 @@ impl LunaSampler { .input .into_iter() .map(|text| ContentItem::InputText { text }) + .chain(request.images.into_iter().map(|mut image| { + if let ContentItem::InputImage { detail, .. } = &mut image { + *detail = None; + } + image + })) .collect(), phase: None, internal_chat_message_metadata_passthrough: None, diff --git a/codex-rs/ext/guardian-v2/src/sampler_tests.rs b/codex-rs/ext/guardian-v2/src/sampler_tests.rs index 5e27d62c9f..c4b55b6fef 100644 --- a/codex-rs/ext/guardian-v2/src/sampler_tests.rs +++ b/codex-rs/ext/guardian-v2/src/sampler_tests.rs @@ -75,6 +75,7 @@ fn sample_request(turn_id: &str) -> LunaSamplingRequest { LunaSamplingRequest { instructions: "Return a risk score.".to_owned(), input: vec!["The user requested a README summary.".to_owned()], + images: Vec::new(), parent_compaction: None, parent_compaction_hash: None, output_schema: json!({ @@ -169,6 +170,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ "The user requested a README summary.".to_owned(), "The assistant inspected README.md.".to_owned(), ], + images: Vec::new(), parent_compaction: None, parent_compaction_hash: None, output_schema: schema.clone(), @@ -192,6 +194,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ .sample(LunaSamplingRequest { instructions: "Return a risk score.".to_owned(), input: vec!["The user requested a source review.".to_owned()], + images: Vec::new(), parent_compaction: None, parent_compaction_hash: None, output_schema: schema, @@ -342,6 +345,7 @@ async fn sampler_returns_complete_json_before_terminal_response_events() -> Resu sampler.sample(LunaSamplingRequest { instructions: "Return a risk score.".to_owned(), input: vec!["The user requested a README summary.".to_owned()], + images: Vec::new(), parent_compaction: None, parent_compaction_hash: None, output_schema: json!({ diff --git a/codex-rs/ext/guardian-v2/src/transcript.rs b/codex-rs/ext/guardian-v2/src/transcript.rs index 4262e57ecd..10f9ac2f8b 100644 --- a/codex-rs/ext/guardian-v2/src/transcript.rs +++ b/codex-rs/ext/guardian-v2/src/transcript.rs @@ -1,8 +1,11 @@ use std::collections::HashMap; +use std::collections::VecDeque; use codex_extension_api::ResponseItem; pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageDetail; use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::plaintext_agent_message_content; @@ -13,6 +16,8 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000; pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000; pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000; pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40; +const MAX_TRANSCRIPT_IMAGES: usize = 4; +const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024; const MANUAL_APPROVAL_DEVELOPER_PREFIX: &str = "The user has manually approved a specific action that was previously `Rejected`."; @@ -32,6 +37,7 @@ struct TranscriptEntry { #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct TranscriptConfig { pub(crate) sources: Vec, + pub(crate) include_images: bool, pub(crate) max_message_entry_tokens: usize, pub(crate) max_tool_entry_tokens: usize, pub(crate) max_message_transcript_tokens: usize, @@ -43,6 +49,7 @@ impl Default for TranscriptConfig { fn default() -> Self { Self { sources: vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs], + include_images: false, max_message_entry_tokens: MAX_MESSAGE_ENTRY_TOKENS, max_tool_entry_tokens: MAX_TOOL_ENTRY_TOKENS, max_message_transcript_tokens: MAX_MESSAGE_TRANSCRIPT_TOKENS, @@ -53,6 +60,67 @@ impl Default for TranscriptConfig { } impl TranscriptConfig { + pub(crate) fn images<'a>( + &self, + items: impl IntoIterator, + ) -> Vec { + if !self.include_images { + return Vec::new(); + } + + let mut images = VecDeque::new(); + let mut image_bytes = 0usize; + let mut include_image = |image_url: &str, detail: Option| { + if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES { + return; + } + while images.len() >= MAX_TRANSCRIPT_IMAGES + || image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES + { + let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else { + break; + }; + image_bytes -= image_url.len(); + } + image_bytes += image_url.len(); + images.push_back(ContentItem::InputImage { + image_url: image_url.to_owned(), + detail, + }); + }; + + for item in items { + match item { + ResponseItem::Message { role, content, .. } + if matches!(role.as_str(), "user" | "assistant") => + { + for item in content { + if let ContentItem::InputImage { image_url, detail } = item { + include_image(image_url, *detail); + } + } + } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } + if self.sources.contains(&TranscriptSource::ToolOutputs) => + { + if let Some(content) = output.content_items() { + for item in content { + if let FunctionCallOutputContentItem::InputImage { image_url, detail } = + item + { + include_image(image_url, *detail); + } + } + } + } + _ => {} + } + } + + images.into_iter().collect() + } + pub(crate) fn build<'a>( &self, items: impl IntoIterator, diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs index 24a92ebb2c..2c2d385529 100644 --- a/codex-rs/features/src/feature_configs.rs +++ b/codex-rs/features/src/feature_configs.rs @@ -88,6 +88,9 @@ pub enum GuardianV2TranscriptSource { pub struct GuardianV2TranscriptConfigToml { #[serde(skip_serializing_if = "Option::is_none")] pub sources: Option>, + /// Include recent screenshots from messages and configured tool outputs. + #[serde(skip_serializing_if = "Option::is_none")] + pub include_images: Option, #[serde(skip_serializing_if = "Option::is_none")] #[schemars(range(min = 100, max = 100000))] pub max_message_entry_tokens: Option, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 054f5d7aff..4c5003899e 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -143,6 +143,7 @@ max_parent_compaction_tokens = 384 [guardianv2.transcript] sources = ["tool_outputs", "reasoning"] +include_images = true max_message_entry_tokens = 128 max_tool_entry_tokens = 128 max_message_transcript_tokens = 512 @@ -167,6 +168,7 @@ max_recent_non_user_entries = 12 crate::GuardianV2TranscriptSource::ToolOutputs, crate::GuardianV2TranscriptSource::Reasoning, ]), + include_images: Some(true), max_message_entry_tokens: Some(128), max_tool_entry_tokens: Some(128), max_message_transcript_tokens: Some(512),