github/codex
4
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-19 16:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Account for file images in context budgets and Guardian reviews (#46072)

Browse Source 
## Why

File images contributed zero to history and compaction token estimates and were dropped from Guardian context. This undercounted context usage and omitted image evidence from reviews.

## What changed

- Estimate file images using the fixed resized-image cost, or 10,000 patch tokens for `original` detail because file references do not expose dimensions.
- Charge file images against remote compaction and Guardian request budgets.
- Preserve file images from user messages and tool outputs through Guardian selection and delivery, and deduplicate review images by file ID.
- Apply transcript image count and byte limits to both reference types, counting file ID bytes rather than referenced file contents.

## Testing

Add coverage for file-image estimates, selection limits, delivery, and compaction retention. Extend Guardian tests to verify file-image evidence reaches reviews and uploaded original-detail images trigger reviewer-history compaction.

GitOrigin-RevId: ee024204dcbe16e277fea9347c6a4d7fc120e18d
This commit is contained in:
Krish Chainani
2026-09-17 01:38:13 +00:00
committed by copyberry
parent ce03f22af6
commit b974893c90
18 changed files with 481 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/compact_remote_v2_image_budget_tests.rs
+26 -15
View File
@@ -25,6 +25,15 @@ fn image() -> ContentItem {
}
}
fn file_image() -> ContentItem {
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(codex_protocol::models::ImageDetail::Original),
}
}
fn text(value: &str) -> ContentItem {
ContentItem::InputText {
text: value.to_string(),
@@ -45,21 +54,23 @@ fn trim(items: Vec<ResponseItem>, max_tokens: usize) -> Vec<ResponseItem> {
#[test]
fn image_only_boundary_is_atomic_and_does_not_backfill_older_messages() {
let newest = message(vec![text("new")]);
let items = vec![
message(vec![text("old")]),
message(vec![image()]),
newest.clone(),
];
let image_tokens = images::content_item_token_count(&image());
for (max_tokens, expected) in [
(
image_tokens + 1,
vec![message(vec![image()]), newest.clone()],
),
(image_tokens, vec![newest.clone()]),
(1, vec![newest]),
] {
assert_eq!(trim(items.clone(), max_tokens), expected);
for image in [image(), file_image()] {
let items = vec![
message(vec![text("old")]),
message(vec![image.clone()]),
newest.clone(),
];
let image_tokens = images::content_item_token_count(&image);
for (max_tokens, expected) in [
(
image_tokens + 1,
vec![message(vec![image.clone()]), newest.clone()],
),
(image_tokens, vec![newest.clone()]),
(1, vec![newest.clone()]),
] {
assert_eq!(trim(items.clone(), max_tokens), expected);
}
}
}
codex-rs/core/src/compact_remote_v2_images.rs
+4 -12
View File
@@ -1,10 +1,9 @@
use crate::context_manager::estimate_image_bytes;
use crate::context_manager::estimate_image_reference_bytes;
use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageReference;
use codex_protocol::models::is_image_close_tag_text;
use codex_protocol::models::is_image_open_tag_text;
use codex_protocol::models::is_local_image_open_tag_text;
@@ -18,17 +17,10 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
approx_token_count(text)
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => usize::try_from(approx_tokens_from_byte_count_i64(estimate_image_bytes(
image_url, *detail,
)))
ContentItem::InputImage { image, detail } => usize::try_from(
approx_tokens_from_byte_count_i64(estimate_image_reference_bytes(image, *detail)),
)
.unwrap_or(usize::MAX),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
ContentItem::InputAudio { .. } => 0,
}
}
codex-rs/core/src/context_manager/history.rs
+24 -19
View File
@@ -890,6 +890,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {
/// Returns the same coarse, model-visible token estimate used for full history estimates.
///
/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.
/// Original-detail file images use the maximum patch count.
pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {
let model_visible_bytes = estimate_response_item_model_visible_bytes(item);
approx_tokens_from_byte_count_i64(model_visible_bytes)
@@ -918,7 +919,6 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option
});
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
// TODO(kc) Account for file-backed image size after its token-cost contract is defined.
match item {
ResponseItem::Message { content, .. } => content
.iter()
@@ -926,14 +926,9 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text_bytes(text)
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
ContentItem::InputImage { image, detail } => {
estimate_image_reference_bytes(image, *detail)
}
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
@@ -1112,8 +1107,8 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
})
}
/// Shared image estimate, excluding message framing.
pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {
/// Inline image estimate, excluding the data URL prefix and message framing.
fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {
match detail {
Some(ImageDetail::Original) => {
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
@@ -1122,6 +1117,21 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>)
}
}
/// Image estimate for callers that only have the reference. Original-detail file images use the
/// maximum patch count because their dimensions are not available from the reference.
pub(crate) fn estimate_image_reference_bytes(
image: &ImageReference,
detail: Option<ImageDetail>,
) -> i64 {
match image {
ImageReference::Inline { image_url } => estimate_image_bytes(image_url, detail),
ImageReference::File { .. } if detail == Some(ImageDetail::Original) => {
i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap_or(i64::MAX)
}
ImageReference::File { .. } => RESIZED_IMAGE_BYTES_ESTIMATE,
}
}
fn estimate_audio_bytes(audio_url: &str) -> i64 {
i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
audio_url,
@@ -1136,14 +1146,9 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
.iter()
.map(|part| match part {
FunctionCallOutputContentItem::InputText { text } => text_bytes(text),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
} => 0,
FunctionCallOutputContentItem::InputImage { image, detail } => {
estimate_image_reference_bytes(image, *detail)
}
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
codex-rs/core/src/context_manager/history_tests.rs
+42 -3
View File
@@ -2577,8 +2577,39 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
assert!(estimated > text_only_estimated);
}
/// File images use the fixed estimate for normal detail and the maximum patch count for original.
#[test]
fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
fn file_images_use_detail_appropriate_estimates() {
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_high".to_string(),
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_original".to_string(),
},
detail: Some(ImageDetail::Original),
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = RESIZED_IMAGE_BYTES_ESTIMATE
.saturating_add(approx_bytes_for_tokens(/*tokens*/ 10_000) as i64);
assert_eq!(estimated, expected);
}
#[test]
fn function_call_output_estimates_inline_and_file_images() {
let payload = "B".repeat(50_000);
let image_url = format!("data:image/png;base64,{payload}");
let item = ResponseItem::FunctionCallOutput {
@@ -2594,14 +2625,22 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: "file_original".to_string(),
},
detail: Some(ImageDetail::Original),
},
]),
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected =
"call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let expected = "call-abc".len() as i64
+ "Screenshot captured".len() as i64
+ RESIZED_IMAGE_BYTES_ESTIMATE
+ approx_bytes_for_tokens(/*tokens*/ 10_000) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
codex-rs/core/src/context_manager/mod.rs
+1 -1
View File
@@ -4,6 +4,6 @@ pub(crate) mod updates;
pub(crate) use history::ContextManager;
pub(crate) use history::HistoryReplacement;
pub(crate) use history::estimate_image_bytes;
pub(crate) use history::estimate_image_reference_bytes;
pub(crate) use history::estimate_item_token_count;
pub(crate) use history::is_user_turn_boundary;
codex-rs/core/src/guardian/review_session.rs
+37 -23
View File
@@ -508,20 +508,26 @@ async fn run_review_on_session(
.any(|(_, cost)| cost.image_count > 0)
{
let reviewer_history = review_session.session.clone_history().await;
let reviewer_image_urls = reviewer_history
.raw_items()
.flat_map(|item| match item {
ResponseItem::Message { content, .. } => content.as_slice(),
_ => &[],
})
.filter_map(|item| match item {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} => Some(image_url.as_str()),
_ => None,
})
.collect::<HashSet<_>>();
let mut reviewer_image_urls = HashSet::new();
let mut reviewer_file_ids = HashSet::new();
for item in reviewer_history.raw_items() {
let ResponseItem::Message { content, .. } = item else {
continue;
};
for item in content {
let ContentItem::InputImage { image, .. } = item else {
continue;
};
match image {
ImageReference::Inline { image_url } => {
reviewer_image_urls.insert(image_url.as_str());
}
ImageReference::File { file_id } => {
reviewer_file_ids.insert(file_id.as_str());
}
}
}
}
let context_window = model_info.resolved_context_window().map(|supported| {
params
.spawn_config
@@ -542,7 +548,7 @@ async fn run_review_on_session(
} else {
ImagePreparationMode::DetailBased
};
prompt_items.context.retain_images(|image_url, detail| {
prompt_items.context.retain_images(|image, detail| {
*detail = match normalize_output_image_detail(&model_info, *detail) {
_ if mode == ImagePreparationMode::UnifiedBudget => {
Some(ImageDetail::Original)
@@ -550,15 +556,23 @@ async fn run_review_on_session(
Some(ImageDetail::Low) => Some(ImageDetail::High),
detail => detail,
};
let prepared_image_url = match resize_image(image_url, detail, mode) {
Ok(Some(prepared)) => Cow::Owned(prepared.into_data_url()),
Ok(None) => Cow::Borrowed(image_url),
Err(error) => {
warn!(%error, "failed to prepare guardian review image");
return false;
match image {
ImageReference::Inline { image_url } => {
let prepared_image_url = match resize_image(image_url, detail, mode)
{
Ok(Some(prepared)) => Cow::Owned(prepared.into_data_url()),
Ok(None) => Cow::Borrowed(image_url),
Err(error) => {
warn!(%error, "failed to prepare guardian review image");
return false;
}
};
!reviewer_image_urls.contains(prepared_image_url.as_str())
}
};
!reviewer_image_urls.contains(prepared_image_url.as_ref())
ImageReference::File { file_id } => {
!reviewer_file_ids.contains(file_id.as_str())
}
}
});
let prompt: ResponseItem =
ResponseInputItem::from(prompt_items.context.clone().into_user_inputs()?)
codex-rs/core/tests/suite/compact_remote.rs
+55 -10
View File
@@ -605,11 +605,14 @@ async fn amazon_bedrock_automatic_compaction_uses_v2_responses_endpoint() -> Res
Ok(())
}
#[test_case(None; "default_trims_images")]
#[test_case(Some(false); "disabled_preserves_images")]
#[test_case(None, "image_url"; "default_trims_images")]
#[test_case(Some(false), "image_url"; "disabled_preserves_images")]
#[test_case(None, "file_id"; "default_trims_file_images")]
#[test_case(Some(false), "file_id"; "disabled_preserves_file_images")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_compact_v2_charges_retained_images_to_token_budget(
image_budget_enabled: Option<bool>,
image_field: &str,
) -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -630,6 +633,14 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
// Each original-detail image costs 10,000 estimated patch tokens.
let image_inputs = (1..=8)
.map(|number| {
if image_field == "file_id" {
return Ok(UserInput::Image {
image: ImageReference::File {
file_id: format!("file_{number}"),
},
detail: Some(codex_protocol::models::ImageDetail::Original),
});
}
let image = image::ImageBuffer::from_pixel(
/*width*/ 3200,
/*height*/ 3200,
@@ -666,7 +677,24 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
.await?;
wait_for_turn_complete(codex).await;
let initial_request = initial_mock.single_request();
let prepared_images = initial_request.message_input_image_urls("user");
let prepared_images = initial_request
.inputs_of_type("message")
.into_iter()
.filter(|item| item["role"] == "user")
.flat_map(|item| {
item["content"]
.as_array()
.expect("message content is an array")
.clone()
})
.filter(|item| item["type"] == "input_image")
.map(|item| {
item[image_field]
.as_str()
.expect("image input has a string reference")
.to_owned()
})
.collect::<Vec<_>>();
assert_eq!(prepared_images.len(), 7);
for cycle in 1..=2 {
@@ -717,16 +745,33 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
};
let mut expected_images = prepared_images[dropped..].to_vec();
if cycle == 2 {
let UserInput::Image {
image: ImageReference::Inline { image_url },
..
} = &image_inputs[7]
else {
let UserInput::Image { image, .. } = &image_inputs[7] else {
unreachable!()
};
expected_images.push(image_url.clone());
expected_images.push(match image {
ImageReference::Inline { image_url } => image_url.clone(),
ImageReference::File { file_id } => file_id.clone(),
});
}
assert_eq!(follow_up.message_input_image_urls("user"), expected_images);
let retained_images = follow_up
.inputs_of_type("message")
.into_iter()
.filter(|item| item["role"] == "user")
.flat_map(|item| {
item["content"]
.as_array()
.expect("message content is an array")
.clone()
})
.filter(|item| item["type"] == "input_image")
.map(|item| {
item[image_field]
.as_str()
.expect("image input has a string reference")
.to_owned()
})
.collect::<Vec<_>>();
assert_eq!(retained_images, expected_images);
assert!(
follow_up
.message_input_texts("user")
codex-rs/core/tests/suite/guardian_context_budget.rs
+52 -8
View File
@@ -30,8 +30,11 @@ use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::sync::Arc;
use test_case::test_case;
use super::image_rollout::RecordingFileAttachmentStore;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(160_000, &[0, 0]; "complete_instructions_fit")]
#[test_case(28_000, &[0, 1]; "oversized_history_compacts_then_truncates")]
@@ -198,6 +201,7 @@ async fn review_preserves_user_instructions_until_request_budgeting(
enum ReviewerResponse {
Decision,
ToolContinuation,
FileImageContinuation,
UncompactableContinuation,
CompactionError,
NextReview,
@@ -206,6 +210,7 @@ enum ReviewerResponse {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(1, ReviewerResponse::Decision; "required_context_fails_closed")]
#[test_case(4_500, ReviewerResponse::ToolContinuation; "oversized_tool_continuation_compacts")]
#[test_case(4_500, ReviewerResponse::FileImageContinuation; "uploaded_original_image_history_compacts")]
#[test_case(4_500, ReviewerResponse::UncompactableContinuation; "ineffective_compaction_fails_closed")]
#[test_case(4_500, ReviewerResponse::CompactionError; "compaction_service_error_does_not_request_user_approval")]
#[test_case(6_000, ReviewerResponse::NextReview; "incoming_review_compacts_existing_history")]
@@ -231,6 +236,10 @@ async fn review_respects_complete_context_budget(
)
.with_model_info_override("gpt-5.6-luna", move |model| {
model.context_window = Some(window);
if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {
model.supports_image_detail_original = true;
model.use_responses_lite = false;
}
model
.model_messages
.as_mut()
@@ -268,12 +277,17 @@ async fn review_respects_complete_context_budget(
if matches!(
reviewer_response,
ReviewerResponse::ToolContinuation
| ReviewerResponse::FileImageContinuation
| ReviewerResponse::UncompactableContinuation
| ReviewerResponse::CompactionError
) {
builder = builder
.with_code_mode_host_program(codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?);
}
let image_store = Arc::new(RecordingFileAttachmentStore::default());
if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {
builder = builder.with_image_store(image_store.clone());
}
let test = builder.build_with_auto_env(&server).await?;
let command = json!({"cmd": "echo guardian-budget-test", "sandbox_permissions": "require_escalated", "justification": "Run the requested command."}).to_string();
let mut commentary =
@@ -321,12 +335,22 @@ async fn review_respects_complete_context_budget(
"exec",
"text('inspection-output'.repeat(600));",
),
// A tiny inline image fits before upload. Its opaque original-detail file
// reference must reserve 10k tokens in reviewer history and force compaction.
ReviewerResponse::FileImageContinuation => ev_custom_tool_call(
"reviewer-inspect",
"exec",
r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", "original");"#,
),
},
ev_completed("review"),
]),
);
}
if matches!(reviewer_response, ReviewerResponse::ToolContinuation) {
if matches!(
reviewer_response,
ReviewerResponse::ToolContinuation | ReviewerResponse::FileImageContinuation
) {
events.insert(
/*index*/ 2,
sse(vec![
@@ -424,18 +448,38 @@ async fn review_respects_complete_context_budget(
"expected required-evidence budget rejection: {output}"
);
} else {
let recovered = matches!(reviewer_response, ReviewerResponse::ToolContinuation);
let recovered = matches!(
reviewer_response,
ReviewerResponse::ToolContinuation | ReviewerResponse::FileImageContinuation
);
assert_eq!(requests.len(), if recovered { 4 } else { 3 });
assert_eq!(guardian_requests.len(), if recovered { 2 } else { 1 });
if recovered {
assert_eq!(compact_requests.len(), 1);
let compact = &compact_requests[0];
assert!(
compact
.body_json()
.to_string()
.contains("inspection-output")
);
if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {
assert_eq!(
image_store
.uploads
.lock()
.expect("image upload tracker lock is not poisoned")
.len(),
1
);
// The file reservation also protects the compaction request itself: an output
// larger than its window is replaced before the summary request is sent.
assert_eq!(
compact.custom_tool_call_output("reviewer-inspect")["output"],
"Output exceeded the available model context and was truncated"
);
} else {
assert!(
compact
.body_json()
.to_string()
.contains("inspection-output")
);
}
let recovered = guardian_requests[1].body_json();
assert!(
recovered["input"]
codex-rs/core/tests/suite/image_rollout.rs
+2 -2
View File
@@ -57,8 +57,8 @@ use std::time::Duration;
const UPLOADED_FILE_ID: &str = "file_uploaded_image";
#[derive(Default)]
struct RecordingFileAttachmentStore {
uploads: Mutex<Vec<UploadRequest>>,
pub(super) struct RecordingFileAttachmentStore {
pub(super) uploads: Mutex<Vec<UploadRequest>>,
}
impl AttachmentStore for RecordingFileAttachmentStore {
codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
+23 -1
View File
@@ -1677,6 +1677,8 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
let user_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC";
let tool_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGOQE+ECAACQAD304kFaAAAAAElFTkSuQmCC";
let user_file_id = "file_user";
let tool_file_id = "file_tool";
let history = vec![
ResponseItem::Message {
id: None,
@@ -1691,6 +1693,12 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
},
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
image: ImageReference::File {
file_id: user_file_id.to_owned(),
},
detail: Some(ImageDetail::High),
},
],
phase: None,
internal_chat_message_metadata_passthrough: None,
@@ -1719,6 +1727,12 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
},
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File {
file_id: tool_file_id.to_owned(),
},
detail: Some(ImageDetail::High),
},
]),
internal_chat_message_metadata_passthrough: None,
},
@@ -1740,16 +1754,24 @@ enabled = true
.expect("Luna user content should be an array");
assert_eq!(
content[content.len() - 2..],
content[content.len() - 4..],
[
json!({
"type": "input_image",
"image_url": user_image,
}),
json!({
"type": "input_image",
"file_id": user_file_id,
}),
json!({
"type": "input_image",
"image_url": tool_image,
}),
json!({
"type": "input_image",
"file_id": tool_file_id,
}),
]
);
codex-rs/guardian-context/src/budget.rs
+11 -10
View File
@@ -108,7 +108,9 @@ impl SectionCost {
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => {}
} => {
self.image_count = self.image_count.saturating_add(1);
}
ContentItem::InputAudio { audio_url } => {
// Guardian currently has no audio contributor. Count a future opaque
// payload conservatively until its consumer supplies modality costs.
@@ -214,17 +216,16 @@ pub(super) fn content_framing_tokens(item_count: usize) -> usize {
fn adjusted_tokens(mut bytes: usize, content: &[ContentItem]) -> usize {
for item in content {
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
} = item
{
let payload = ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));
if payload == usize::MAX {
return usize::MAX;
if let ContentItem::InputImage { image, .. } = item {
if let ImageReference::Inline { image_url } = image {
let payload =
ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));
if payload == usize::MAX {
return usize::MAX;
}
bytes = bytes.saturating_sub(payload.saturating_sub(2));
}
bytes = bytes
.saturating_sub(payload.saturating_sub(2))
.saturating_add(TruncationPolicy::Tokens(IMAGE_TOKEN_RESERVATION).byte_budget());
}
}
codex-rs/guardian-context/src/budget_tests.rs
+15 -1
View File
@@ -23,6 +23,12 @@ fn section_costs_keep_multimodal_payloads_separate() {
},
detail: None,
}),
Budgeted::required(ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_owned(),
},
detail: None,
}),
]),
},
SectionOutput {
@@ -44,7 +50,7 @@ fn section_costs_keep_multimodal_payloads_separate() {
SectionCost {
text_bytes: "évidence".len(),
image_bytes: "data:image/png;base64,AAAA".len(),
image_count: 1
image_count: 2
}
),
(
@@ -71,6 +77,14 @@ fn request_estimate_reserves_images_independently_of_encoded_size() {
let short = estimate_input_tokens(&image("AAAA"));
assert_eq!(short, estimate_input_tokens(&image(&"A".repeat(200_000))));
assert!(short >= IMAGE_TOKEN_RESERVATION);
let file = message(vec![ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_owned(),
},
detail: Some(codex_protocol::models::ImageDetail::Original),
}]);
assert!(estimate_input_tokens(&file) >= IMAGE_TOKEN_RESERVATION);
}
#[test]
codex-rs/guardian-context/src/composition.rs
+1 -15
View File
@@ -292,17 +292,7 @@ impl ComposedContext {
}));
continue;
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => UserInput::Image {
image: ImageReference::Inline { image_url },
detail,
},
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => continue,
ContentItem::InputImage { image, detail } => UserInput::Image { image, detail },
ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {
return Err(SectionError::UnsupportedDelivery {
section: section.id,
@@ -330,10 +320,6 @@ impl ComposedContext {
}
}));
}
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => {}
content => user_content.push(content),
}
}
codex-rs/guardian-context/src/composition_tests.rs
+34 -13
View File
@@ -1,4 +1,6 @@
use super::*;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[test]
@@ -56,12 +58,21 @@ fn delivery_preserves_arbitrary_message_boundaries_and_rejects_them_for_sync() {
}
#[test]
fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {
fn long_text_and_file_image_delivery_is_lossless_bounded_and_fully_budgeted() {
let text = "é🙂\"\n".repeat(/*n*/ 20_000);
let file_image = ImageReference::File {
file_id: "file_123".to_owned(),
};
let context = ComposedContext {
sections: vec![SectionOutput {
id: "planned_action",
delivery: text_content(vec![text.clone()]),
delivery: SectionDelivery::UserContent(vec![
Budgeted::required(ContentItem::InputText { text: text.clone() }),
Budgeted::required(ContentItem::InputImage {
image: file_image.clone(),
detail: Some(ImageDetail::High),
}),
]),
}],
truncations: Vec::new(),
};
@@ -70,7 +81,15 @@ fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {
let ResponseItem::Message { content, .. } = &messages[0] else {
panic!("user message")
};
let parts = content
let (image, text_content) = content.split_last().expect("message content");
assert_eq!(
image,
&ContentItem::InputImage {
image: file_image.clone(),
detail: Some(ImageDetail::High),
}
);
let parts = text_content
.iter()
.map(|item| match item {
ContentItem::InputText { text } => text.as_str(),
@@ -84,14 +103,16 @@ fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {
.all(|part| part.len() <= TruncationPolicy::Tokens(9_000).byte_budget())
);
assert!(estimated >= crate::estimate_input_tokens(&messages[0]));
assert_eq!(
context.into_user_inputs().unwrap(),
parts
.into_iter()
.map(|part| UserInput::Text {
text: part.to_owned(),
text_elements: Vec::new(),
})
.collect::<Vec<_>>()
);
let mut expected_inputs = parts
.into_iter()
.map(|part| UserInput::Text {
text: part.to_owned(),
text_elements: Vec::new(),
})
.collect::<Vec<_>>();
expected_inputs.push(UserInput::Image {
image: file_image,
detail: Some(ImageDetail::High),
});
assert_eq!(context.into_user_inputs().unwrap(), expected_inputs);
}
codex-rs/guardian-context/src/enforcement.rs
+5 -9
View File
@@ -88,17 +88,13 @@ pub enum HistoryTruncation {
impl ComposedContext {
/// Applies host image admission before aggregate selection, preserving section
/// identity and each retained item's selection policy.
pub fn retain_images(&mut self, mut admit: impl FnMut(&str, &mut Option<ImageDetail>) -> bool) {
pub fn retain_images(
&mut self,
mut admit: impl FnMut(&ImageReference, &mut Option<ImageDetail>) -> bool,
) {
for section in &mut self.sections {
retain_content(section, &mut self.truncations, |_, item| match item {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => admit(image_url, detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
} => false,
ContentItem::InputImage { image, detail } => admit(image, detail),
_ => true,
});
}
codex-rs/guardian-context/src/enforcement_tests.rs
+18 -6
View File
@@ -236,8 +236,14 @@ fn budget_reserves_existing_context_and_preserves_required_messages() {
}
#[test]
fn image_omission_preserves_text_and_later_eviction_policy() {
fn image_accounting_preserves_later_eviction_policy() {
let evidence = text(&"optional commentary ".repeat(/*n*/ 100));
let file_image = ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_owned(),
},
detail: None,
};
let mut context = ComposedContext {
sections: vec![SectionOutput {
id: "evidence",
@@ -251,6 +257,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
},
BudgetPriority::Image,
),
Budgeted::optional(file_image.clone(), BudgetPriority::Image),
Budgeted::optional(evidence.clone(), BudgetPriority::Commentary),
Budgeted::required(text("user restriction")),
]),
@@ -261,7 +268,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
.clone()
.enforce_budget(
RequestBudget {
max_input_tokens: 1_000,
max_input_tokens: content_tokens(&file_image).saturating_add(1_000),
existing_context_tokens: 0,
},
"evidence omitted".to_owned(),
@@ -271,12 +278,12 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
assert_eq!(
without_oversized_image.into_messages(),
vec![user_message(vec![
evidence.clone(),
file_image.clone(),
text("user restriction"),
text("evidence omitted")
])]
);
context.retain_images(|_, _| false);
context.retain_images(|image, _| matches!(image, ImageReference::File { .. }));
let available = context.estimated_tokens();
let retained = context
.clone()
@@ -291,12 +298,16 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
.unwrap();
assert_eq!(
retained.into_messages(),
vec![user_message(vec![evidence, text("user restriction")])]
vec![user_message(vec![
file_image.clone(),
evidence,
text("user restriction")
])]
);
let smaller = context
.enforce_budget(
RequestBudget {
max_input_tokens: 100,
max_input_tokens: content_tokens(&file_image).saturating_add(100),
existing_context_tokens: 0,
},
"evidence omitted".to_owned(),
@@ -306,6 +317,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
assert_eq!(
smaller.into_messages(),
vec![user_message(vec![
file_image,
text("user restriction"),
text("evidence omitted")
])]
codex-rs/guardian-context/src/images.rs
+28 -36
View File
@@ -1,5 +1,6 @@
//! Bounded image selection shared by Guardian consumers.
//! Keeps source order and evicts oldest images using the existing count/byte caps.
//! Byte caps cover inline URLs and file ID strings, not referenced file contents.
//! Consumers still choose source visibility and model-specific image detail.
use crate::ContextSection;
@@ -29,6 +30,7 @@ pub struct TranscriptImageInput<'a> {
#[derive(Clone, Default, PartialEq)]
pub struct TranscriptImages {
pub images: Vec<ContentItem>,
/// Omitted inline payload or file ID bytes, not the size of referenced files.
pub omitted_bytes: usize,
}
@@ -56,37 +58,37 @@ impl TranscriptImages {
}
let mut images = VecDeque::new();
let mut image_bytes = 0usize;
let mut image_reference_bytes = 0usize;
let mut omitted_bytes = 0usize;
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
let mut include_image = |image: &ImageReference, detail: Option<ImageDetail>| {
let reference_bytes = match image {
ImageReference::Inline { image_url } => image_url.len(),
ImageReference::File { file_id } => file_id.len(),
};
if reference_bytes > MAX_TRANSCRIPT_IMAGE_BYTES {
omitted_bytes = omitted_bytes.saturating_add(reference_bytes);
return;
}
while images.len() >= MAX_TRANSCRIPT_IMAGES
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
|| image_reference_bytes + reference_bytes > MAX_TRANSCRIPT_IMAGE_BYTES
{
let Some(ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
}) = images.pop_front()
else {
let Some(ContentItem::InputImage { image, .. }) = images.pop_front() else {
break;
};
image_bytes -= image_url.len();
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
let evicted_bytes = match image {
ImageReference::Inline { image_url } => image_url.len(),
ImageReference::File { file_id } => file_id.len(),
};
image_reference_bytes = image_reference_bytes.saturating_sub(evicted_bytes);
omitted_bytes = omitted_bytes.saturating_add(evicted_bytes);
}
image_bytes += image_url.len();
image_reference_bytes = image_reference_bytes.saturating_add(reference_bytes);
images.push_back(ContentItem::InputImage {
image: ImageReference::Inline {
image_url: image_url.to_owned(),
},
image: image.clone(),
detail,
});
};
// TODO(kc) Include file-backed images once Guardian can admit them without resolving or
// applying byte-based checks in Core.
for item in items {
match item {
ResponseItem::Message { role, content, .. }
@@ -94,15 +96,10 @@ impl TranscriptImages {
{
for item in content {
match item {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => include_image(image_url, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
ContentItem::InputImage { image, detail } => {
include_image(image, *detail)
}
| ContentItem::InputText { .. }
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => {}
}
@@ -115,15 +112,10 @@ impl TranscriptImages {
if let Some(content) = output.content_items() {
for item in content {
match item {
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} => include_image(image_url, *detail),
FunctionCallOutputContentItem::InputImage {
image: ImageReference::File { .. },
..
FunctionCallOutputContentItem::InputImage { image, detail } => {
include_image(image, *detail)
}
| FunctionCallOutputContentItem::InputText { .. }
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => {}
}
@@ -137,9 +129,9 @@ impl TranscriptImages {
for image in input.node_repl_images {
match image {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
image: image @ ImageReference::Inline { .. },
detail,
} => include_image(image_url, *detail),
} => include_image(image, *detail),
ContentItem::InputImage {
image: ImageReference::File { .. },
..
codex-rs/guardian-context/src/images_tests.rs
+103 -15
View File
@@ -1,6 +1,11 @@
use super::MAX_TRANSCRIPT_IMAGE_BYTES;
use super::TranscriptImageInput;
use super::TranscriptImages;
use crate::CollectedContext;
use crate::ContextPresentation;
use crate::ContextSection;
use crate::RenderedTranscript;
use crate::composition::user_message;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
@@ -99,20 +104,26 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {
);
}
/// File-backed images stay out of Guardian without consuming its inline-image capacity.
/// File-backed history images share count and reference-byte limits with inline images.
#[test]
fn file_images_are_omitted_without_displacing_inline_images() {
fn file_images_are_selected_from_history_sources() {
let file_image = ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_123".to_string(),
},
detail: Some(ImageDetail::High),
};
let tool_file_image = ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_tool".to_string(),
},
detail: Some(ImageDetail::High),
};
let history = [
ResponseItem::Message {
id: None,
role: "user".into(),
content: vec![image("first"), file_image.clone(), image("second")],
content: vec![image("first"), file_image.clone()],
phase: None,
internal_chat_message_metadata_passthrough: None,
},
@@ -138,16 +149,7 @@ fn file_images_are_omitted_without_displacing_inline_images() {
internal_chat_message_metadata_passthrough: None,
},
];
let repl = [
file_image,
image("fourth"),
ContentItem::InputImage {
image: ImageReference::File {
file_id: "file_repl".to_string(),
},
detail: Some(ImageDetail::High),
},
];
let repl = [image("fourth")];
assert_eq!(
TranscriptImages::collect(
@@ -159,8 +161,94 @@ fn file_images_are_omitted_without_displacing_inline_images() {
}
),
TranscriptImages {
images: ["first", "second", "third", "fourth"].map(image).to_vec(),
omitted_bytes: 0,
images: vec![file_image, tool_file_image, image("third"), image("fourth")],
omitted_bytes: "first".len(),
}
);
// Count-only eviction must remain observable even when no inline bytes were omitted.
let file_images =
["file_1", "file_2", "file_3", "file_4", "file_5"].map(|file_id| ContentItem::InputImage {
image: ImageReference::File {
file_id: file_id.to_owned(),
},
detail: Some(ImageDetail::High),
});
let history = [ResponseItem::Message {
id: None,
role: "user".into(),
content: file_images.to_vec(),
phase: None,
internal_chat_message_metadata_passthrough: None,
}];
let selected = TranscriptImages::collect(
&history,
TranscriptImageInput {
enabled: true,
include_tool_outputs: true,
node_repl_images: &[],
},
);
assert_eq!(
selected,
TranscriptImages {
images: file_images[1..].to_vec(),
omitted_bytes: "file_1".len(),
}
);
let context = CollectedContext {
sections: vec![ContextSection::TranscriptImages(selected)],
}
.compose(
ContextPresentation::Async,
RenderedTranscript {
items: Vec::new(),
omission_note: None,
truncations: Vec::new(),
},
)
.unwrap();
assert_eq!(
context
.truncations
.iter()
.map(|observation| (
observation.component,
observation.original_bytes,
observation.retained_bytes,
))
.collect::<Vec<_>>(),
vec![("transcript_image", "file_1".len(), 0)]
);
// Newer evidence evicts the full-budget ID; the oversized ID must not displace it.
let content = [
ImageReference::File {
file_id: "x".repeat(MAX_TRANSCRIPT_IMAGE_BYTES),
},
ImageReference::Inline {
image_url: "recent".into(),
},
ImageReference::File {
file_id: "y".repeat(MAX_TRANSCRIPT_IMAGE_BYTES + 1),
},
]
.map(|image| ContentItem::InputImage {
image,
detail: Some(ImageDetail::High),
});
assert_eq!(
TranscriptImages::collect(
&[user_message(content.to_vec())],
TranscriptImageInput {
enabled: true,
include_tool_outputs: true,
node_repl_images: &[],
},
),
TranscriptImages {
images: vec![image("recent")],
omitted_bytes: 2 * MAX_TRANSCRIPT_IMAGE_BYTES + 1,
}
);
}