mirror of
https://github.com/openai/codex.git
synced 2026-11-05 16:29:31 +00:00
Budget retained images during remote compaction (#40280)
## Why Remote compaction's retained-message budget counted text but not images, so image-heavy history could retain more context than the budget represented. ## What changed - Add the opt-in `compaction_image_budget` feature to charge retained images using the existing image size estimate. - Keep images and their adjacent labels atomic when truncating a boundary message, while preserving existing text, audio, metadata, and client-authored developer-message behavior. - Stop backfilling older messages when an image at the truncation boundary does not fit. ## Testing - Add unit coverage for image boundaries, annotations, audio, text-only messages, and client-authored developer messages. - Add an integration test covering repeated remote compaction with the feature enabled, disabled, and left at its default. GitOrigin-RevId: 88868bbd09d2858939dbae3f8402e1f497b4e3c6
This commit is contained in:
1 parent
aec653daa9
commit
6677fd827d
10 files changed
+507
-28
No files matched your search
@@ -640,6 +640,9 @@
|
|||||||
"collaboration_modes": {
|
"collaboration_modes": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
|
"compaction_image_budget": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"computer_use": {
|
"computer_use": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
@@ -5610,6 +5613,9 @@
|
|||||||
"collaboration_modes": {
|
"collaboration_modes": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
|
"compaction_image_budget": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"computer_use": {
|
"computer_use": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -62,6 +62,15 @@ mod attempt;
|
|||||||
use attempt::RemoteCompactV2Attempt;
|
use attempt::RemoteCompactV2Attempt;
|
||||||
use attempt::run_remote_compact_v2_attempt;
|
use attempt::run_remote_compact_v2_attempt;
|
||||||
|
|
||||||
|
#[path = "compact_remote_v2_images.rs"]
|
||||||
|
mod images;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum RetainedImageBudget {
|
||||||
|
Disabled,
|
||||||
|
Enabled,
|
||||||
|
}
|
||||||
|
|
||||||
// Mirror the current /responses/compact retained-message default while the
|
// Mirror the current /responses/compact retained-message default while the
|
||||||
// server-side path remains the reference implementation.
|
// server-side path remains the reference implementation.
|
||||||
pub(crate) const RETAINED_MESSAGE_TOKEN_BUDGET: usize = 64_000;
|
pub(crate) const RETAINED_MESSAGE_TOKEN_BUDGET: usize = 64_000;
|
||||||
@@ -301,6 +310,11 @@ async fn run_remote_compact_task_inner_impl(
|
|||||||
prompt_input_metadata,
|
prompt_input_metadata,
|
||||||
compaction_output,
|
compaction_output,
|
||||||
sess.enabled(Feature::RetainClientDeveloperMessages),
|
sess.enabled(Feature::RetainClientDeveloperMessages),
|
||||||
|
if sess.enabled(Feature::CompactionImageBudget) {
|
||||||
|
RetainedImageBudget::Enabled
|
||||||
|
} else {
|
||||||
|
RetainedImageBudget::Disabled
|
||||||
|
},
|
||||||
);
|
);
|
||||||
analytics_details.retained_image_count = Some(retained_images);
|
analytics_details.retained_image_count = Some(retained_images);
|
||||||
let (new_window_number, new_window_ids) = sess.advance_auto_compact_window().await;
|
let (new_window_number, new_window_ids) = sess.advance_auto_compact_window().await;
|
||||||
@@ -463,6 +477,7 @@ fn build_v2_compacted_history(
|
|||||||
prompt_input_metadata: Vec<Option<CodexHarnessMetadata>>,
|
prompt_input_metadata: Vec<Option<CodexHarnessMetadata>>,
|
||||||
compaction_output: ResponseItem,
|
compaction_output: ResponseItem,
|
||||||
retain_client_developer_messages: bool,
|
retain_client_developer_messages: bool,
|
||||||
|
image_budget: RetainedImageBudget,
|
||||||
) -> (Vec<ResponseItemEnvelope>, usize) {
|
) -> (Vec<ResponseItemEnvelope>, usize) {
|
||||||
debug_assert_eq!(prompt_input.len(), prompt_input_metadata.len());
|
debug_assert_eq!(prompt_input.len(), prompt_input_metadata.len());
|
||||||
let prompt_input = prompt_input
|
let prompt_input = prompt_input
|
||||||
@@ -480,7 +495,7 @@ fn build_v2_compacted_history(
|
|||||||
.flat_map(HistoryItemGroup::into_items)
|
.flat_map(HistoryItemGroup::into_items)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut retained =
|
let mut retained =
|
||||||
truncate_retained_messages_for_remote_compaction(retained, RETAINED_MESSAGE_TOKEN_BUDGET);
|
truncate_retained_messages(retained, RETAINED_MESSAGE_TOKEN_BUDGET, image_budget);
|
||||||
let retained_image_count = retained
|
let retained_image_count = retained
|
||||||
.iter()
|
.iter()
|
||||||
.map(|envelope| retained_input_image_count(&envelope.item))
|
.map(|envelope| retained_input_image_count(&envelope.item))
|
||||||
@@ -558,6 +573,14 @@ fn retained_input_image_count(item: &ResponseItem) -> usize {
|
|||||||
pub(crate) fn truncate_retained_messages_for_remote_compaction(
|
pub(crate) fn truncate_retained_messages_for_remote_compaction(
|
||||||
items: Vec<ResponseItemEnvelope>,
|
items: Vec<ResponseItemEnvelope>,
|
||||||
max_tokens: usize,
|
max_tokens: usize,
|
||||||
|
) -> Vec<ResponseItemEnvelope> {
|
||||||
|
truncate_retained_messages(items, max_tokens, RetainedImageBudget::Disabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_retained_messages(
|
||||||
|
items: Vec<ResponseItemEnvelope>,
|
||||||
|
max_tokens: usize,
|
||||||
|
image_budget: RetainedImageBudget,
|
||||||
) -> Vec<ResponseItemEnvelope> {
|
) -> Vec<ResponseItemEnvelope> {
|
||||||
let mut remaining = max_tokens;
|
let mut remaining = max_tokens;
|
||||||
let mut truncated_reversed = Vec::with_capacity(items.len());
|
let mut truncated_reversed = Vec::with_capacity(items.len());
|
||||||
@@ -571,14 +594,22 @@ pub(crate) fn truncate_retained_messages_for_remote_compaction(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let client_developer = is_client_authored_developer_message(&group.source);
|
let client_developer = is_client_authored_developer_message(&group.source);
|
||||||
|
let charge_images = image_budget == RetainedImageBudget::Enabled && !client_developer;
|
||||||
let notice_tokens = group
|
let notice_tokens = group
|
||||||
.attached_notice
|
.attached_notice
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map_or(0, |notice| message_text_token_count(¬ice.item).max(1));
|
.map_or(0, |notice| message_text_token_count(¬ice.item).max(1));
|
||||||
|
// Client-authored developer messages already charge non-text content via
|
||||||
|
// the serialized estimate. Preserve their text-only boundary correction.
|
||||||
|
let content_tokens = if charge_images {
|
||||||
|
message_content_token_count(&group.source.item)
|
||||||
|
} else {
|
||||||
|
message_text_token_count(&group.source.item)
|
||||||
|
};
|
||||||
let source_tokens = if client_developer {
|
let source_tokens = if client_developer {
|
||||||
usize::try_from(estimate_item_token_count(&group.source.item)).unwrap_or(usize::MAX)
|
usize::try_from(estimate_item_token_count(&group.source.item)).unwrap_or(usize::MAX)
|
||||||
} else {
|
} else {
|
||||||
message_text_token_count(&group.source.item).max(1)
|
content_tokens.max(1)
|
||||||
};
|
};
|
||||||
let token_count = source_tokens.saturating_add(notice_tokens);
|
let token_count = source_tokens.saturating_add(notice_tokens);
|
||||||
if token_count <= remaining {
|
if token_count <= remaining {
|
||||||
@@ -589,23 +620,30 @@ pub(crate) fn truncate_retained_messages_for_remote_compaction(
|
|||||||
remaining = remaining.saturating_sub(token_count);
|
remaining = remaining.saturating_sub(token_count);
|
||||||
} else if remaining > notice_tokens {
|
} else if remaining > notice_tokens {
|
||||||
let available_tokens = remaining - notice_tokens;
|
let available_tokens = remaining - notice_tokens;
|
||||||
let text_budget = if client_developer {
|
let content_budget = if client_developer {
|
||||||
available_tokens.saturating_sub(
|
available_tokens.saturating_sub(source_tokens.saturating_sub(content_tokens))
|
||||||
source_tokens.saturating_sub(message_text_token_count(&group.source.item)),
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
available_tokens
|
available_tokens
|
||||||
};
|
};
|
||||||
let Some(mut truncated_item) =
|
let image_count = retained_input_image_count(&group.source.item);
|
||||||
truncate_message_text_to_token_budget(group.source, text_budget)
|
if charge_images && image_count > 0 {
|
||||||
else {
|
// An oversized image can leave no boundary content. Do not backfill
|
||||||
|
// the remaining budget with older messages in that case.
|
||||||
|
remaining = 0;
|
||||||
|
}
|
||||||
|
let truncated_item = if charge_images && image_count > 0 {
|
||||||
|
images::truncate_message_to_token_budget(group.source, content_budget)
|
||||||
|
} else {
|
||||||
|
truncate_message_text_to_token_budget(group.source, content_budget)
|
||||||
|
};
|
||||||
|
let Some(mut truncated_item) = truncated_item else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
if client_developer {
|
if client_developer {
|
||||||
let item_tokens = usize::try_from(estimate_item_token_count(&truncated_item.item))
|
let item_tokens = usize::try_from(estimate_item_token_count(&truncated_item.item))
|
||||||
.unwrap_or(usize::MAX);
|
.unwrap_or(usize::MAX);
|
||||||
if item_tokens > available_tokens {
|
if item_tokens > available_tokens {
|
||||||
let adjusted_budget = text_budget
|
let adjusted_budget = content_budget
|
||||||
.saturating_sub(item_tokens - available_tokens)
|
.saturating_sub(item_tokens - available_tokens)
|
||||||
.saturating_sub(1);
|
.saturating_sub(1);
|
||||||
let Some(adjusted) =
|
let Some(adjusted) =
|
||||||
@@ -627,12 +665,22 @@ pub(crate) fn truncate_retained_messages_for_remote_compaction(
|
|||||||
}
|
}
|
||||||
truncated_reversed.push(truncated_item);
|
truncated_reversed.push(truncated_item);
|
||||||
remaining = 0;
|
remaining = 0;
|
||||||
|
} else if charge_images && retained_input_image_count(&group.source.item) > 0 {
|
||||||
|
remaining = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
truncated_reversed.reverse();
|
truncated_reversed.reverse();
|
||||||
truncated_reversed
|
truncated_reversed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn message_content_token_count(item: &ResponseItem) -> usize {
|
||||||
|
let ResponseItem::Message { content, .. } = item else {
|
||||||
|
return usize::try_from(estimate_item_token_count(item)).unwrap_or(usize::MAX);
|
||||||
|
};
|
||||||
|
|
||||||
|
content.iter().map(images::content_item_token_count).sum()
|
||||||
|
}
|
||||||
|
|
||||||
fn message_text_token_count(item: &ResponseItem) -> usize {
|
fn message_text_token_count(item: &ResponseItem) -> usize {
|
||||||
let ResponseItem::Message { content, .. } = item else {
|
let ResponseItem::Message { content, .. } = item else {
|
||||||
return usize::try_from(estimate_item_token_count(item)).unwrap_or(usize::MAX);
|
return usize::try_from(estimate_item_token_count(item)).unwrap_or(usize::MAX);
|
||||||
@@ -718,7 +766,11 @@ mod tests {
|
|||||||
) -> (Vec<ResponseItemEnvelope>, usize) {
|
) -> (Vec<ResponseItemEnvelope>, usize) {
|
||||||
let metadata = vec![None; input.len()];
|
let metadata = vec![None; input.len()];
|
||||||
build_v2_compacted_history(
|
build_v2_compacted_history(
|
||||||
input, metadata, output, /*retain_client_developer_messages*/ false,
|
input,
|
||||||
|
metadata,
|
||||||
|
output,
|
||||||
|
/*retain_client_developer_messages*/ false,
|
||||||
|
RetainedImageBudget::Disabled,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -829,6 +881,7 @@ mod tests {
|
|||||||
],
|
],
|
||||||
output.clone(),
|
output.clone(),
|
||||||
enabled,
|
enabled,
|
||||||
|
RetainedImageBudget::Disabled,
|
||||||
);
|
);
|
||||||
let mut expected = vec![
|
let mut expected = vec![
|
||||||
ResponseItemEnvelope {
|
ResponseItemEnvelope {
|
||||||
@@ -1132,3 +1185,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "compact_remote_v2_image_budget_tests.rs"]
|
||||||
|
mod image_budget_tests;
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
use super::*;
|
||||||
|
use codex_protocol::models::ContentItemKind;
|
||||||
|
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
|
||||||
|
use codex_protocol::models::image_close_tag_text;
|
||||||
|
use codex_protocol::models::local_image_open_tag_text_with_path;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
|
fn message(content: Vec<ContentItem>) -> ResponseItem {
|
||||||
|
ResponseItem::Message {
|
||||||
|
id: None,
|
||||||
|
role: "user".to_string(),
|
||||||
|
content,
|
||||||
|
phase: None,
|
||||||
|
internal_chat_message_metadata_passthrough: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn image() -> ContentItem {
|
||||||
|
ContentItem::InputImage {
|
||||||
|
image_url: "data:image/png;base64,abc".to_string(),
|
||||||
|
detail: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(value: &str) -> ContentItem {
|
||||||
|
ContentItem::InputText {
|
||||||
|
text: value.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trim(items: Vec<ResponseItem>, max_tokens: usize) -> Vec<ResponseItem> {
|
||||||
|
truncate_retained_messages(
|
||||||
|
items.into_iter().map(ResponseItemEnvelope::new).collect(),
|
||||||
|
max_tokens,
|
||||||
|
RetainedImageBudget::Enabled,
|
||||||
|
)
|
||||||
|
.into_iter()
|
||||||
|
.map(ResponseItemEnvelope::into_item)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn image_only_boundary_is_atomic_and_does_not_backfill_older_messages() {
|
||||||
|
let newest = message(vec![text("new")]);
|
||||||
|
let items = vec![
|
||||||
|
message(vec![text("old")]),
|
||||||
|
message(vec![image()]),
|
||||||
|
newest.clone(),
|
||||||
|
];
|
||||||
|
let image_tokens = images::content_item_token_count(&image());
|
||||||
|
for (max_tokens, expected) in [
|
||||||
|
(
|
||||||
|
image_tokens + 1,
|
||||||
|
vec![message(vec![image()]), newest.clone()],
|
||||||
|
),
|
||||||
|
(image_tokens, vec![newest.clone()]),
|
||||||
|
(1, vec![newest]),
|
||||||
|
] {
|
||||||
|
assert_eq!(trim(items.clone(), max_tokens), expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn later_image_parts_preserve_labels_audio_and_annotations() {
|
||||||
|
let parts = vec![
|
||||||
|
text("earlier text"),
|
||||||
|
text(&local_image_open_tag_text_with_path(
|
||||||
|
/*label_number*/ 7,
|
||||||
|
std::path::Path::new("image.png"),
|
||||||
|
)),
|
||||||
|
image(),
|
||||||
|
text(&image_close_tag_text()),
|
||||||
|
text("keep"),
|
||||||
|
ContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let kinds = (0..parts.len())
|
||||||
|
.map(|i| ContentItemKind(format!("part.{i}")))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let mut source = message(parts.clone());
|
||||||
|
let ResponseItem::Message {
|
||||||
|
internal_chat_message_metadata_passthrough,
|
||||||
|
..
|
||||||
|
} = &mut source
|
||||||
|
else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
*internal_chat_message_metadata_passthrough = Some(InternalChatMessageMetadataPassthrough {
|
||||||
|
turn_id: Some("turn-1".to_string()),
|
||||||
|
content_item_kinds: Some(kinds.clone()),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let image_tokens = parts[1..4]
|
||||||
|
.iter()
|
||||||
|
.map(images::content_item_token_count)
|
||||||
|
.sum::<usize>();
|
||||||
|
for (max_tokens, start) in [(image_tokens, 4), (image_tokens + 1, 1)] {
|
||||||
|
let mut expected = source.clone();
|
||||||
|
let ResponseItem::Message {
|
||||||
|
content,
|
||||||
|
internal_chat_message_metadata_passthrough,
|
||||||
|
..
|
||||||
|
} = &mut expected
|
||||||
|
else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
*content = parts[start..].to_vec();
|
||||||
|
internal_chat_message_metadata_passthrough
|
||||||
|
.as_mut()
|
||||||
|
.unwrap()
|
||||||
|
.content_item_kinds = Some(kinds[start..].to_vec());
|
||||||
|
assert_eq!(trim(vec![source.clone()], max_tokens), vec![expected]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn image_treatment_preserves_text_only_boundary_behavior() {
|
||||||
|
let source = ResponseItemEnvelope::new(message(vec![
|
||||||
|
text("earlier text"),
|
||||||
|
text(&"middle".repeat(20)),
|
||||||
|
text("later text"),
|
||||||
|
ContentItem::InputAudio {
|
||||||
|
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||||
|
},
|
||||||
|
]));
|
||||||
|
for max_tokens in [0, 1, 4, 16, 100] {
|
||||||
|
assert_eq!(
|
||||||
|
truncate_retained_messages(
|
||||||
|
vec![source.clone()],
|
||||||
|
max_tokens,
|
||||||
|
RetainedImageBudget::Enabled,
|
||||||
|
),
|
||||||
|
truncate_retained_messages_for_remote_compaction(vec![source.clone()], max_tokens),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn image_treatment_preserves_client_developer_boundary_behavior() {
|
||||||
|
let mut item = message(vec![image(), text(&"a".repeat(1000))]);
|
||||||
|
let ResponseItem::Message { role, .. } = &mut item else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
*role = "developer".to_string();
|
||||||
|
let source = ResponseItemEnvelope {
|
||||||
|
item,
|
||||||
|
metadata: Some(CodexHarnessMetadata {
|
||||||
|
client_authored: true,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
for max_tokens in [1871, 1879, 1880, 1950, 2200] {
|
||||||
|
assert_eq!(
|
||||||
|
truncate_retained_messages(
|
||||||
|
vec![source.clone()],
|
||||||
|
max_tokens,
|
||||||
|
RetainedImageBudget::Enabled
|
||||||
|
),
|
||||||
|
truncate_retained_messages_for_remote_compaction(vec![source.clone()], max_tokens),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
use crate::context_manager::estimate_image_bytes;
|
||||||
|
use codex_context_fragments::AnnotatedContent;
|
||||||
|
use codex_context_fragments::set_annotated_content;
|
||||||
|
use codex_context_fragments::to_annotated_content;
|
||||||
|
use codex_history::ResponseItemEnvelope;
|
||||||
|
use codex_protocol::models::ContentItem;
|
||||||
|
use codex_protocol::models::is_image_close_tag_text;
|
||||||
|
use codex_protocol::models::is_image_open_tag_text;
|
||||||
|
use codex_protocol::models::is_local_image_open_tag_text;
|
||||||
|
use codex_protocol::protocol::TruncationPolicy;
|
||||||
|
use codex_utils_output_truncation::approx_token_count;
|
||||||
|
use codex_utils_output_truncation::approx_tokens_from_byte_count_i64;
|
||||||
|
use codex_utils_output_truncation::truncate_text;
|
||||||
|
|
||||||
|
pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
|
||||||
|
match item {
|
||||||
|
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
|
||||||
|
approx_token_count(text)
|
||||||
|
}
|
||||||
|
ContentItem::InputImage { image_url, detail } => usize::try_from(
|
||||||
|
approx_tokens_from_byte_count_i64(estimate_image_bytes(image_url, *detail)),
|
||||||
|
)
|
||||||
|
.unwrap_or(usize::MAX),
|
||||||
|
ContentItem::InputAudio { .. } => 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retain later parts of an image-containing boundary message, keeping each
|
||||||
|
/// image and its adjacent harness labels atomic. Text keeps its existing middle
|
||||||
|
/// truncation policy; audio remains uncharged and is preserved as before.
|
||||||
|
pub(super) fn truncate_message_to_token_budget(
|
||||||
|
mut envelope: ResponseItemEnvelope,
|
||||||
|
max_tokens: usize,
|
||||||
|
) -> Option<ResponseItemEnvelope> {
|
||||||
|
let mut content = to_annotated_content(&mut envelope.item)?;
|
||||||
|
let mut remaining = max_tokens;
|
||||||
|
let mut retained = Vec::with_capacity(content.len());
|
||||||
|
while !content.is_empty() {
|
||||||
|
let last = content.len() - 1;
|
||||||
|
let image_index = match content[last].content() {
|
||||||
|
ContentItem::InputImage { .. } => Some(last),
|
||||||
|
ContentItem::InputText { text }
|
||||||
|
if is_image_close_tag_text(text)
|
||||||
|
&& last > 0
|
||||||
|
&& matches!(content[last - 1].content(), ContentItem::InputImage { .. }) =>
|
||||||
|
{
|
||||||
|
Some(last - 1)
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
if let Some(image_index) = image_index {
|
||||||
|
let has_open_tag = image_index > 0
|
||||||
|
&& matches!(
|
||||||
|
content[image_index - 1].content(),
|
||||||
|
ContentItem::InputText { text }
|
||||||
|
if is_local_image_open_tag_text(text) || is_image_open_tag_text(text)
|
||||||
|
);
|
||||||
|
let start = image_index - usize::from(has_open_tag);
|
||||||
|
let token_count = content[start..]
|
||||||
|
.iter()
|
||||||
|
.map(AnnotatedContent::content)
|
||||||
|
.map(content_item_token_count)
|
||||||
|
.sum::<usize>();
|
||||||
|
let fits = token_count <= remaining;
|
||||||
|
remaining = if fits { remaining - token_count } else { 0 };
|
||||||
|
for item in content.drain(start..).rev() {
|
||||||
|
if fits {
|
||||||
|
retained.push(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut item = content.pop()?;
|
||||||
|
match item.content_mut() {
|
||||||
|
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
|
||||||
|
if remaining == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let token_count = approx_token_count(text);
|
||||||
|
if token_count <= remaining {
|
||||||
|
remaining -= token_count;
|
||||||
|
} else {
|
||||||
|
*text = truncate_text(text, TruncationPolicy::Tokens(remaining));
|
||||||
|
remaining = 0;
|
||||||
|
}
|
||||||
|
if !text.is_empty() {
|
||||||
|
retained.push(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ContentItem::InputAudio { .. } => retained.push(item),
|
||||||
|
ContentItem::InputImage { .. } => unreachable!("images are handled atomically above"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if retained.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
retained.reverse();
|
||||||
|
set_annotated_content(&mut envelope.item, retained)?;
|
||||||
|
Some(envelope)
|
||||||
|
}
|
||||||
@@ -332,7 +332,9 @@ fn build_compacted_history_preserves_user_message_passthrough_metadata() {
|
|||||||
},
|
},
|
||||||
metadata: Some(CodexHarnessMetadata::default()),
|
metadata: Some(CodexHarnessMetadata::default()),
|
||||||
},
|
},
|
||||||
ResponseItemEnvelope::new(user_message("summary text")),
|
ResponseItemEnvelope::new(ContextualUserFragment::into(CompactionSummary::new(
|
||||||
|
"summary text",
|
||||||
|
))),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -769,6 +769,16 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Shared image estimate, excluding the data URL prefix and message framing.
|
||||||
|
pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {
|
||||||
|
match detail {
|
||||||
|
Some(ImageDetail::Original) => {
|
||||||
|
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
|
||||||
|
}
|
||||||
|
_ => RESIZED_IMAGE_BYTES_ESTIMATE,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Scans one response item for discount-eligible inline image data URLs and
|
/// Scans one response item for discount-eligible inline image data URLs and
|
||||||
/// returns:
|
/// returns:
|
||||||
/// - total base64 payload bytes to subtract from raw serialized size
|
/// - total base64 payload bytes to subtract from raw serialized size
|
||||||
@@ -781,12 +791,8 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
|||||||
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
|
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
|
||||||
payload_bytes =
|
payload_bytes =
|
||||||
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
|
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
|
||||||
replacement_bytes = replacement_bytes.saturating_add(match detail {
|
replacement_bytes =
|
||||||
Some(ImageDetail::Original) => {
|
replacement_bytes.saturating_add(estimate_image_bytes(image_url, detail));
|
||||||
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
|
|
||||||
}
|
|
||||||
_ => RESIZED_IMAGE_BYTES_ESTIMATE,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ mod normalize;
|
|||||||
pub(crate) mod updates;
|
pub(crate) mod updates;
|
||||||
|
|
||||||
pub(crate) use history::ContextManager;
|
pub(crate) use history::ContextManager;
|
||||||
|
pub(crate) use history::estimate_image_bytes;
|
||||||
pub(crate) use history::estimate_item_token_count;
|
pub(crate) use history::estimate_item_token_count;
|
||||||
pub(crate) use history::is_user_turn_boundary;
|
pub(crate) use history::is_user_turn_boundary;
|
||||||
pub(crate) use history::truncate_function_output_payload;
|
pub(crate) use history::truncate_function_output_payload;
|
||||||
@@ -11215,9 +11215,7 @@ async fn sample_rollout(
|
|||||||
rollout_items.push(RolloutItem::ResponseItem(assistant1.clone().into()));
|
rollout_items.push(RolloutItem::ResponseItem(assistant1.clone().into()));
|
||||||
|
|
||||||
let summary1 = "summary one";
|
let summary1 = "summary one";
|
||||||
let snapshot1 = live_history
|
let snapshot1 = raw_history_items(&live_history);
|
||||||
.clone()
|
|
||||||
.for_prompt(&reconstruction_turn.model_info.input_modalities);
|
|
||||||
let user_messages1 = collect_user_messages(&snapshot1);
|
let user_messages1 = collect_user_messages(&snapshot1);
|
||||||
let rebuilt1 = compact::build_compacted_history(Vec::new(), &user_messages1, summary1);
|
let rebuilt1 = compact::build_compacted_history(Vec::new(), &user_messages1, summary1);
|
||||||
live_history.replace_annotated(rebuilt1);
|
live_history.replace_annotated(rebuilt1);
|
||||||
@@ -11263,9 +11261,7 @@ async fn sample_rollout(
|
|||||||
rollout_items.push(RolloutItem::ResponseItem(assistant2.clone().into()));
|
rollout_items.push(RolloutItem::ResponseItem(assistant2.clone().into()));
|
||||||
|
|
||||||
let summary2 = "summary two";
|
let summary2 = "summary two";
|
||||||
let snapshot2 = live_history
|
let snapshot2 = raw_history_items(&live_history);
|
||||||
.clone()
|
|
||||||
.for_prompt(&reconstruction_turn.model_info.input_modalities);
|
|
||||||
let user_messages2 = collect_user_messages(&snapshot2);
|
let user_messages2 = collect_user_messages(&snapshot2);
|
||||||
let rebuilt2 = compact::build_compacted_history(Vec::new(), &user_messages2, summary2);
|
let rebuilt2 = compact::build_compacted_history(Vec::new(), &user_messages2, summary2);
|
||||||
live_history.replace_annotated(rebuilt2);
|
live_history.replace_annotated(rebuilt2);
|
||||||
@@ -11310,10 +11306,7 @@ async fn sample_rollout(
|
|||||||
);
|
);
|
||||||
rollout_items.push(RolloutItem::ResponseItem(assistant3.into()));
|
rollout_items.push(RolloutItem::ResponseItem(assistant3.into()));
|
||||||
|
|
||||||
(
|
(rollout_items, raw_history_items(&live_history))
|
||||||
rollout_items,
|
|
||||||
strip_metadata_from_items(&raw_history_items(&live_history)),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -1328,6 +1328,150 @@ async fn remote_manual_compact_chatgpt_auth_reuses_service_tier_and_prompt_cache
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test_case(None; "default_preserves_images")]
|
||||||
|
#[test_case(Some(false); "disabled_preserves_images")]
|
||||||
|
#[test_case(Some(true); "enabled_trims_images")]
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn remote_compact_v2_charges_retained_images_to_token_budget(
|
||||||
|
image_budget_enabled: Option<bool>,
|
||||||
|
) -> Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
|
||||||
|
let harness = TestCodexHarness::with_auto_env_builder(
|
||||||
|
test_codex()
|
||||||
|
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||||
|
.with_config(move |config| {
|
||||||
|
let _ = config.features.enable(Feature::RemoteCompactionV2);
|
||||||
|
let _ = config.features.enable(Feature::UnifiedImageBudget);
|
||||||
|
if let Some(enabled) = image_budget_enabled {
|
||||||
|
let _ = config
|
||||||
|
.features
|
||||||
|
.set_enabled(Feature::CompactionImageBudget, enabled);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let codex = &harness.test().codex;
|
||||||
|
// Each original-detail image costs 10,000 estimated patch tokens.
|
||||||
|
let image_inputs = (1..=8)
|
||||||
|
.map(|number| {
|
||||||
|
let image = image::ImageBuffer::from_pixel(
|
||||||
|
/*width*/ 3200,
|
||||||
|
/*height*/ 3200,
|
||||||
|
image::Luma([number as u8]),
|
||||||
|
);
|
||||||
|
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||||
|
image.write_to(&mut bytes, image::ImageFormat::Png)?;
|
||||||
|
Ok(UserInput::Image {
|
||||||
|
image_url: format!(
|
||||||
|
"data:image/png;base64,{}",
|
||||||
|
BASE64_STANDARD.encode(bytes.get_ref())
|
||||||
|
),
|
||||||
|
detail: Some(codex_protocol::models::ImageDetail::Original),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
let mut input = image_inputs[..7].to_vec();
|
||||||
|
input.push(UserInput::Text {
|
||||||
|
text: "Compare these images".to_string(),
|
||||||
|
text_elements: Vec::new(),
|
||||||
|
});
|
||||||
|
let initial_mock = mount_sse_once(
|
||||||
|
harness.server(),
|
||||||
|
sse(vec![
|
||||||
|
responses::ev_assistant_message("initial", "done"),
|
||||||
|
responses::ev_completed("initial"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
codex
|
||||||
|
.start_or_steer_turn(TurnInputRequest::user_input(input))
|
||||||
|
.await?;
|
||||||
|
wait_for_turn_complete(codex).await;
|
||||||
|
let initial_request = initial_mock.single_request();
|
||||||
|
let prepared_images = initial_request.message_input_image_urls("user");
|
||||||
|
assert_eq!(prepared_images.len(), 7);
|
||||||
|
|
||||||
|
for cycle in 1..=2 {
|
||||||
|
let compact_mock = mount_sse_once(
|
||||||
|
harness.server(),
|
||||||
|
sse(vec![
|
||||||
|
json!({
|
||||||
|
"type": "response.output_item.done",
|
||||||
|
"item": { "type": "compaction", "encrypted_content": "IMAGE_BUDGET_SUMMARY" },
|
||||||
|
}),
|
||||||
|
responses::ev_completed("compact-images"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
codex.submit(Op::Compact).await?;
|
||||||
|
wait_for_turn_complete(codex).await;
|
||||||
|
let compact_request = compact_mock.single_request();
|
||||||
|
assert_eq!(compact_request.path(), "/v1/responses");
|
||||||
|
assert_eq!(
|
||||||
|
compact_request.inputs_of_type("compaction_trigger").len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
|
||||||
|
let follow_up_mock = mount_sse_once(
|
||||||
|
harness.server(),
|
||||||
|
sse(vec![
|
||||||
|
responses::ev_assistant_message("after", "done"),
|
||||||
|
responses::ev_completed("after"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
codex
|
||||||
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
|
||||||
|
text: "after compact".to_string(),
|
||||||
|
text_elements: Vec::new(),
|
||||||
|
}]))
|
||||||
|
.await?;
|
||||||
|
wait_for_turn_complete(codex).await;
|
||||||
|
let follow_up = follow_up_mock.single_request();
|
||||||
|
assert_eq!(
|
||||||
|
follow_up.inputs_of_type("compaction")[0]["encrypted_content"],
|
||||||
|
"IMAGE_BUDGET_SUMMARY"
|
||||||
|
);
|
||||||
|
let dropped = if image_budget_enabled == Some(true) {
|
||||||
|
cycle
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let mut expected_images = prepared_images[dropped..].to_vec();
|
||||||
|
if cycle == 2 {
|
||||||
|
let UserInput::Image { image_url, .. } = &image_inputs[7] else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
expected_images.push(image_url.clone());
|
||||||
|
}
|
||||||
|
assert_eq!(follow_up.message_input_image_urls("user"), expected_images);
|
||||||
|
assert!(
|
||||||
|
follow_up
|
||||||
|
.message_input_texts("user")
|
||||||
|
.iter()
|
||||||
|
.any(|text| text == "Compare these images")
|
||||||
|
);
|
||||||
|
|
||||||
|
if cycle == 1 {
|
||||||
|
let append_mock = mount_sse_once(
|
||||||
|
harness.server(),
|
||||||
|
sse(vec![
|
||||||
|
responses::ev_assistant_message("append", "done"),
|
||||||
|
responses::ev_completed("append"),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
codex
|
||||||
|
.start_or_steer_turn(TurnInputRequest::user_input(vec![image_inputs[7].clone()]))
|
||||||
|
.await?;
|
||||||
|
wait_for_turn_complete(codex).await;
|
||||||
|
let _ = append_mock.single_request();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<()> {
|
async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
|
|||||||
@@ -305,6 +305,8 @@ pub enum Feature {
|
|||||||
PreventIdleSleep,
|
PreventIdleSleep,
|
||||||
/// Enable remote compaction v2 over the normal Responses API.
|
/// Enable remote compaction v2 over the normal Responses API.
|
||||||
RemoteCompactionV2,
|
RemoteCompactionV2,
|
||||||
|
/// Include retained images in the remote compaction context budget.
|
||||||
|
CompactionImageBudget,
|
||||||
/// Retain client-authored developer messages across compacted context windows.
|
/// Retain client-authored developer messages across compacted context windows.
|
||||||
RetainClientDeveloperMessages,
|
RetainClientDeveloperMessages,
|
||||||
/// Use Agent Identity for ChatGPT-authenticated sessions.
|
/// Use Agent Identity for ChatGPT-authenticated sessions.
|
||||||
@@ -1545,6 +1547,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
|||||||
stage: Stage::Stable,
|
stage: Stage::Stable,
|
||||||
default_enabled: true,
|
default_enabled: true,
|
||||||
},
|
},
|
||||||
|
FeatureSpec {
|
||||||
|
id: Feature::CompactionImageBudget,
|
||||||
|
key: "compaction_image_budget",
|
||||||
|
stage: Stage::UnderDevelopment,
|
||||||
|
default_enabled: false,
|
||||||
|
},
|
||||||
FeatureSpec {
|
FeatureSpec {
|
||||||
id: Feature::RetainClientDeveloperMessages,
|
id: Feature::RetainClientDeveloperMessages,
|
||||||
key: "retain_client_developer_messages",
|
key: "retain_client_developer_messages",
|
||||||
|
|||||||
Reference in new issue
Block a user