bound prompt image cache retention (#28294)

## Why

The prompt image cache was bounded to 32 entries, but not by the size of
those entries. A set of large encoded images could therefore retain
substantially more memory than intended. Cache hits also cloned the full
encoded payload.

## What changed

- cap the cache at 64 MiB of encoded image data while preserving its
existing 32-entry limit
- skip caching an image that exceeds the entire byte budget
- evict least-recently-used entries until the cache is back within its
byte budget
- share cached encoded bytes with `Arc<[u8]>` so cache hits do not
deep-clone image payloads

## Validation

- `just test -p codex-utils-image`
This commit is contained in:
jif authored and GitHub committed 2026-06-15 19:52:30 +02:00
1 parent 29224d945b
commit aae4c0b02b
2 files changed
+64 -10

No files matched your search

+38 -7
View File
@@ -1,6 +1,7 @@
use std::io::Cursor;
use std::num::NonZeroUsize;
use std::path::Path;
use std::sync::Arc;
use std::sync::LazyLock;
use base64::Engine;
@@ -28,6 +29,7 @@ pub const MAX_DIMENSION: u32 = 2048;
/// This is a high sanity guard against pathological inputs, not a protocol
/// requirement or target upload size.
pub const MAX_PROMPT_IMAGE_INPUT_BYTES: usize = 1024 * 1024 * 1024;
const MAX_IMAGE_CACHE_BYTES: usize = 64 * 1024 * 1024;
pub mod error;
@@ -35,7 +37,7 @@ pub use crate::error::ImageProcessingError;
#[derive(Debug, Clone)]
pub struct EncodedImage {
pub bytes: Vec<u8>,
pub bytes: Arc<[u8]>,
pub mime: String,
pub width: u32,
pub height: u32,
@@ -77,7 +79,9 @@ struct ImageCacheKey {
mode: PromptImageMode,
}
static IMAGE_CACHE: LazyLock<BlockingLruCache<ImageCacheKey, EncodedImage>> =
type ImageCache = BlockingLruCache<ImageCacheKey, EncodedImage>;
static IMAGE_CACHE: LazyLock<ImageCache> =
LazyLock::new(|| BlockingLruCache::new(NonZeroUsize::new(32).unwrap_or(NonZeroUsize::MIN)));
pub fn load_for_prompt_bytes(
@@ -92,7 +96,11 @@ pub fn load_for_prompt_bytes(
mode,
};
IMAGE_CACHE.get_or_try_insert_with(key, move || {
if let Some(image) = IMAGE_CACHE.get(&key) {
return Ok(image);
}
let image = (move || {
let guessed_format = image::guess_format(&file_bytes)
.map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?;
let format = match guessed_format {
@@ -151,7 +159,7 @@ pub fn load_for_prompt_bytes(
let (bytes, output_format) = encode_image(&resized, target_format, metadata)?;
let mime = format_to_mime(output_format);
EncodedImage {
bytes,
bytes: bytes.into(),
mime,
width,
height,
@@ -160,7 +168,7 @@ pub fn load_for_prompt_bytes(
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) {
let mime = format_to_mime(format);
EncodedImage {
bytes: file_bytes,
bytes: file_bytes.into(),
mime,
width,
height,
@@ -169,7 +177,7 @@ pub fn load_for_prompt_bytes(
let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png, metadata)?;
let mime = format_to_mime(output_format);
EncodedImage {
bytes,
bytes: bytes.into(),
mime,
width,
height,
@@ -178,7 +186,30 @@ pub fn load_for_prompt_bytes(
};
Ok(encoded)
})
})()?;
cache_image(&IMAGE_CACHE, key, image.clone(), MAX_IMAGE_CACHE_BYTES);
Ok(image)
}
fn cache_image(cache: &ImageCache, key: ImageCacheKey, image: EncodedImage, byte_capacity: usize) {
if image.bytes.len() > byte_capacity {
return;
}
cache.with_mut(|cache| {
cache.put(key, image);
let mut cached_bytes = cache
.iter()
.map(|(_, image)| image.bytes.len())
.sum::<usize>();
while cached_bytes > byte_capacity {
let Some((_, evicted)) = cache.pop_lru() else {
break;
};
cached_bytes -= evicted.bytes.len();
}
});
}
pub fn load_data_url_for_prompt(