fix(redis): bound collection value pages by payload size

This commit is contained in:
Loc Nguyen authored and GitHub committed 2026-09-26 22:48:24 +08:00
1 parent ed863e9e59
commit daebae9b50
1 file changed
+117 -17
+117 -17
View File
@@ -17,6 +17,15 @@ const STREAM_ENTRY_PAGE_SIZE: usize = 50;
const STREAM_PENDING_PAGE_SIZE: usize = 100;
const COLLECTION_PAGE_SIZE: usize = 200;
const STRING_PREVIEW_MAX_BYTES: usize = 64 * 1024;
/// A collection page is capped by payload size as well as item count.
///
/// `COLLECTION_PAGE_SIZE` alone bounds a page only when the members are small. A hash, list, set
/// or sorted set whose members are serialized blobs turns 200 members into hundreds of megabytes,
/// which then has to cross the IPC boundary and be parsed and rendered, so opening the key looks
/// like it does nothing. Strings already avoid this through `STRING_PREVIEW_MAX_BYTES`; this is
/// the same guard for collections, except that no member is ever truncated — the page simply ends
/// early and the remainder is fetched by the existing "load more" cursor.
const COLLECTION_PAGE_MAX_BYTES: usize = 4 * 1024 * 1024;
const COLLECTION_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
const DEFAULT_REDIS_DATABASES: u32 = 16;
const JS_MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
@@ -2682,8 +2691,10 @@ where
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
let v: RedisRawValue =
redis::cmd("LRANGE").arg(key).arg(0).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None };
RedisValueData::List { items: redis_list_items_from_raw(v, 0), total: len, scan_cursor: cursor }
let mut items = redis_list_items_from_raw(v, 0);
items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, list_item_page_bytes));
let next = items.len() as u64;
RedisValueData::List { items, total: len, scan_cursor: (next < len).then_some(next) }
}
"set" => {
let len: u64 = redis::cmd("SCARD").arg(key).query_async(con).await.unwrap_or(0);
@@ -2693,9 +2704,10 @@ where
"zset" => {
let len: u64 = redis::cmd("ZCARD").arg(key).query_async(con).await.unwrap_or(0);
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
let items = zrange_page_raw(con, key, 0, end, false).await?;
let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None };
RedisValueData::Zset { items, total: len, scan_cursor: cursor }
let mut items = zrange_page_raw(con, key, 0, end, false).await?;
items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, zset_item_page_bytes));
let next = items.len() as u64;
RedisValueData::Zset { items, total: len, scan_cursor: (next < len).then_some(next) }
}
"hash" => {
let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0);
@@ -3924,11 +3936,10 @@ where
let end = start + count as i64 - 1;
let v: RedisRawValue =
redis::cmd("LRANGE").arg(key).arg(start).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
let next = cursor + count as u64;
Ok(RedisCollectionPage::List {
items: redis_list_items_from_raw(v, cursor),
scan_cursor: (next < len).then_some(next),
})
let mut items = redis_list_items_from_raw(v, cursor);
items.truncate(budgeted_page_len(&items, count, list_item_page_bytes));
let next = cursor + items.len() as u64;
Ok(RedisCollectionPage::List { items, scan_cursor: (next < len).then_some(next) })
}
"set" => {
let (next_cursor, items) = if let Some(query) = filter_query {
@@ -3952,10 +3963,14 @@ where
let start = cursor as i64;
let end = start + count as i64;
let mut items = zrange_page_raw(con, key, start, end, descending).await?;
// `zrange_page_raw` reads one extra member to detect a further page; the byte budget
// can also end the page before `count`, which likewise leaves members behind.
let has_more = items.len() > count;
items.truncate(count);
let next = cursor + count as u64;
Ok(RedisCollectionPage::Zset { items, scan_cursor: has_more.then_some(next) })
let page_len = budgeted_page_len(&items, count, zset_item_page_bytes);
let ended_early = page_len < items.len().min(count);
items.truncate(page_len);
let next = cursor + items.len() as u64;
Ok(RedisCollectionPage::Zset { items, scan_cursor: (has_more || ended_early).then_some(next) })
}
"hash" => {
let (next_cursor, mut items) = if let Some(query) = filter_query {
@@ -4150,7 +4165,8 @@ where
} else {
hscan_page_raw(con, key, cursor, count, None).await?
};
let next_cursor = store_hash_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
let page_len = budgeted_page_len(&items, count, hash_item_page_bytes);
let next_cursor = store_hash_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await;
Ok((next_cursor, items))
}
@@ -4174,7 +4190,8 @@ where
} else {
sscan_page_raw(con, key, cursor, count).await?
};
let next_cursor = store_set_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
let page_len = budgeted_page_len(&items, count, set_item_page_bytes);
let next_cursor = store_set_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await;
Ok((next_cursor, items))
}
@@ -4191,7 +4208,9 @@ async fn take_hash_overflow_page(
let mut sessions = collection_overflow_sessions().lock().await;
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Hash, filter_query, None)?;
let page = match &mut session.items {
RedisCollectionOverflowItems::Hash(items) => take_vec_page(items, count),
RedisCollectionOverflowItems::Hash(items) => {
take_vec_page(items, budgeted_page_len(items, count, hash_item_page_bytes))
}
RedisCollectionOverflowItems::Set(_) => return None,
};
let has_buffered_items = match &session.items {
@@ -4218,7 +4237,9 @@ async fn take_set_overflow_page(
let mut sessions = collection_overflow_sessions().lock().await;
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Set, filter_query, None)?;
let page = match &mut session.items {
RedisCollectionOverflowItems::Set(items) => take_vec_page(items, count),
RedisCollectionOverflowItems::Set(items) => {
take_vec_page(items, budgeted_page_len(items, count, set_item_page_bytes))
}
RedisCollectionOverflowItems::Hash(_) => return None,
};
let has_buffered_items = match &session.items {
@@ -4280,6 +4301,42 @@ async fn store_set_overflow_page(
collection_overflow_sessions().lock().await.insert(session)
}
/// How many leading items fit in one page, honouring both the item count and the byte budget.
///
/// Always returns at least one item so a member larger than the whole budget still loads (and
/// pagination cannot stall), and never more than `items.len()`.
fn budgeted_page_len<T>(items: &[T], count: usize, size_of: impl Fn(&T) -> usize) -> usize {
let limit = count.max(1).min(items.len());
let mut used = 0usize;
for (index, item) in items.iter().take(limit).enumerate() {
used = used.saturating_add(size_of(item));
if used > COLLECTION_PAGE_MAX_BYTES {
return index.max(1);
}
}
limit
}
fn blob_page_bytes(blob: &RedisBlob) -> usize {
blob.raw_base64.len()
}
fn hash_item_page_bytes(item: &RedisHashItem) -> usize {
blob_page_bytes(&item.field) + blob_page_bytes(&item.value)
}
fn set_item_page_bytes(item: &RedisSetItem) -> usize {
blob_page_bytes(&item.member)
}
fn list_item_page_bytes(item: &RedisListItem) -> usize {
blob_page_bytes(&item.value)
}
fn zset_item_page_bytes(item: &RedisZsetItem) -> usize {
blob_page_bytes(&item.member) + item.score.len()
}
fn take_vec_page<T>(items: &mut Vec<T>, count: usize) -> Vec<T> {
let page_size = items.len().min(count.max(1));
let overflow = items.split_off(page_size);
@@ -5006,6 +5063,49 @@ mod tests {
assert_eq!(con.command_count("STRLEN"), 1);
}
#[test]
fn collection_pages_stop_at_the_byte_budget_without_truncating_a_member() {
fn hash_items(count: usize, value_bytes: usize) -> Vec<RedisHashItem> {
(0..count)
.map(|index| RedisHashItem {
field: text_blob(&format!("f{index}")),
value: text_blob(&"v".repeat(value_bytes)),
field_ttl: None,
})
.collect()
}
fn page_bytes(items: &[RedisHashItem]) -> usize {
items.iter().map(super::hash_item_page_bytes).sum()
}
// Members are sized as a fraction of the budget; base64 inflates them by 4/3 on the wire,
// so the assertions below are on the resulting page rather than on a hand-computed count.
for divisor in [2, 3, 8] {
let items = hash_items(200, super::COLLECTION_PAGE_MAX_BYTES / divisor);
let page_len = super::budgeted_page_len(&items, 200, super::hash_item_page_bytes);
assert!((1..200).contains(&page_len), "divisor {divisor}: page_len {page_len}");
assert!(
page_bytes(&items[..page_len]) <= super::COLLECTION_PAGE_MAX_BYTES,
"divisor {divisor} over budget"
);
assert!(
page_bytes(&items[..page_len + 1]) > super::COLLECTION_PAGE_MAX_BYTES,
"divisor {divisor}: page ended earlier than the budget required"
);
}
// Small members still fill a whole page, and a short input is returned whole.
let small = hash_items(500, 1);
assert_eq!(super::budgeted_page_len(&small, 200, super::hash_item_page_bytes), 200);
assert_eq!(super::budgeted_page_len(&small[..5], 200, super::hash_item_page_bytes), 5);
// A single member larger than the whole budget still loads, so paging cannot stall.
let huge = hash_items(3, super::COLLECTION_PAGE_MAX_BYTES * 2);
assert_eq!(super::budgeted_page_len(&huge, 200, super::hash_item_page_bytes), 1);
assert_eq!(super::budgeted_page_len::<RedisHashItem>(&[], 200, super::hash_item_page_bytes), 0);
}
#[tokio::test]
async fn hash_value_pages_compact_hscan_before_field_ttls() {
let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();