From daebae9b5058fb6f4e4e871c5746cad0c877ca53 Mon Sep 17 00:00:00 2001 From: Loc Nguyen <113083148+thailoc-dev@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:48:24 +0700 Subject: [PATCH] fix(redis): bound collection value pages by payload size --- crates/dbx-driver-redis/src/lib.rs | 134 +++++++++++++++++++++++++---- 1 file changed, 117 insertions(+), 17 deletions(-) diff --git a/crates/dbx-driver-redis/src/lib.rs b/crates/dbx-driver-redis/src/lib.rs index 46ab0384fd..1d96258d78 100644 --- a/crates/dbx-driver-redis/src/lib.rs +++ b/crates/dbx-driver-redis/src/lib.rs @@ -17,6 +17,15 @@ const STREAM_ENTRY_PAGE_SIZE: usize = 50; const STREAM_PENDING_PAGE_SIZE: usize = 100; const COLLECTION_PAGE_SIZE: usize = 200; const STRING_PREVIEW_MAX_BYTES: usize = 64 * 1024; +/// A collection page is capped by payload size as well as item count. +/// +/// `COLLECTION_PAGE_SIZE` alone bounds a page only when the members are small. A hash, list, set +/// or sorted set whose members are serialized blobs turns 200 members into hundreds of megabytes, +/// which then has to cross the IPC boundary and be parsed and rendered, so opening the key looks +/// like it does nothing. Strings already avoid this through `STRING_PREVIEW_MAX_BYTES`; this is +/// the same guard for collections, except that no member is ever truncated — the page simply ends +/// early and the remainder is fetched by the existing "load more" cursor. +const COLLECTION_PAGE_MAX_BYTES: usize = 4 * 1024 * 1024; const COLLECTION_FILTER_SCAN_MAX_ITERATIONS: usize = 10; const DEFAULT_REDIS_DATABASES: u32 = 16; const JS_MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991; @@ -2682,8 +2691,10 @@ where let end = (COLLECTION_PAGE_SIZE as i64) - 1; let v: RedisRawValue = redis::cmd("LRANGE").arg(key).arg(0).arg(end).query_async(con).await.map_err(|e| e.to_string())?; - let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None }; - RedisValueData::List { items: redis_list_items_from_raw(v, 0), total: len, scan_cursor: cursor } + let mut items = redis_list_items_from_raw(v, 0); + items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, list_item_page_bytes)); + let next = items.len() as u64; + RedisValueData::List { items, total: len, scan_cursor: (next < len).then_some(next) } } "set" => { let len: u64 = redis::cmd("SCARD").arg(key).query_async(con).await.unwrap_or(0); @@ -2693,9 +2704,10 @@ where "zset" => { let len: u64 = redis::cmd("ZCARD").arg(key).query_async(con).await.unwrap_or(0); let end = (COLLECTION_PAGE_SIZE as i64) - 1; - let items = zrange_page_raw(con, key, 0, end, false).await?; - let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None }; - RedisValueData::Zset { items, total: len, scan_cursor: cursor } + let mut items = zrange_page_raw(con, key, 0, end, false).await?; + items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, zset_item_page_bytes)); + let next = items.len() as u64; + RedisValueData::Zset { items, total: len, scan_cursor: (next < len).then_some(next) } } "hash" => { let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0); @@ -3924,11 +3936,10 @@ where let end = start + count as i64 - 1; let v: RedisRawValue = redis::cmd("LRANGE").arg(key).arg(start).arg(end).query_async(con).await.map_err(|e| e.to_string())?; - let next = cursor + count as u64; - Ok(RedisCollectionPage::List { - items: redis_list_items_from_raw(v, cursor), - scan_cursor: (next < len).then_some(next), - }) + let mut items = redis_list_items_from_raw(v, cursor); + items.truncate(budgeted_page_len(&items, count, list_item_page_bytes)); + let next = cursor + items.len() as u64; + Ok(RedisCollectionPage::List { items, scan_cursor: (next < len).then_some(next) }) } "set" => { let (next_cursor, items) = if let Some(query) = filter_query { @@ -3952,10 +3963,14 @@ where let start = cursor as i64; let end = start + count as i64; let mut items = zrange_page_raw(con, key, start, end, descending).await?; + // `zrange_page_raw` reads one extra member to detect a further page; the byte budget + // can also end the page before `count`, which likewise leaves members behind. let has_more = items.len() > count; - items.truncate(count); - let next = cursor + count as u64; - Ok(RedisCollectionPage::Zset { items, scan_cursor: has_more.then_some(next) }) + let page_len = budgeted_page_len(&items, count, zset_item_page_bytes); + let ended_early = page_len < items.len().min(count); + items.truncate(page_len); + let next = cursor + items.len() as u64; + Ok(RedisCollectionPage::Zset { items, scan_cursor: (has_more || ended_early).then_some(next) }) } "hash" => { let (next_cursor, mut items) = if let Some(query) = filter_query { @@ -4150,7 +4165,8 @@ where } else { hscan_page_raw(con, key, cursor, count, None).await? }; - let next_cursor = store_hash_overflow_page(key, &mut items, count, next_cursor, filter_query).await; + let page_len = budgeted_page_len(&items, count, hash_item_page_bytes); + let next_cursor = store_hash_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await; Ok((next_cursor, items)) } @@ -4174,7 +4190,8 @@ where } else { sscan_page_raw(con, key, cursor, count).await? }; - let next_cursor = store_set_overflow_page(key, &mut items, count, next_cursor, filter_query).await; + let page_len = budgeted_page_len(&items, count, set_item_page_bytes); + let next_cursor = store_set_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await; Ok((next_cursor, items)) } @@ -4191,7 +4208,9 @@ async fn take_hash_overflow_page( let mut sessions = collection_overflow_sessions().lock().await; let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Hash, filter_query, None)?; let page = match &mut session.items { - RedisCollectionOverflowItems::Hash(items) => take_vec_page(items, count), + RedisCollectionOverflowItems::Hash(items) => { + take_vec_page(items, budgeted_page_len(items, count, hash_item_page_bytes)) + } RedisCollectionOverflowItems::Set(_) => return None, }; let has_buffered_items = match &session.items { @@ -4218,7 +4237,9 @@ async fn take_set_overflow_page( let mut sessions = collection_overflow_sessions().lock().await; let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Set, filter_query, None)?; let page = match &mut session.items { - RedisCollectionOverflowItems::Set(items) => take_vec_page(items, count), + RedisCollectionOverflowItems::Set(items) => { + take_vec_page(items, budgeted_page_len(items, count, set_item_page_bytes)) + } RedisCollectionOverflowItems::Hash(_) => return None, }; let has_buffered_items = match &session.items { @@ -4280,6 +4301,42 @@ async fn store_set_overflow_page( collection_overflow_sessions().lock().await.insert(session) } +/// How many leading items fit in one page, honouring both the item count and the byte budget. +/// +/// Always returns at least one item so a member larger than the whole budget still loads (and +/// pagination cannot stall), and never more than `items.len()`. +fn budgeted_page_len(items: &[T], count: usize, size_of: impl Fn(&T) -> usize) -> usize { + let limit = count.max(1).min(items.len()); + let mut used = 0usize; + for (index, item) in items.iter().take(limit).enumerate() { + used = used.saturating_add(size_of(item)); + if used > COLLECTION_PAGE_MAX_BYTES { + return index.max(1); + } + } + limit +} + +fn blob_page_bytes(blob: &RedisBlob) -> usize { + blob.raw_base64.len() +} + +fn hash_item_page_bytes(item: &RedisHashItem) -> usize { + blob_page_bytes(&item.field) + blob_page_bytes(&item.value) +} + +fn set_item_page_bytes(item: &RedisSetItem) -> usize { + blob_page_bytes(&item.member) +} + +fn list_item_page_bytes(item: &RedisListItem) -> usize { + blob_page_bytes(&item.value) +} + +fn zset_item_page_bytes(item: &RedisZsetItem) -> usize { + blob_page_bytes(&item.member) + item.score.len() +} + fn take_vec_page(items: &mut Vec, count: usize) -> Vec { let page_size = items.len().min(count.max(1)); let overflow = items.split_off(page_size); @@ -5006,6 +5063,49 @@ mod tests { assert_eq!(con.command_count("STRLEN"), 1); } + #[test] + fn collection_pages_stop_at_the_byte_budget_without_truncating_a_member() { + fn hash_items(count: usize, value_bytes: usize) -> Vec { + (0..count) + .map(|index| RedisHashItem { + field: text_blob(&format!("f{index}")), + value: text_blob(&"v".repeat(value_bytes)), + field_ttl: None, + }) + .collect() + } + fn page_bytes(items: &[RedisHashItem]) -> usize { + items.iter().map(super::hash_item_page_bytes).sum() + } + + // Members are sized as a fraction of the budget; base64 inflates them by 4/3 on the wire, + // so the assertions below are on the resulting page rather than on a hand-computed count. + for divisor in [2, 3, 8] { + let items = hash_items(200, super::COLLECTION_PAGE_MAX_BYTES / divisor); + let page_len = super::budgeted_page_len(&items, 200, super::hash_item_page_bytes); + assert!((1..200).contains(&page_len), "divisor {divisor}: page_len {page_len}"); + assert!( + page_bytes(&items[..page_len]) <= super::COLLECTION_PAGE_MAX_BYTES, + "divisor {divisor} over budget" + ); + assert!( + page_bytes(&items[..page_len + 1]) > super::COLLECTION_PAGE_MAX_BYTES, + "divisor {divisor}: page ended earlier than the budget required" + ); + } + + // Small members still fill a whole page, and a short input is returned whole. + let small = hash_items(500, 1); + assert_eq!(super::budgeted_page_len(&small, 200, super::hash_item_page_bytes), 200); + assert_eq!(super::budgeted_page_len(&small[..5], 200, super::hash_item_page_bytes), 5); + + // A single member larger than the whole budget still loads, so paging cannot stall. + let huge = hash_items(3, super::COLLECTION_PAGE_MAX_BYTES * 2); + assert_eq!(super::budgeted_page_len(&huge, 200, super::hash_item_page_bytes), 1); + + assert_eq!(super::budgeted_page_len::(&[], 200, super::hash_item_page_bytes), 0); + } + #[tokio::test] async fn hash_value_pages_compact_hscan_before_field_ttls() { let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();