fix(redis): bound collection value pages by payload size
This commit is contained in:
1 parent
ed863e9e59
commit
daebae9b50
1 file changed
+117
-17
@@ -17,6 +17,15 @@ const STREAM_ENTRY_PAGE_SIZE: usize = 50;
|
|||||||
const STREAM_PENDING_PAGE_SIZE: usize = 100;
|
const STREAM_PENDING_PAGE_SIZE: usize = 100;
|
||||||
const COLLECTION_PAGE_SIZE: usize = 200;
|
const COLLECTION_PAGE_SIZE: usize = 200;
|
||||||
const STRING_PREVIEW_MAX_BYTES: usize = 64 * 1024;
|
const STRING_PREVIEW_MAX_BYTES: usize = 64 * 1024;
|
||||||
|
/// A collection page is capped by payload size as well as item count.
|
||||||
|
///
|
||||||
|
/// `COLLECTION_PAGE_SIZE` alone bounds a page only when the members are small. A hash, list, set
|
||||||
|
/// or sorted set whose members are serialized blobs turns 200 members into hundreds of megabytes,
|
||||||
|
/// which then has to cross the IPC boundary and be parsed and rendered, so opening the key looks
|
||||||
|
/// like it does nothing. Strings already avoid this through `STRING_PREVIEW_MAX_BYTES`; this is
|
||||||
|
/// the same guard for collections, except that no member is ever truncated — the page simply ends
|
||||||
|
/// early and the remainder is fetched by the existing "load more" cursor.
|
||||||
|
const COLLECTION_PAGE_MAX_BYTES: usize = 4 * 1024 * 1024;
|
||||||
const COLLECTION_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
const COLLECTION_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
||||||
const DEFAULT_REDIS_DATABASES: u32 = 16;
|
const DEFAULT_REDIS_DATABASES: u32 = 16;
|
||||||
const JS_MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
|
const JS_MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
|
||||||
@@ -2682,8 +2691,10 @@ where
|
|||||||
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
|
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
|
||||||
let v: RedisRawValue =
|
let v: RedisRawValue =
|
||||||
redis::cmd("LRANGE").arg(key).arg(0).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
|
redis::cmd("LRANGE").arg(key).arg(0).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
|
||||||
let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None };
|
let mut items = redis_list_items_from_raw(v, 0);
|
||||||
RedisValueData::List { items: redis_list_items_from_raw(v, 0), total: len, scan_cursor: cursor }
|
items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, list_item_page_bytes));
|
||||||
|
let next = items.len() as u64;
|
||||||
|
RedisValueData::List { items, total: len, scan_cursor: (next < len).then_some(next) }
|
||||||
}
|
}
|
||||||
"set" => {
|
"set" => {
|
||||||
let len: u64 = redis::cmd("SCARD").arg(key).query_async(con).await.unwrap_or(0);
|
let len: u64 = redis::cmd("SCARD").arg(key).query_async(con).await.unwrap_or(0);
|
||||||
@@ -2693,9 +2704,10 @@ where
|
|||||||
"zset" => {
|
"zset" => {
|
||||||
let len: u64 = redis::cmd("ZCARD").arg(key).query_async(con).await.unwrap_or(0);
|
let len: u64 = redis::cmd("ZCARD").arg(key).query_async(con).await.unwrap_or(0);
|
||||||
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
|
let end = (COLLECTION_PAGE_SIZE as i64) - 1;
|
||||||
let items = zrange_page_raw(con, key, 0, end, false).await?;
|
let mut items = zrange_page_raw(con, key, 0, end, false).await?;
|
||||||
let cursor = if len > COLLECTION_PAGE_SIZE as u64 { Some(COLLECTION_PAGE_SIZE as u64) } else { None };
|
items.truncate(budgeted_page_len(&items, COLLECTION_PAGE_SIZE, zset_item_page_bytes));
|
||||||
RedisValueData::Zset { items, total: len, scan_cursor: cursor }
|
let next = items.len() as u64;
|
||||||
|
RedisValueData::Zset { items, total: len, scan_cursor: (next < len).then_some(next) }
|
||||||
}
|
}
|
||||||
"hash" => {
|
"hash" => {
|
||||||
let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0);
|
let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0);
|
||||||
@@ -3924,11 +3936,10 @@ where
|
|||||||
let end = start + count as i64 - 1;
|
let end = start + count as i64 - 1;
|
||||||
let v: RedisRawValue =
|
let v: RedisRawValue =
|
||||||
redis::cmd("LRANGE").arg(key).arg(start).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
|
redis::cmd("LRANGE").arg(key).arg(start).arg(end).query_async(con).await.map_err(|e| e.to_string())?;
|
||||||
let next = cursor + count as u64;
|
let mut items = redis_list_items_from_raw(v, cursor);
|
||||||
Ok(RedisCollectionPage::List {
|
items.truncate(budgeted_page_len(&items, count, list_item_page_bytes));
|
||||||
items: redis_list_items_from_raw(v, cursor),
|
let next = cursor + items.len() as u64;
|
||||||
scan_cursor: (next < len).then_some(next),
|
Ok(RedisCollectionPage::List { items, scan_cursor: (next < len).then_some(next) })
|
||||||
})
|
|
||||||
}
|
}
|
||||||
"set" => {
|
"set" => {
|
||||||
let (next_cursor, items) = if let Some(query) = filter_query {
|
let (next_cursor, items) = if let Some(query) = filter_query {
|
||||||
@@ -3952,10 +3963,14 @@ where
|
|||||||
let start = cursor as i64;
|
let start = cursor as i64;
|
||||||
let end = start + count as i64;
|
let end = start + count as i64;
|
||||||
let mut items = zrange_page_raw(con, key, start, end, descending).await?;
|
let mut items = zrange_page_raw(con, key, start, end, descending).await?;
|
||||||
|
// `zrange_page_raw` reads one extra member to detect a further page; the byte budget
|
||||||
|
// can also end the page before `count`, which likewise leaves members behind.
|
||||||
let has_more = items.len() > count;
|
let has_more = items.len() > count;
|
||||||
items.truncate(count);
|
let page_len = budgeted_page_len(&items, count, zset_item_page_bytes);
|
||||||
let next = cursor + count as u64;
|
let ended_early = page_len < items.len().min(count);
|
||||||
Ok(RedisCollectionPage::Zset { items, scan_cursor: has_more.then_some(next) })
|
items.truncate(page_len);
|
||||||
|
let next = cursor + items.len() as u64;
|
||||||
|
Ok(RedisCollectionPage::Zset { items, scan_cursor: (has_more || ended_early).then_some(next) })
|
||||||
}
|
}
|
||||||
"hash" => {
|
"hash" => {
|
||||||
let (next_cursor, mut items) = if let Some(query) = filter_query {
|
let (next_cursor, mut items) = if let Some(query) = filter_query {
|
||||||
@@ -4150,7 +4165,8 @@ where
|
|||||||
} else {
|
} else {
|
||||||
hscan_page_raw(con, key, cursor, count, None).await?
|
hscan_page_raw(con, key, cursor, count, None).await?
|
||||||
};
|
};
|
||||||
let next_cursor = store_hash_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
|
let page_len = budgeted_page_len(&items, count, hash_item_page_bytes);
|
||||||
|
let next_cursor = store_hash_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await;
|
||||||
Ok((next_cursor, items))
|
Ok((next_cursor, items))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4174,7 +4190,8 @@ where
|
|||||||
} else {
|
} else {
|
||||||
sscan_page_raw(con, key, cursor, count).await?
|
sscan_page_raw(con, key, cursor, count).await?
|
||||||
};
|
};
|
||||||
let next_cursor = store_set_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
|
let page_len = budgeted_page_len(&items, count, set_item_page_bytes);
|
||||||
|
let next_cursor = store_set_overflow_page(key, &mut items, page_len, next_cursor, filter_query).await;
|
||||||
Ok((next_cursor, items))
|
Ok((next_cursor, items))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4191,7 +4208,9 @@ async fn take_hash_overflow_page(
|
|||||||
let mut sessions = collection_overflow_sessions().lock().await;
|
let mut sessions = collection_overflow_sessions().lock().await;
|
||||||
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Hash, filter_query, None)?;
|
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Hash, filter_query, None)?;
|
||||||
let page = match &mut session.items {
|
let page = match &mut session.items {
|
||||||
RedisCollectionOverflowItems::Hash(items) => take_vec_page(items, count),
|
RedisCollectionOverflowItems::Hash(items) => {
|
||||||
|
take_vec_page(items, budgeted_page_len(items, count, hash_item_page_bytes))
|
||||||
|
}
|
||||||
RedisCollectionOverflowItems::Set(_) => return None,
|
RedisCollectionOverflowItems::Set(_) => return None,
|
||||||
};
|
};
|
||||||
let has_buffered_items = match &session.items {
|
let has_buffered_items = match &session.items {
|
||||||
@@ -4218,7 +4237,9 @@ async fn take_set_overflow_page(
|
|||||||
let mut sessions = collection_overflow_sessions().lock().await;
|
let mut sessions = collection_overflow_sessions().lock().await;
|
||||||
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Set, filter_query, None)?;
|
let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Set, filter_query, None)?;
|
||||||
let page = match &mut session.items {
|
let page = match &mut session.items {
|
||||||
RedisCollectionOverflowItems::Set(items) => take_vec_page(items, count),
|
RedisCollectionOverflowItems::Set(items) => {
|
||||||
|
take_vec_page(items, budgeted_page_len(items, count, set_item_page_bytes))
|
||||||
|
}
|
||||||
RedisCollectionOverflowItems::Hash(_) => return None,
|
RedisCollectionOverflowItems::Hash(_) => return None,
|
||||||
};
|
};
|
||||||
let has_buffered_items = match &session.items {
|
let has_buffered_items = match &session.items {
|
||||||
@@ -4280,6 +4301,42 @@ async fn store_set_overflow_page(
|
|||||||
collection_overflow_sessions().lock().await.insert(session)
|
collection_overflow_sessions().lock().await.insert(session)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// How many leading items fit in one page, honouring both the item count and the byte budget.
|
||||||
|
///
|
||||||
|
/// Always returns at least one item so a member larger than the whole budget still loads (and
|
||||||
|
/// pagination cannot stall), and never more than `items.len()`.
|
||||||
|
fn budgeted_page_len<T>(items: &[T], count: usize, size_of: impl Fn(&T) -> usize) -> usize {
|
||||||
|
let limit = count.max(1).min(items.len());
|
||||||
|
let mut used = 0usize;
|
||||||
|
for (index, item) in items.iter().take(limit).enumerate() {
|
||||||
|
used = used.saturating_add(size_of(item));
|
||||||
|
if used > COLLECTION_PAGE_MAX_BYTES {
|
||||||
|
return index.max(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
limit
|
||||||
|
}
|
||||||
|
|
||||||
|
fn blob_page_bytes(blob: &RedisBlob) -> usize {
|
||||||
|
blob.raw_base64.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash_item_page_bytes(item: &RedisHashItem) -> usize {
|
||||||
|
blob_page_bytes(&item.field) + blob_page_bytes(&item.value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_item_page_bytes(item: &RedisSetItem) -> usize {
|
||||||
|
blob_page_bytes(&item.member)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_item_page_bytes(item: &RedisListItem) -> usize {
|
||||||
|
blob_page_bytes(&item.value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn zset_item_page_bytes(item: &RedisZsetItem) -> usize {
|
||||||
|
blob_page_bytes(&item.member) + item.score.len()
|
||||||
|
}
|
||||||
|
|
||||||
fn take_vec_page<T>(items: &mut Vec<T>, count: usize) -> Vec<T> {
|
fn take_vec_page<T>(items: &mut Vec<T>, count: usize) -> Vec<T> {
|
||||||
let page_size = items.len().min(count.max(1));
|
let page_size = items.len().min(count.max(1));
|
||||||
let overflow = items.split_off(page_size);
|
let overflow = items.split_off(page_size);
|
||||||
@@ -5006,6 +5063,49 @@ mod tests {
|
|||||||
assert_eq!(con.command_count("STRLEN"), 1);
|
assert_eq!(con.command_count("STRLEN"), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn collection_pages_stop_at_the_byte_budget_without_truncating_a_member() {
|
||||||
|
fn hash_items(count: usize, value_bytes: usize) -> Vec<RedisHashItem> {
|
||||||
|
(0..count)
|
||||||
|
.map(|index| RedisHashItem {
|
||||||
|
field: text_blob(&format!("f{index}")),
|
||||||
|
value: text_blob(&"v".repeat(value_bytes)),
|
||||||
|
field_ttl: None,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
fn page_bytes(items: &[RedisHashItem]) -> usize {
|
||||||
|
items.iter().map(super::hash_item_page_bytes).sum()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Members are sized as a fraction of the budget; base64 inflates them by 4/3 on the wire,
|
||||||
|
// so the assertions below are on the resulting page rather than on a hand-computed count.
|
||||||
|
for divisor in [2, 3, 8] {
|
||||||
|
let items = hash_items(200, super::COLLECTION_PAGE_MAX_BYTES / divisor);
|
||||||
|
let page_len = super::budgeted_page_len(&items, 200, super::hash_item_page_bytes);
|
||||||
|
assert!((1..200).contains(&page_len), "divisor {divisor}: page_len {page_len}");
|
||||||
|
assert!(
|
||||||
|
page_bytes(&items[..page_len]) <= super::COLLECTION_PAGE_MAX_BYTES,
|
||||||
|
"divisor {divisor} over budget"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
page_bytes(&items[..page_len + 1]) > super::COLLECTION_PAGE_MAX_BYTES,
|
||||||
|
"divisor {divisor}: page ended earlier than the budget required"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Small members still fill a whole page, and a short input is returned whole.
|
||||||
|
let small = hash_items(500, 1);
|
||||||
|
assert_eq!(super::budgeted_page_len(&small, 200, super::hash_item_page_bytes), 200);
|
||||||
|
assert_eq!(super::budgeted_page_len(&small[..5], 200, super::hash_item_page_bytes), 5);
|
||||||
|
|
||||||
|
// A single member larger than the whole budget still loads, so paging cannot stall.
|
||||||
|
let huge = hash_items(3, super::COLLECTION_PAGE_MAX_BYTES * 2);
|
||||||
|
assert_eq!(super::budgeted_page_len(&huge, 200, super::hash_item_page_bytes), 1);
|
||||||
|
|
||||||
|
assert_eq!(super::budgeted_page_len::<RedisHashItem>(&[], 200, super::hash_item_page_bytes), 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn hash_value_pages_compact_hscan_before_field_ttls() {
|
async fn hash_value_pages_compact_hscan_before_field_ttls() {
|
||||||
let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();
|
let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();
|
||||||
|
|||||||
Reference in new issue
Block a user