fix(redis): speed up large key scans

This commit is contained in:
t8y2 committed 2026-06-23 15:14:24 +08:00
1 parent ff79ec05ad
commit fc9f97b6b5
9 files changed
+154 -52

No files matched your search

+48 -13
View File
@@ -760,6 +760,16 @@ pub async fn scan_cluster_keys_page(
cursor: u64,
pattern: &str,
count: usize,
) -> Result<RedisScanResult, String> {
scan_cluster_keys_page_with_options(pool, cursor, pattern, count, true).await
}
pub async fn scan_cluster_keys_page_with_options(
pool: &RedisClusterPool,
cursor: u64,
pattern: &str,
count: usize,
include_types: bool,
) -> Result<RedisScanResult, String> {
let master_nodes = cluster_master_nodes(pool).await?;
if master_nodes.is_empty() {
@@ -776,7 +786,7 @@ pub async fn scan_cluster_keys_page(
let endpoint = &master_nodes[index];
let mut con = connect_cluster_node(pool, endpoint).await?;
let current_cursor = if index == node_index { node_cursor } else { 0 };
let result = scan_keys_page(&mut con, current_cursor, pattern, count).await?;
let result = scan_keys_page_with_options(&mut con, current_cursor, pattern, count, include_types).await?;
if !result.keys.is_empty() {
let next_cursor = if result.cursor != 0 {
encode_cluster_cursor(index, result.cursor)?
@@ -1434,13 +1444,26 @@ pub async fn scan_keys_page<C>(con: &mut C, cursor: u64, pattern: &str, count: u
where
C: ConnectionLike + Send + Sync + Unpin,
{
scan_keys_batch(con, cursor, pattern, count, 1).await
scan_keys_batch(con, cursor, pattern, count, 1, true).await
}
pub async fn scan_keys_page_with_options<C>(
con: &mut C,
cursor: u64,
pattern: &str,
count: usize,
include_types: bool,
) -> Result<RedisScanResult, String>
where
C: ConnectionLike + Send + Sync + Unpin,
{
scan_keys_batch(con, cursor, pattern, count, 1, include_types).await
}
/// Batch-scan keys with server-side multi-SCAN support.
///
/// Performs up to `max_iterations` SCAN→TYPE cycles in a single call,
/// dramatically reducing frontend↔backend roundtrips when fetching many keys.
/// Performs up to `max_iterations` SCAN cycles in a single call. TYPE metadata
/// is optional so large key-name searches can avoid extra Redis work.
/// DBSIZE is only called on the first iteration (cursor == 0).
pub async fn scan_keys_batch<C>(
con: &mut C,
@@ -1448,6 +1471,7 @@ pub async fn scan_keys_batch<C>(
pattern: &str,
count: usize,
max_iterations: usize,
include_types: bool,
) -> Result<RedisScanResult, String>
where
C: ConnectionLike + Send + Sync + Unpin,
@@ -1472,23 +1496,34 @@ where
let (next_cursor, keys) = parse_scan_keys(raw)?;
if !keys.is_empty() {
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
let key_types: Vec<String> = pipe.query_async(con).await.unwrap_or_default();
let key_types: Vec<String> = if include_types {
let mut pipe = redis::pipe();
for key in &keys {
pipe.cmd("TYPE").arg(key);
}
pipe.query_async(con).await.unwrap_or_default()
} else {
Vec::new()
};
for (index, key) in keys.iter().enumerate() {
let key_type = key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string());
let key_type = if include_types {
key_types.get(index).cloned().unwrap_or_else(|| "unknown".to_string())
} else {
"unknown".to_string()
};
let value_preview = if include_types {
redis_key_value_preview(key_types.get(index).map(String::as_str).unwrap_or("unknown"))
} else {
String::new()
};
all_keys.push(RedisKeyInfo {
key_display: redis_key_bytes_to_display(key),
key_raw: redis_key_bytes_to_raw(key),
key_type,
ttl: -2,
size: 0,
value_preview: redis_key_value_preview(
key_types.get(index).map(String::as_str).unwrap_or("unknown"),
),
value_preview,
});
}
}