mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-21 06:00:39 +00:00
fix(redis): continue sparse key scans
This commit is contained in:
@@ -216,14 +216,37 @@ function mergeTree(newKeys: RedisKeyInfo[]) {
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchScanPage(): Promise<RedisScanResult> {
|
||||
async function fetchScanPage(requestId = searchRequestId): Promise<RedisScanResult> {
|
||||
const pageSize = redisScanPageSize.value;
|
||||
// Redis SCAN may return an empty page with a non-zero cursor; batch a few
|
||||
// key-search pages so sparse MATCH patterns do not look empty immediately.
|
||||
const keySearchIterations = 8;
|
||||
return isValueSearchMode.value
|
||||
? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all")
|
||||
: await api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, keySearchIterations, false);
|
||||
if (isValueSearchMode.value) {
|
||||
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
|
||||
}
|
||||
|
||||
// Keep each backend call small so a changed search can cancel between calls.
|
||||
// The total COUNT budget bounds Redis work while giving sparse MATCH patterns
|
||||
// substantially more coverage than a fixed number of SCAN calls.
|
||||
const scanCountBudget = 50_000;
|
||||
const iterationsPerCall = 8;
|
||||
const maxIterations = Math.max(1, Math.ceil(scanCountBudget / Math.max(1, pageSize)));
|
||||
let completedIterations = 0;
|
||||
let cursor = scanCursor.value;
|
||||
let totalKeys = 0;
|
||||
|
||||
while (completedIterations < maxIterations) {
|
||||
if (requestId !== searchRequestId) {
|
||||
return { cursor, keys: [], total_keys: totalKeys };
|
||||
}
|
||||
const iterations = Math.min(iterationsPerCall, maxIterations - completedIterations);
|
||||
const result = await api.redisScanKeysBatch(props.connectionId, props.db, cursor, effectivePattern.value, pageSize, iterations, false);
|
||||
if (totalKeys === 0) totalKeys = result.total_keys;
|
||||
if (result.keys.length > 0 || result.cursor === 0) {
|
||||
return { ...result, total_keys: totalKeys };
|
||||
}
|
||||
cursor = result.cursor;
|
||||
completedIterations += iterations;
|
||||
}
|
||||
|
||||
return { cursor, keys: [], total_keys: totalKeys };
|
||||
}
|
||||
|
||||
/// Batch-scan variant that performs multiple SCAN iterations server-side.
|
||||
@@ -275,7 +298,7 @@ function appendScanResult(result: RedisScanResult, options: { updateTree?: boole
|
||||
}
|
||||
|
||||
async function scanNextPage(requestId = searchRequestId): Promise<boolean> {
|
||||
const result = await fetchScanPage();
|
||||
const result = await fetchScanPage(requestId);
|
||||
if (requestId !== searchRequestId) return false;
|
||||
appendScanResult(result);
|
||||
return true;
|
||||
|
||||
@@ -1687,7 +1687,7 @@ where
|
||||
|
||||
let mut current_cursor = cursor;
|
||||
|
||||
for _ in 0..iterations {
|
||||
for iteration in 0..iterations {
|
||||
let raw: RedisRawValue = redis::cmd("SCAN")
|
||||
.arg(current_cursor)
|
||||
.arg("MATCH")
|
||||
@@ -1737,11 +1737,26 @@ where
|
||||
return Ok(RedisScanResult { cursor: 0, keys: all_keys, total_keys });
|
||||
}
|
||||
current_cursor = next_cursor;
|
||||
|
||||
// MATCH may yield sparse or empty batches. Keep scanning within the
|
||||
// caller's bounded budget, but stop once this result page is full.
|
||||
if !should_continue_key_scan(all_keys.len(), count, iteration + 1, iterations) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(RedisScanResult { cursor: current_cursor, keys: all_keys, total_keys })
|
||||
}
|
||||
|
||||
fn should_continue_key_scan(
|
||||
matched_keys: usize,
|
||||
target_keys: usize,
|
||||
completed_iterations: usize,
|
||||
max_iterations: usize,
|
||||
) -> bool {
|
||||
matched_keys < target_keys.max(1) && completed_iterations < max_iterations.max(1)
|
||||
}
|
||||
|
||||
pub async fn scan_values_page<C>(
|
||||
con: &mut C,
|
||||
cursor: u64,
|
||||
@@ -2884,6 +2899,23 @@ mod tests {
|
||||
assert!(!redis_key_matches_query("User:42:Profile", "User:42:Profile", "order"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_key_scan_continues_after_empty_iterations() {
|
||||
assert!(super::should_continue_key_scan(0, 1000, 8, 50));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_scan_stops_when_result_page_is_full() {
|
||||
assert!(!super::should_continue_key_scan(1000, 1000, 3, 50));
|
||||
assert!(!super::should_continue_key_scan(1200, 1000, 3, 50));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_scan_respects_iteration_budget() {
|
||||
assert!(!super::should_continue_key_scan(0, 1000, 50, 50));
|
||||
assert!(!super::should_continue_key_scan(0, 0, 1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_hash_field_name_in_value_search() {
|
||||
let hash_value = hash_value(&[("name", "Alice"), ("email", "alice@example.com")]);
|
||||
|
||||
Reference in New Issue
Block a user