fix(redis): batch SCAN server-side to reduce fetch-all latency

This commit is contained in:
t8y2 committed 2026-06-15 15:01:13 +08:00
1 parent 08fb5bbf4d
commit ab5bd1dd25
10 files changed
+204 -49

No files matched your search

@@ -200,13 +200,30 @@ async function fetchScanPage(): Promise<RedisScanResult> {
return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize);
}
/// Batch-scan variant that performs multiple SCAN iterations server-side.
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
async function fetchScanBatchPage(maxIterations: number): Promise<RedisScanResult> {
const pageSize = settingsStore.editorSettings.redisScanPageSize;
// Value search cannot be batched because each key requires a GET.
if (isValueSearchMode.value) {
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
}
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations);
}
function appendScanResult(result: RedisScanResult) {
const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw));
const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw));
flatKeys.value = [...flatKeys.value, ...newKeys];
scanCursor.value = result.cursor;
hasMore.value = result.cursor !== 0;
lastTotalKeys.value = result.total_keys;
// DBSIZE is only called on the first batch page (cursor==0); subsequent
// pages return total_keys=0. Preserve the previously-fetched total when
// we get a zero from a continuation. A truly empty DB returns cursor==0
// and keys==[] along with total_keys==0, which we do record.
if (result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0)) {
lastTotalKeys.value = result.total_keys;
}
if (treeKeys.value.length === 0) {
rebuildTree(isSearchMode.value);
@@ -216,7 +233,7 @@ function appendScanResult(result: RedisScanResult) {
connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, {
loaded: isSearchMode.value ? undefined : flatKeys.value.length,
total: result.total_keys,
total: result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0) ? result.total_keys : undefined,
});
}
@@ -236,15 +253,18 @@ async function streamValueSearch(requestId: number) {
async function fillInitialKeyBatch(requestId: number) {
const targetCount = Math.max(1, settingsStore.editorSettings.redisScanPageSize);
let rounds = 0;
while (requestId === searchRequestId && searchMode.value === "key" && hasMore.value && flatKeys.value.length < targetCount) {
const beforeCount = flatKeys.value.length;
const applied = await scanNextPage(requestId);
if (!applied) return;
rounds += 1;
if (flatKeys.value.length >= targetCount) return;
if (rounds >= 12 && flatKeys.value.length === beforeCount) return;
if (rounds >= 24) return;
// Use server-side batching to fill the initial view quickly.
// Each batch iteration does one SCAN round-trip; 5 iterations with
// COUNT=1000 should return enough keys for most cases.
const maxIter = Math.max(1, Math.ceil(targetCount / Math.max(1, settingsStore.editorSettings.redisScanPageSize)));
const result = await fetchScanBatchPage(Math.min(maxIter, 8));
if (requestId !== searchRequestId) return;
appendScanResult(result);
// If we still need more keys, do one more batch
if (flatKeys.value.length < targetCount && hasMore.value && requestId === searchRequestId) {
const result2 = await fetchScanBatchPage(Math.min(maxIter, 8));
if (requestId !== searchRequestId) return;
appendScanResult(result2);
}
}
@@ -290,14 +310,22 @@ async function loadMore() {
}
}
/// Fetch-all with server-side multi-SCAN batching.
///
/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per
/// batch at COUNT=1000). This keeps the UI responsive with frequent progress
/// updates while still avoiding the per-page overhead of single-SCAN calls.
const FETCH_ALL_BATCH_ITERATIONS = 15;
async function fetchAll() {
if (!hasMore.value || isFetchingAll.value) return;
const requestId = searchRequestId;
isFetchingAll.value = true;
try {
while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) {
const applied = await scanNextPage(requestId);
if (!applied) break;
const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS);
if (requestId !== searchRequestId) break;
appendScanResult(result);
}
} finally {
if (requestId === searchRequestId) {