fix(redis): batch SCAN server-side to reduce fetch-all latency
This commit is contained in:
1 parent
08fb5bbf4d
commit
ab5bd1dd25
10 files changed
+204
-49
No files matched your search
@@ -200,13 +200,30 @@ async function fetchScanPage(): Promise<RedisScanResult> {
|
||||
return isValueSearchMode.value ? await api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all") : await api.redisScanKeys(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize);
|
||||
}
|
||||
|
||||
/// Batch-scan variant that performs multiple SCAN iterations server-side.
|
||||
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
|
||||
async function fetchScanBatchPage(maxIterations: number): Promise<RedisScanResult> {
|
||||
const pageSize = settingsStore.editorSettings.redisScanPageSize;
|
||||
// Value search cannot be batched because each key requires a GET.
|
||||
if (isValueSearchMode.value) {
|
||||
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
|
||||
}
|
||||
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations);
|
||||
}
|
||||
|
||||
function appendScanResult(result: RedisScanResult) {
|
||||
const existingKeys = new Set(flatKeys.value.map((key) => key.key_raw));
|
||||
const newKeys = result.keys.filter((key) => !existingKeys.has(key.key_raw));
|
||||
flatKeys.value = [...flatKeys.value, ...newKeys];
|
||||
scanCursor.value = result.cursor;
|
||||
hasMore.value = result.cursor !== 0;
|
||||
lastTotalKeys.value = result.total_keys;
|
||||
// DBSIZE is only called on the first batch page (cursor==0); subsequent
|
||||
// pages return total_keys=0. Preserve the previously-fetched total when
|
||||
// we get a zero from a continuation. A truly empty DB returns cursor==0
|
||||
// and keys==[] along with total_keys==0, which we do record.
|
||||
if (result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0)) {
|
||||
lastTotalKeys.value = result.total_keys;
|
||||
}
|
||||
|
||||
if (treeKeys.value.length === 0) {
|
||||
rebuildTree(isSearchMode.value);
|
||||
@@ -216,7 +233,7 @@ function appendScanResult(result: RedisScanResult) {
|
||||
|
||||
connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, {
|
||||
loaded: isSearchMode.value ? undefined : flatKeys.value.length,
|
||||
total: result.total_keys,
|
||||
total: result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0) ? result.total_keys : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -236,15 +253,18 @@ async function streamValueSearch(requestId: number) {
|
||||
|
||||
async function fillInitialKeyBatch(requestId: number) {
|
||||
const targetCount = Math.max(1, settingsStore.editorSettings.redisScanPageSize);
|
||||
let rounds = 0;
|
||||
while (requestId === searchRequestId && searchMode.value === "key" && hasMore.value && flatKeys.value.length < targetCount) {
|
||||
const beforeCount = flatKeys.value.length;
|
||||
const applied = await scanNextPage(requestId);
|
||||
if (!applied) return;
|
||||
rounds += 1;
|
||||
if (flatKeys.value.length >= targetCount) return;
|
||||
if (rounds >= 12 && flatKeys.value.length === beforeCount) return;
|
||||
if (rounds >= 24) return;
|
||||
// Use server-side batching to fill the initial view quickly.
|
||||
// Each batch iteration does one SCAN round-trip; 5 iterations with
|
||||
// COUNT=1000 should return enough keys for most cases.
|
||||
const maxIter = Math.max(1, Math.ceil(targetCount / Math.max(1, settingsStore.editorSettings.redisScanPageSize)));
|
||||
const result = await fetchScanBatchPage(Math.min(maxIter, 8));
|
||||
if (requestId !== searchRequestId) return;
|
||||
appendScanResult(result);
|
||||
// If we still need more keys, do one more batch
|
||||
if (flatKeys.value.length < targetCount && hasMore.value && requestId === searchRequestId) {
|
||||
const result2 = await fetchScanBatchPage(Math.min(maxIter, 8));
|
||||
if (requestId !== searchRequestId) return;
|
||||
appendScanResult(result2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,14 +310,22 @@ async function loadMore() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch-all with server-side multi-SCAN batching.
|
||||
///
|
||||
/// Each call performs up to 15 SCAN→TYPE cycles server-side (~0.5s per
|
||||
/// batch at COUNT=1000). This keeps the UI responsive with frequent progress
|
||||
/// updates while still avoiding the per-page overhead of single-SCAN calls.
|
||||
const FETCH_ALL_BATCH_ITERATIONS = 15;
|
||||
|
||||
async function fetchAll() {
|
||||
if (!hasMore.value || isFetchingAll.value) return;
|
||||
const requestId = searchRequestId;
|
||||
isFetchingAll.value = true;
|
||||
try {
|
||||
while (requestId === searchRequestId && isFetchingAll.value && hasMore.value) {
|
||||
const applied = await scanNextPage(requestId);
|
||||
if (!applied) break;
|
||||
const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS);
|
||||
if (requestId !== searchRequestId) break;
|
||||
appendScanResult(result);
|
||||
}
|
||||
} finally {
|
||||
if (requestId === searchRequestId) {
|
||||
|
||||
Reference in new issue
Block a user