feat(redis): fuzzy key search keyspace-aware scan budget
This commit is contained in:
3 files changed
+39
-5
No files matched your search
@@ -51,7 +51,7 @@ import { isCancelSearchShortcut } from "@/lib/editor/keyboardShortcuts";
|
|||||||
import { copyToClipboard } from "@/lib/common/clipboard";
|
import { copyToClipboard } from "@/lib/common/clipboard";
|
||||||
import { useEditorFontFamilyStyle } from "@/composables/useEditorFontFamilyStyle";
|
import { useEditorFontFamilyStyle } from "@/composables/useEditorFontFamilyStyle";
|
||||||
import { useToast } from "@/composables/useToast";
|
import { useToast } from "@/composables/useToast";
|
||||||
import { redisKeySearchPattern, redisGroupSubtreePattern } from "@/lib/redis/redisKeyPattern";
|
import { redisFuzzySearchScanBudget, redisKeySearchPattern, redisGroupSubtreePattern } from "@/lib/redis/redisKeyPattern";
|
||||||
import { filterRedisKeyTemplates, resolveRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
|
import { filterRedisKeyTemplates, resolveRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
|
||||||
import { REDIS_SCAN_PAGE_SIZE_DEFAULT } from "@/lib/redis/redisKeyPattern";
|
import { REDIS_SCAN_PAGE_SIZE_DEFAULT } from "@/lib/redis/redisKeyPattern";
|
||||||
import { chunkRedisKeyRaws, collectUniqueRedisKeys } from "@/lib/redis/redisKeyBatch";
|
import { chunkRedisKeyRaws, collectUniqueRedisKeys } from "@/lib/redis/redisKeyBatch";
|
||||||
@@ -651,14 +651,15 @@ async function fetchScanPage(requestId = searchRequestId, operationId?: number,
|
|||||||
// complete. Bound every user-triggered page to a cumulative COUNT budget,
|
// complete. Bound every user-triggered page to a cumulative COUNT budget,
|
||||||
// preserve the returned cursor, and let the existing "Load more" action
|
// preserve the returned cursor, and let the existing "Load more" action
|
||||||
// continue sparse searches without turning one request into a full scan.
|
// continue sparse searches without turning one request into a full scan.
|
||||||
const scanCountBudget = 50_000;
|
const isFuzzyKeySearch = searchMode.value === "key" && fuzzyKeySearch.value;
|
||||||
|
let scanCountBudget = redisFuzzySearchScanBudget(0);
|
||||||
const iterationsPerCall = 8;
|
const iterationsPerCall = 8;
|
||||||
const perCallMaxIterations = Math.max(1, Math.ceil(scanCountBudget / Math.max(1, pageSize)));
|
const perCallMaxIterations = Math.max(1, Math.ceil(scanCountBudget / Math.max(1, pageSize)));
|
||||||
// When part of the automatic-fill chain, also cap this call to whatever is
|
// When part of the automatic-fill chain, also cap this call to whatever is
|
||||||
// left of the shared iteration budget — this is what actually bounds the
|
// left of the shared iteration budget — this is what actually bounds the
|
||||||
// total backend work across every page that chain triggers, not just this
|
// total backend work across every page that chain triggers, not just this
|
||||||
// one call's own per-call cap.
|
// one call's own per-call cap.
|
||||||
const maxIterations = iterationBudget ? Math.max(0, Math.min(perCallMaxIterations, iterationBudget.remaining)) : perCallMaxIterations;
|
let maxIterations = iterationBudget ? Math.max(0, Math.min(perCallMaxIterations, iterationBudget.remaining)) : perCallMaxIterations;
|
||||||
let completedIterations = 0;
|
let completedIterations = 0;
|
||||||
let cursor = scanCursor.value;
|
let cursor = scanCursor.value;
|
||||||
let totalKeys = 0;
|
let totalKeys = 0;
|
||||||
@@ -670,6 +671,12 @@ async function fetchScanPage(requestId = searchRequestId, operationId?: number,
|
|||||||
const result = await api.redisScanKeysBatch(props.connectionId, props.db, cursor, effectivePattern.value, pageSize, iterations, true);
|
const result = await api.redisScanKeysBatch(props.connectionId, props.db, cursor, effectivePattern.value, pageSize, iterations, true);
|
||||||
completedIterations += iterations;
|
completedIterations += iterations;
|
||||||
if (totalKeys === 0) totalKeys = result.total_keys;
|
if (totalKeys === 0) totalKeys = result.total_keys;
|
||||||
|
if (isFuzzyKeySearch && totalKeys > 0) {
|
||||||
|
scanCountBudget = redisFuzzySearchScanBudget(totalKeys);
|
||||||
|
const fuzzyMaxIterations = Math.ceil(scanCountBudget / Math.max(1, pageSize));
|
||||||
|
const availableIterations = iterationBudget ? completedIterations + iterationBudget.remaining : fuzzyMaxIterations;
|
||||||
|
maxIterations = Math.min(fuzzyMaxIterations, Math.max(maxIterations, availableIterations));
|
||||||
|
}
|
||||||
if (result.keys.length > 0 || result.cursor === 0) {
|
if (result.keys.length > 0 || result.cursor === 0) {
|
||||||
return { ...result, total_keys: totalKeys };
|
return { ...result, total_keys: totalKeys };
|
||||||
}
|
}
|
||||||
@@ -782,7 +789,8 @@ async function loadKeys() {
|
|||||||
hasMore.value = false;
|
hasMore.value = false;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const applied = await scanNextPage(requestId);
|
const initialScanBudget = isFuzzyKeySearch.value ? autoLoadBudget : undefined;
|
||||||
|
const applied = await scanNextPage(requestId, undefined, initialScanBudget);
|
||||||
if (applied && isValueSearchMode.value) {
|
if (applied && isValueSearchMode.value) {
|
||||||
await streamValueSearch(requestId);
|
await streamValueSearch(requestId);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { redisFuzzySearchScanBudget, REDIS_FUZZY_SEARCH_SCAN_COUNT_MAX, REDIS_KEY_SEARCH_SCAN_COUNT_BUDGET, REDIS_SCAN_PAGE_SIZE_DEFAULT, REDIS_SCAN_PAGE_SIZE_OPTIONS } from "@/lib/redis/redisKeyPattern";
|
||||||
|
|
||||||
|
describe("Redis key search settings", () => {
|
||||||
|
it("keeps the default scan count conservative for initial loading", () => {
|
||||||
|
expect(REDIS_SCAN_PAGE_SIZE_DEFAULT).toBe(1000);
|
||||||
|
expect(REDIS_SCAN_PAGE_SIZE_OPTIONS).toContain(2000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("covers the reported keyspace while retaining a large-instance cap", () => {
|
||||||
|
expect(redisFuzzySearchScanBudget(0)).toBe(REDIS_KEY_SEARCH_SCAN_COUNT_BUDGET);
|
||||||
|
expect(redisFuzzySearchScanBudget(378543)).toBe(378543);
|
||||||
|
expect(redisFuzzySearchScanBudget(REDIS_FUZZY_SEARCH_SCAN_COUNT_MAX + 1)).toBe(REDIS_FUZZY_SEARCH_SCAN_COUNT_MAX);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -21,9 +21,20 @@ export function redisGroupSubtreePattern(pathSegments: readonly string[], separa
|
|||||||
return `${prefix}${separator}*`;
|
return `${prefix}${separator}*`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fuzzy key search has to walk the Redis keyspace because MATCH has no index.
|
||||||
|
// Start conservatively, then use DBSIZE to cover ordinary databases while
|
||||||
|
// retaining a hard upper bound for unusually large instances.
|
||||||
|
export const REDIS_KEY_SEARCH_SCAN_COUNT_BUDGET = 50_000;
|
||||||
|
export const REDIS_FUZZY_SEARCH_SCAN_COUNT_MAX = 1_000_000;
|
||||||
|
|
||||||
|
export function redisFuzzySearchScanBudget(totalKeys: number): number {
|
||||||
|
const normalizedTotalKeys = Number.isFinite(totalKeys) ? Math.max(0, Math.floor(totalKeys)) : 0;
|
||||||
|
return Math.min(Math.max(REDIS_KEY_SEARCH_SCAN_COUNT_BUDGET, normalizedTotalKeys), REDIS_FUZZY_SEARCH_SCAN_COUNT_MAX);
|
||||||
|
}
|
||||||
|
|
||||||
// Redis scan page size (COUNT parameter per SCAN round-trip) — shared defaults
|
// Redis scan page size (COUNT parameter per SCAN round-trip) — shared defaults
|
||||||
// and validation bounds used by the connection form and key browser.
|
// and validation bounds used by the connection form and key browser.
|
||||||
export const REDIS_SCAN_PAGE_SIZE_DEFAULT = 1000;
|
export const REDIS_SCAN_PAGE_SIZE_DEFAULT = 1000;
|
||||||
export const REDIS_SCAN_PAGE_SIZE_MIN = 200;
|
export const REDIS_SCAN_PAGE_SIZE_MIN = 200;
|
||||||
export const REDIS_SCAN_PAGE_SIZE_MAX = 10_000;
|
export const REDIS_SCAN_PAGE_SIZE_MAX = 10_000;
|
||||||
export const REDIS_SCAN_PAGE_SIZE_OPTIONS = [200, 1000, 5000, 10_000] as const;
|
export const REDIS_SCAN_PAGE_SIZE_OPTIONS = [200, 1000, 2000, 5000, 10_000] as const;
|
||||||
Reference in new issue
Block a user