From 6ac2b685538eca3fabf394ddc6728cd63c34b71b Mon Sep 17 00:00:00 2001 From: Guoyu Su Date: Sat, 5 Sep 2026 00:40:52 +0800 Subject: [PATCH] perf(redis): keep fetch-all finalization responsive --- .../redis/RedisKeyBrowser.expiry.spec.ts | 213 +++++++++- .../src/components/redis/RedisKeyBrowser.vue | 260 +++++++++++- apps/desktop/src/lib/redis/redisKeyTree.ts | 381 +++++++++++++++--- packages/app-tests/redisKeyTree.test.ts | 70 ++++ 4 files changed, 843 insertions(+), 81 deletions(-) diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.expiry.spec.ts b/apps/desktop/src/components/redis/RedisKeyBrowser.expiry.spec.ts index c0ee00ea51..6b57de8358 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.expiry.spec.ts +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.expiry.spec.ts @@ -25,8 +25,11 @@ const mocks = vi.hoisted(() => ({ redisExecuteCommand: vi.fn(), saveHistory: vi.fn(), canBuildRedisFuzzyTree: vi.fn((loadedKeyCount: number) => loadedKeyCount <= 200_000), + buildRedisKeySnapshotCooperatively: vi.fn(), createRedisKeyTreeIndex: vi.fn(), flattenVisibleRedisKeyTree: vi.fn(), + scrollerUpdateVisibleItems: vi.fn(), + scrollerItems: [] as unknown[][], toast: vi.fn(), updateRedisDbKeyStats: vi.fn(), listRedisCompletionCommandDocs: vi.fn(), @@ -61,9 +64,11 @@ vi.mock("@/lib/redis/redisKeyTree", async (importOriginal) => { const actual = await importOriginal(); mocks.createRedisKeyTreeIndex.mockImplementation(actual.createRedisKeyTreeIndex); mocks.flattenVisibleRedisKeyTree.mockImplementation(actual.flattenVisibleRedisKeyTree); + mocks.buildRedisKeySnapshotCooperatively.mockImplementation(actual.buildRedisKeySnapshotCooperatively); return { ...actual, canBuildRedisFuzzyTree: mocks.canBuildRedisFuzzyTree, + buildRedisKeySnapshotCooperatively: mocks.buildRedisKeySnapshotCooperatively, createRedisKeyTreeIndex: mocks.createRedisKeyTreeIndex, flattenVisibleRedisKeyTree: mocks.flattenVisibleRedisKeyTree, }; @@ -330,21 +335,36 @@ vi.mock("./RedisSlowlogPanel.vue", async () => { }); vi.mock("vue-virtual-scroller", async () => { - const { defineComponent, h } = await import("vue"); + const { defineComponent, h, shallowRef, watch } = await import("vue"); return { RecycleScroller: defineComponent({ inheritAttrs: false, props: { items: { type: Array, default: () => [] } }, - setup(props, { attrs, slots }) { + setup(props, { attrs, slots, expose }) { // Mirror the real scroller: interaction tests should render a viewport, // not every row in a deliberately large result set. const visibleItemCount = 50; - return () => - h( + const renderedItems = shallowRef([]); + const bindVisibleItems = () => { + renderedItems.value = props.items.slice(0, visibleItemCount); + }; + watch(() => props.items, bindVisibleItems, { immediate: true }); + expose({ + updateVisibleItems: (itemsChanged?: boolean) => { + mocks.scrollerUpdateVisibleItems(itemsChanged); + // The real scroller keeps a keyed view pool when false. Rebind only + // when callers identify in-place item changes explicitly. + if (itemsChanged) bindVisibleItems(); + }, + }); + return () => { + mocks.scrollerItems.push(props.items as unknown[]); + return h( "div", attrs, - props.items.slice(0, visibleItemCount).map((item) => slots.default?.({ item })), + renderedItems.value.map((item) => slots.default?.({ item })), ); + }; }, }), }; @@ -409,6 +429,7 @@ function resetApiMocks() { mocks.queryResultMaxRowsEnabled = true; mocks.queryResultMaxRows = 5000; mocks.loadedTtl = -1; + mocks.scrollerItems.length = 0; mocks.redisScanKeysBatch.mockResolvedValue({ cursor: 0, keys: [], total_keys: 0 }); mocks.redisGetValue.mockImplementation((_connectionId: string, _db: number, keyRaw: string) => Promise.resolve(redisValue(keyRaw))); mocks.redisSetString.mockResolvedValue(undefined); @@ -1773,6 +1794,188 @@ describe("RedisKeyBrowser fuzzy key hierarchy", () => { }); describe("RedisKeyBrowser interrupted Fetch All", () => { + it("publishes Fetch All rows through the existing scroller array and explicitly refreshes its viewport", async () => { + const initial = { key_display: "initial", key_raw: "aW5pdGlhbA==", key_type: "string", ttl: -1 }; + const buffered = { key_display: "buffered", key_raw: "YnVmZmVyZWQ=", key_type: "string", ttl: -1 }; + mocks.redisScanKeysBatch.mockImplementation((_connectionId: string, _db: number, cursor: number) => Promise.resolve(cursor === 0 ? { cursor: 1, keys: [initial], total_keys: 2 } : { cursor: 0, keys: [buffered], total_keys: 0 })); + mountBrowser(); + await settle(); + const itemsBeforeFetchAll = mocks.scrollerItems[mocks.scrollerItems.length - 1]; + + clickButtonWithText("redis.fetchAllKeys"); + await vi.waitFor(() => expect(document.body.textContent).toContain("buffered")); + + expect(mocks.scrollerItems[mocks.scrollerItems.length - 1]).toBe(itemsBeforeFetchAll); + expect(mocks.scrollerUpdateVisibleItems).toHaveBeenCalledWith(true); + expect(document.body.textContent).toContain("initial"); + expect(document.body.textContent).not.toContain("redis.stopFetchAll"); + }); + + it("rebinds rendered row labels after multi-chunk in-place publication", async () => { + const initial = { key_display: "initial", key_raw: "aW5pdGlhbA==", key_type: "string", ttl: -1 }; + const buffered = { key_display: "buffered", key_raw: "YnVmZmVyZWQ=", key_type: "string", ttl: -1 }; + mocks.redisScanKeysBatch.mockImplementation((_connectionId: string, _db: number, cursor: number) => Promise.resolve(cursor === 0 ? { cursor: 1, keys: [initial], total_keys: 2 } : { cursor: 0, keys: [buffered], total_keys: 0 })); + mountBrowser(); + await settle(); + + const visibleRows = Array.from({ length: 25_001 }, (_, index) => { + const label = `next-${String(index).padStart(5, "0")}`; + return { + id: `leaf:0:${label}`, + depth: 0, + node: { + kind: "leaf" as const, + id: `leaf:0:${label}`, + label, + fullKeyDisplay: label, + keyRaw: label, + db: 0, + keyType: "string", + ttl: -1, + size: 0, + valuePreview: "", + pathSegments: [label], + }, + }; + }); + mocks.buildRedisKeySnapshotCooperatively.mockResolvedValueOnce({ + flatKeys: [initial, buffered], + flatKeyByRaw: new Map([ + [initial.key_raw, initial], + [buffered.key_raw, buffered], + ]), + loadedKeyRaws: new Set([initial.key_raw, buffered.key_raw]), + filteredKeyCount: 2, + treeIndex: null, + expandedGroupIds: new Set(), + visibleRows, + }); + + clickButtonWithText("redis.fetchAllKeys"); + await vi.waitFor(() => expect(document.body.textContent).toContain("next-00000")); + await vi.waitFor(() => expect(document.body.textContent).not.toContain("redis.stopFetchAll")); + + expect(mocks.scrollerUpdateVisibleItems.mock.calls.length).toBeGreaterThanOrEqual(2); + expect(mocks.scrollerUpdateVisibleItems.mock.calls.every(([itemsChanged]) => itemsChanged === true)).toBe(true); + expect(document.body.textContent).toContain("next-00049"); + expect(document.body.textContent).not.toContain("initial"); + }); + + it("keeps stable Fetch All rows for metadata-only detail updates and exits them for no-expiry membership changes", async () => { + const initial = { key_display: "initial", key_raw: "aW5pdGlhbA==", key_type: "string", ttl: -1 }; + const buffered = { key_display: "buffered", key_raw: "YnVmZmVyZWQ=", key_type: "string", ttl: -1 }; + mocks.redisScanKeysBatch.mockImplementation((_connectionId: string, _db: number, cursor: number) => Promise.resolve(cursor === 0 ? { cursor: 1, keys: [initial], total_keys: 2 } : { cursor: 0, keys: [buffered], total_keys: 0 })); + mountBrowser(); + await settle(); + requiredElement("[data-redis-no-expiry-filter]").click(); + await settle(); + + clickButtonWithText("redis.fetchAllKeys"); + await vi.waitFor(() => expect(document.querySelector(`[data-redis-leaf="${buffered.key_raw}"]`)).not.toBeNull()); + const stableItems = mocks.scrollerItems[mocks.scrollerItems.length - 1]; + requiredElement(`[data-redis-leaf="${initial.key_raw}"]`).closest(".group")?.click(); + await settle(); + + mocks.loadedTtl = -1; + requiredElement("[data-test-emit-key-loaded]").click(); + await settle(); + expect(mocks.scrollerItems[mocks.scrollerItems.length - 1]).toBe(stableItems); + + mocks.loadedTtl = 60; + requiredElement("[data-test-emit-key-loaded]").click(); + await settle(); + expect(mocks.scrollerItems[mocks.scrollerItems.length - 1]).not.toBe(stableItems); + expect(document.querySelector(`[data-redis-leaf="${initial.key_raw}"]`)).toBeNull(); + expect(document.querySelector(`[data-redis-leaf="${buffered.key_raw}"]`)).not.toBeNull(); + }); + + it("keeps the previously published rows when Fetch All is stopped during cooperative preparation", async () => { + const initial = { key_display: "initial", key_raw: "aW5pdGlhbA==", key_type: "string", ttl: -1 }; + const buffered = { key_display: "buffered", key_raw: "YnVmZmVyZWQ=", key_type: "string", ttl: -1 }; + mocks.redisScanKeysBatch.mockImplementation((_connectionId: string, _db: number, cursor: number) => Promise.resolve(cursor === 0 ? { cursor: 1, keys: [initial], total_keys: 2 } : { cursor: 0, keys: [buffered], total_keys: 0 })); + const buildStarted = deferred(); + const releaseBuild = deferred(); + const buildSnapshot = mocks.buildRedisKeySnapshotCooperatively.getMockImplementation(); + expect(buildSnapshot).toBeDefined(); + mocks.buildRedisKeySnapshotCooperatively.mockImplementationOnce(async (...args) => { + buildStarted.resolve(); + await releaseBuild.promise; + return buildSnapshot!(...args); + }); + mountBrowser(); + await settle(); + + clickButtonWithText("redis.fetchAllKeys"); + await buildStarted.promise; + clickButtonWithText("redis.stopFetchAll"); + releaseBuild.resolve(); + await settle(); + + expect(document.body.textContent).toContain("initial"); + expect(document.body.textContent).not.toContain("buffered"); + expect(document.body.textContent).toContain("redis.fetchAllKeys"); + expect(mocks.scrollerUpdateVisibleItems).not.toHaveBeenCalled(); + }); + + it("rolls back a partially published stable row buffer when Fetch All is stopped", async () => { + const initial = { key_display: "initial", key_raw: "aW5pdGlhbA==", key_type: "string", ttl: -1 }; + const buffered = { key_display: "buffered", key_raw: "YnVmZmVyZWQ=", key_type: "string", ttl: -1 }; + mocks.redisScanKeysBatch.mockImplementation((_connectionId: string, _db: number, cursor: number) => Promise.resolve(cursor === 0 ? { cursor: 1, keys: [initial], total_keys: 2 } : { cursor: 0, keys: [buffered], total_keys: 0 })); + mountBrowser(); + await settle(); + + const row = { + id: "leaf:0:buffered", + depth: 0, + node: { + kind: "leaf" as const, + id: "leaf:0:buffered", + label: "buffered", + fullKeyDisplay: "buffered", + keyRaw: buffered.key_raw, + db: 0, + keyType: "string", + ttl: -1, + size: 0, + valuePreview: "", + pathSegments: ["buffered"], + }, + }; + mocks.buildRedisKeySnapshotCooperatively.mockResolvedValueOnce({ + flatKeys: [initial, buffered], + flatKeyByRaw: new Map([ + [initial.key_raw, initial], + [buffered.key_raw, buffered], + ]), + loadedKeyRaws: new Set([initial.key_raw, buffered.key_raw]), + filteredKeyCount: 2, + treeIndex: null, + expandedGroupIds: new Set(), + visibleRows: Array.from({ length: 25_001 }, () => row), + }); + const pendingFrames: FrameRequestCallback[] = []; + const originalRequestAnimationFrame = window.requestAnimationFrame; + window.requestAnimationFrame = (callback: FrameRequestCallback) => { + pendingFrames.push(callback); + return pendingFrames.length; + }; + + try { + clickButtonWithText("redis.fetchAllKeys"); + await vi.waitFor(() => expect(mocks.scrollerUpdateVisibleItems).toHaveBeenCalledTimes(1)); + clickButtonWithText("redis.stopFetchAll"); + expect(pendingFrames).toHaveLength(1); + pendingFrames[0](0); + await settle(); + + expect(document.body.textContent).toContain("initial"); + expect(document.body.textContent).not.toContain("buffered"); + expect(document.body.textContent).toContain("redis.fetchAllKeys"); + } finally { + window.requestAnimationFrame = originalRequestAnimationFrame; + } + }); + it("reloads instead of advancing past an uncommitted buffered page after reactivation", async () => { const bufferedPage = deferred<{ cursor: number; keys: Array<{ key_display: string; key_raw: string; key_type: string; ttl: number }>; total_keys: number }>(); let returnFreshPage = false; diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.vue b/apps/desktop/src/components/redis/RedisKeyBrowser.vue index 215775873f..26825ed00d 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.vue +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.vue @@ -30,6 +30,7 @@ import { continuousQueryResultMaxRows } from "@/lib/dataGrid/queryResultRowLimit import { appendRedisKeysToTreeIndex, buildRedisKeyTree, + buildRedisKeySnapshotCooperatively, canBuildRedisFuzzyTree, collectExpandedGroupIds, collectRedisGroupKeyRaws, @@ -42,6 +43,7 @@ import { type RedisKeyTreeGroupNode, type RedisKeyTreeIndex, type RedisKeyTreeNode, + type RedisKeyTreeRow, } from "@/lib/redis/redisKeyTree"; import { classifyRedisCommandSafety } from "@/lib/redis/redisCommandSafety"; import { isRedisMutatingCommand } from "@/lib/redis/redisCommandTable"; @@ -101,7 +103,7 @@ const redisExpiryTransport = { const flatKeys = shallowRef([]); const treeKeys = shallowRef([]); -const flatKeyByRaw = new Map(); +let flatKeyByRaw = new Map(); const loading = ref(false); const loadingMore = ref(false); const searchPending = ref(false); @@ -201,10 +203,23 @@ const AUTO_LOAD_TOTAL_SCAN_ITERATIONS = 50; let autoLoadBudget: ScanIterationBudget = { remaining: AUTO_LOAD_TOTAL_SCAN_ITERATIONS }; let redisBrowserIsActive = true; let reloadKeysOnActivation = false; +let fetchAllPreparingSnapshot = false; let redisDbFlushedListenerRegistered = false; let redisInfiniteScrollFrame = 0; -const loadedKeyRaws = new Set(); +let loadedKeyRaws = new Set(); let treeIndex: RedisKeyTreeIndex | null = null; +let fetchAllPublicationRollback: null | { + flatKeys: RedisKeyInfo[]; + flatKeyByRaw: Map; + loadedKeyRaws: Set; + treeKeys: RedisKeyTreeNode[]; + treeIndex: RedisKeyTreeIndex | null; + expandedGroupIds: Set; + scanCursor: number; + hasMore: boolean; + lastTotalKeys: number; + bufferedKeyRaws: Set; +} = null; // 展开分组时已定向补扫过的子树（见 fillGroupSubtree）；在 loadKeys 重置。 const subtreeFilledGroupIds = new Set(); // 刷新前已展开分组的快照（#7173）：刷新首屏重建树时，本轮尚未扫到的分组会被 @@ -246,6 +261,7 @@ const redisInfiniteScrollEnabled = computed(() => settingsStore.editorSettings.i const redisInfiniteScrollMaxKeys = computed(() => continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows)); watch(redisKeySeparator, () => { if (flatKeys.value.length === 0) return; + invalidateScanRequests(); if (useFlatKeySearchRows.value) { treeKeys.value = []; treeIndex = null; @@ -258,6 +274,7 @@ const lastTotalKeys = ref(0); // “仅看无过期”过滤开关：开启后只保留 TTL 为 -1（永不过期）的已加载 key。 // TTL 为 -2 的行（fetch-all 链路未查询 TTL）不会出现在过滤结果里。 const noExpiryOnly = ref(false); +const fetchAllFilteredKeyCount = ref(null); // 过滤后的平铺 key 列表：未开启过滤时与 flatKeys 完全一致，避免额外开销 const filteredFlatKeys = computed(() => { if (!noExpiryOnly.value) return flatKeys.value; @@ -273,7 +290,7 @@ const filteredTreeKeys = computed(() => { const displayedKeyCount = computed(() => { if (isFetchingAll.value) return fetchAllLoadedCount.value; // 过滤时展示匹配数量，便于确认“无过期”key 的规模 - if (noExpiryOnly.value) return filteredFlatKeys.value.length; + if (noExpiryOnly.value) return fetchAllFilteredKeyCount.value ?? filteredFlatKeys.value.length; return flatKeys.value.length; }); const fetchAllProgressText = computed(() => { @@ -385,9 +402,51 @@ async function updateCreateKeyTypeHelpOffset() { watch(activeCreateKeyTypeHelp, () => { void updateCreateKeyTypeHelpOffset(); }); -const visibleRows = computed(() => { +const regularVisibleRows = computed(() => { return useFlatKeySearchRows.value ? filteredFlatKeys.value.map((key) => redisKeyToFlatTreeRow(key, props.db)) : flattenVisibleRedisKeyTree(filteredTreeKeys.value, expandedGroupIds.value); }); +// Fetch All keeps this raw array identity bound to RecycleScroller. Replacing +// the prop array makes vue-virtual-scroller slice and inspect every row key; +// bounded in-place publication plus updateVisibleItems avoids that O(N) turn. +const fetchAllVisibleRows = shallowRef([]); +const fetchAllVisibleRowsActive = ref(false); +const visibleRows = computed(() => (fetchAllVisibleRowsActive.value ? fetchAllVisibleRows.value : regularVisibleRows.value)); + +function activateFetchAllVisibleRows() { + if (fetchAllVisibleRowsActive.value) return; + // Return the exact currently-bound identity when the override is enabled, so + // entering Fetch All does not itself trigger the scroller's items watcher. + fetchAllVisibleRows.value = regularVisibleRows.value; + fetchAllVisibleRowsActive.value = true; +} + +function deactivateFetchAllVisibleRows() { + if (!fetchAllVisibleRowsActive.value) return; + fetchAllVisibleRowsActive.value = false; + fetchAllVisibleRows.value = []; + fetchAllFilteredKeyCount.value = null; +} + +function rollbackFetchAllPublication() { + const rollback = fetchAllPublicationRollback; + if (!rollback) return; + flatKeys.value = rollback.flatKeys; + flatKeyByRaw = rollback.flatKeyByRaw; + loadedKeyRaws = rollback.loadedKeyRaws; + treeKeys.value = rollback.treeKeys; + treeIndex = rollback.treeIndex; + expandedGroupIds.value = rollback.expandedGroupIds; + scanCursor.value = rollback.scanCursor; + hasMore.value = rollback.hasMore; + lastTotalKeys.value = rollback.lastTotalKeys; + for (const keyRaw of rollback.bufferedKeyRaws) { + ttlObservedAtByRaw.delete(keyRaw); + positiveTtlKeyRaws.delete(keyRaw); + } + fetchAllPublicationRollback = null; + if (selectedKeyRaw.value && !flatKeyByRaw.has(selectedKeyRaw.value)) selectedKeyRaw.value = null; + refreshSelectedGroupLeafCounts(); +} function redisRowKeyInfo(node: RedisKeyTreeNode): RedisKeyInfo | null { void keyMetadataEpoch.value; @@ -616,6 +675,7 @@ function onKeyPaneKeydown(event: KeyboardEvent) { } function rebuildTree(expandAll = false) { + deactivateFetchAllVisibleRows(); if (useFlatKeySearchRows.value) { treeKeys.value = []; treeIndex = null; @@ -646,6 +706,7 @@ function rebuildTree(expandAll = false) { function mergeTree(newKeys: RedisKeyInfo[]) { if (newKeys.length === 0) return; + deactivateFetchAllVisibleRows(); if (!treeIndex) { rebuildTree(isSearchMode.value); return; @@ -671,6 +732,14 @@ function mergeTree(newKeys: RedisKeyInfo[]) { } function invalidateScanRequests(): number { + rollbackFetchAllPublication(); + deactivateFetchAllVisibleRows(); + // Keep invalidation authoritative even when the old Fetch All continuation + // observes the generation mismatch and therefore skips its own finalizer. + isFetchingAll.value = false; + fetchAllStopRequested.value = true; + fetchAllLoadedCount.value = 0; + fetchAllPreparingSnapshot = false; searchRequestId++; loadMoreOperationId++; loadingMore.value = false; @@ -678,6 +747,10 @@ function invalidateScanRequests(): number { return searchRequestId; } +function invalidateFetchAllForStructuralMutation() { + if (isFetchingAll.value || fetchAllPublicationRollback) invalidateScanRequests(); +} + function isCurrentScanOperation(requestId: number, operationId?: number): boolean { return requestId === searchRequestId && (operationId === undefined || operationId === loadMoreOperationId); } @@ -782,6 +855,25 @@ function appendScanResult(result: RedisScanResult, options: { updateTree?: boole return newKeys.length; } +function bufferFetchAllScanResult(result: RedisScanResult, buffer: RedisKeyInfo[], bufferedKeyRaws: Set): number { + let added = 0; + for (const key of result.keys) { + if (loadedKeyRaws.has(key.key_raw) || bufferedKeyRaws.has(key.key_raw)) continue; + bufferedKeyRaws.add(key.key_raw); + buffer.push(key); + recordKeyTtlObservedAt(key); + added++; + } + scanCursor.value = result.cursor; + hasMore.value = result.cursor !== 0; + if (result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0)) lastTotalKeys.value = result.total_keys; + connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, { + loaded: isSearchMode.value ? undefined : flatKeys.value.length + buffer.length, + total: result.total_keys > 0 || (result.cursor === 0 && result.keys.length === 0) ? result.total_keys : undefined, + }); + return added; +} + async function scanNextPage(requestId = searchRequestId, operationId?: number, iterationBudget?: ScanIterationBudget): Promise { const result = await fetchScanPage(requestId, operationId, iterationBudget); if (!isCurrentScanOperation(requestId, operationId)) return false; @@ -938,16 +1030,81 @@ function onRedisKeyScroll(event: Event) { // end; per-page tree sorting dominates runtime on million-key pattern scans. const FETCH_ALL_SCAN_COUNT = 50000; const FETCH_ALL_BATCH_ITERATIONS = 8; +const FETCH_ALL_PUBLISH_CHUNK_SIZE = 25_000; + +function yieldForRedisKeyBrowserPaint(): Promise { + return new Promise((resolve) => requestAnimationFrame(() => resolve())); +} + +function refreshRedisKeyScroller() { + const scroller = redisKeyScrollerRef.value as unknown as { updateVisibleItems?: (force?: boolean) => unknown } | null; + // Entries changed in place while the array identity stayed stable. Tell the + // scroller to invalidate its keyed view pool for the currently rendered + // range; `false` preserves old key/view mappings and can combine row labels. + scroller?.updateVisibleItems?.(true); +} + +async function publishFetchAllVisibleRows(rows: readonly RedisKeyTreeRow[], requestId: number): Promise { + const target = fetchAllVisibleRows.value; + if (rows.length === 0) { + target.length = 0; + refreshRedisKeyScroller(); + return requestId === searchRequestId && redisBrowserIsActive && !fetchAllStopRequested.value; + } + + for (let offset = 0; offset < rows.length; offset += FETCH_ALL_PUBLISH_CHUNK_SIZE) { + if (requestId !== searchRequestId || !redisBrowserIsActive || fetchAllStopRequested.value) return false; + const end = Math.min(offset + FETCH_ALL_PUBLISH_CHUNK_SIZE, rows.length); + for (let index = offset; index < end; index++) target[index] = rows[index]; + if (end === rows.length) target.length = rows.length; + refreshRedisKeyScroller(); + // Yield only when another bounded publication unit remains. + if (end < rows.length) await yieldForRedisKeyBrowserPaint(); + } + return requestId === searchRequestId && redisBrowserIsActive && !fetchAllStopRequested.value; +} async function fetchAll(): Promise { if (!hasMore.value || isFetchingAll.value) return false; const requestId = searchRequestId; const bufferedKeys: RedisKeyInfo[] = []; + const bufferedKeyRaws = new Set(); + const initialFlatKeys = flatKeys.value; + const initialFlatKeyByRaw = flatKeyByRaw; + const initialLoadedKeyRaws = loadedKeyRaws; + const initialTreeKeys = treeKeys.value; + const initialTreeIndex = treeIndex; + const initialExpandedGroupIds = expandedGroupIds.value; + const initialScanCursor = scanCursor.value; + const initialHasMore = hasMore.value; + const initialLastTotalKeys = lastTotalKeys.value; + const snapshotConfig = { + db: props.db, + separator: redisKeySeparator.value, + flatRows: useFlatKeySearchRows.value, + expandAll: isSearchMode.value, + expandedGroupIds: expandedGroupIds.value, + noExpiryOnly: noExpiryOnly.value, + }; + fetchAllPublicationRollback = { + flatKeys: initialFlatKeys, + flatKeyByRaw: initialFlatKeyByRaw, + loadedKeyRaws: initialLoadedKeyRaws, + treeKeys: initialTreeKeys, + treeIndex: initialTreeIndex, + expandedGroupIds: initialExpandedGroupIds, + scanCursor: initialScanCursor, + hasMore: initialHasMore, + lastTotalKeys: initialLastTotalKeys, + bufferedKeyRaws, + }; + activateFetchAllVisibleRows(); isFetchingAll.value = true; fetchAllStopRequested.value = false; fetchAllLoadedCount.value = flatKeys.value.length; let changed = false; let completed = false; + let publicationSucceeded = true; try { while (requestId === searchRequestId && !fetchAllStopRequested.value && hasMore.value) { const result = await fetchScanBatchPage(FETCH_ALL_BATCH_ITERATIONS, { @@ -955,28 +1112,69 @@ async function fetchAll(): Promise { includeTypes: false, }); if (requestId !== searchRequestId) break; - changed = appendScanResult(result, { updateTree: false, buffer: bufferedKeys }) > 0 || changed; + changed = bufferFetchAllScanResult(result, bufferedKeys, bufferedKeyRaws) > 0 || changed; fetchAllLoadedCount.value = flatKeys.value.length + bufferedKeys.length; } completed = requestId === searchRequestId && !fetchAllStopRequested.value && !hasMore.value; } finally { if (requestId === searchRequestId) { - appendFlatKeyRecords(bufferedKeys); - if (changed) { - if (useFlatKeySearchRows.value) { - treeKeys.value = []; - treeIndex = null; - expandedGroupIds.value = new Set(); - } else { - rebuildTree(isSearchMode.value); + let published = !changed && !fetchAllStopRequested.value; + try { + if (changed) { + snapshotConfig.flatRows = (searchMode.value === "key" && isSearchMode.value && !fuzzyKeySearch.value) || (isFuzzyKeySearch.value && !canBuildRedisFuzzyTree(initialFlatKeys.length + bufferedKeys.length)); + snapshotConfig.expandAll = isSearchMode.value; + snapshotConfig.expandedGroupIds = expandedGroupIds.value; + snapshotConfig.noExpiryOnly = noExpiryOnly.value; + activateFetchAllVisibleRows(); + fetchAllPreparingSnapshot = true; + const snapshot = await buildRedisKeySnapshotCooperatively([initialFlatKeys, bufferedKeys], snapshotConfig, { + shouldContinue: () => requestId === searchRequestId && redisBrowserIsActive && !fetchAllStopRequested.value, + }); + if (snapshot && requestId === searchRequestId && redisBrowserIsActive) { + flatKeys.value = snapshot.flatKeys; + flatKeyByRaw = snapshot.flatKeyByRaw; + loadedKeyRaws = snapshot.loadedKeyRaws; + treeIndex = snapshot.treeIndex; + treeKeys.value = snapshot.treeIndex?.root ?? []; + expandedGroupIds.value = snapshot.expandedGroupIds; + fetchAllFilteredKeyCount.value = snapshotConfig.noExpiryOnly ? snapshot.filteredKeyCount : null; + refreshSelectedGroupLeafCounts(); + published = await publishFetchAllVisibleRows(snapshot.visibleRows, requestId); + } } + } finally { + fetchAllPreparingSnapshot = false; + publicationSucceeded = published; + if (published) { + fetchAllPublicationRollback = null; + if (!hasMore.value) refreshExpandedGroupIds.clear(); + } + if (!published) { + rollbackFetchAllPublication(); + deactivateFetchAllVisibleRows(); + if (requestId === searchRequestId) { + scanCursor.value = initialScanCursor; + hasMore.value = initialHasMore; + lastTotalKeys.value = initialLastTotalKeys; + for (const keyRaw of bufferedKeyRaws) { + ttlObservedAtByRaw.delete(keyRaw); + positiveTtlKeyRaws.delete(keyRaw); + } + connectionStore.updateRedisDbKeyStats(props.connectionId, props.db, { + loaded: isSearchMode.value ? undefined : initialFlatKeys.length, + total: initialLastTotalKeys, + }); + } + } + isFetchingAll.value = false; + fetchAllStopRequested.value = false; + fetchAllLoadedCount.value = 0; } - isFetchingAll.value = false; - fetchAllStopRequested.value = false; - fetchAllLoadedCount.value = 0; + } else { + publicationSucceeded = false; } } - return completed; + return completed && publicationSucceeded; } function stopFetchAll() { @@ -1025,6 +1223,13 @@ async function fillGroupSubtree(group: RedisKeyTreeGroupNode, requestId = search } function toggleGroup(groupId: string) { + if (fetchAllPreparingSnapshot) { + invalidateScanRequests(); + isFetchingAll.value = false; + fetchAllStopRequested.value = false; + fetchAllLoadedCount.value = 0; + } + deactivateFetchAllVisibleRows(); const next = new Set(expandedGroupIds.value); const expanding = !next.has(groupId); if (expanding) next.add(groupId); @@ -1058,6 +1263,7 @@ function onRowClick(node: RedisKeyTreeNode, event?: MouseEvent) { function removeKnownKey(keyRaw: string) { if (!flatKeyByRaw.has(keyRaw)) return; + invalidateFetchAllForStructuralMutation(); loadedKeyRaws.delete(keyRaw); ttlObservedAtByRaw.delete(keyRaw); positiveTtlKeyRaws.delete(keyRaw); @@ -1087,6 +1293,7 @@ function onKeyRenamed(oldKeyRaw: string, newKeyRaw: string, newKeyDisplay: strin return; } + invalidateFetchAllForStructuralMutation(); const previous = flatKeyByRaw.get(oldKeyRaw); if (!previous) { void loadKeys(); @@ -1135,12 +1342,19 @@ function onKeyLoaded(value: RedisValue) { } const keyInfo = redisValueToKeyInfo(value); const previousTtl = flatKeyByRaw.get(keyInfo.key_raw)?.ttl ?? -2; + const noExpiryMembershipChanged = (previousTtl === -1) !== (keyInfo.ttl === -1); if (!updateRedisKeyInfoMetadataByRaw(flatKeyByRaw, keyInfo)) return; // 详情面板回写了最新的 TTL，同步刷新观测时刻，保证两侧倒计时一致 recordKeyTtlObservedAt(keyInfo); loadedKeyRaws.add(keyInfo.key_raw); if (treeIndex) updateRedisKeyTreeLeafMetadata(treeIndex, keyInfo); - if ((previousTtl === -1) !== (keyInfo.ttl === -1)) noExpiryProjectionEpoch.value++; + if (noExpiryMembershipChanged) { + noExpiryProjectionEpoch.value++; + // A Fetch All stable row snapshot intentionally ignores ordinary metadata + // writes. Filter membership is structural, though, so leave that snapshot + // and let the canonical no-expiry projection include/exclude the key. + if (noExpiryOnly.value) deactivateFetchAllVisibleRows(); + } keyMetadataEpoch.value++; syncListTtlTimer(); } @@ -1499,6 +1713,7 @@ function upsertCreatedKey(value: RedisValue) { size: redisValueSize(value), value_preview: redisValuePreview(value), }; + invalidateFetchAllForStructuralMutation(); const existing = flatKeyByRaw.get(keyInfo.key_raw); // 新建 key 携带的 TTL 以当前时刻为观测起点 recordKeyTtlObservedAt(keyInfo); @@ -1901,6 +2116,12 @@ function toggleFuzzyKeySearch() { if (searchMode.value === "key") void loadKeys(); } +function toggleNoExpiryOnly() { + if (isFetchingAll.value) return; + deactivateFetchAllVisibleRows(); + noExpiryOnly.value = !noExpiryOnly.value; +} + function getSearchInput(): HTMLInputElement | null { return rootRef.value?.querySelector("[data-redis-search-input]") ?? null; } @@ -2399,8 +2620,9 @@ defineExpose({ focusSearch, insertCommand, executeCommand: executeAiCommand }); :class="noExpiryOnly ? 'bg-accent text-accent-foreground' : 'border border-dashed border-border/70 text-muted-foreground hover:text-foreground'" :title="t('redis.noExpiryOnlyTitle')" :aria-pressed="noExpiryOnly" + :disabled="isFetchingAll" data-redis-no-expiry-filter - @click="noExpiryOnly = !noExpiryOnly" + @click="toggleNoExpiryOnly" > {{ t("redis.noExpiryOnly") }} diff --git a/apps/desktop/src/lib/redis/redisKeyTree.ts b/apps/desktop/src/lib/redis/redisKeyTree.ts index d41191b673..ebcc209f42 100644 --- a/apps/desktop/src/lib/redis/redisKeyTree.ts +++ b/apps/desktop/src/lib/redis/redisKeyTree.ts @@ -51,6 +51,25 @@ export interface AppendRedisKeysResult { addedGroupIds: Set; } +export interface RedisKeySnapshot { + flatKeys: RedisKeyInfo[]; + flatKeyByRaw: Map; + loadedKeyRaws: Set; + filteredKeyCount: number; + treeIndex: RedisKeyTreeIndex | null; + expandedGroupIds: Set; + visibleRows: RedisKeyTreeRow[]; +} + +export interface CooperativeRedisKeySnapshotOptions { + /** Maximum records handled before yielding back to the browser. */ + workChunkSize?: number; + /** Production yields to a paint-capable browser turn; tests inject a deterministic scheduler. */ + yieldControl?: () => Promise; + /** Checked before work, after every yield, and before returning the completed snapshot. */ + shouldContinue?: () => boolean; +} + export function redisKeyNameCopyText(node: RedisKeyTreeNode): string | null { // keyRaw is base64-encoded for backend roundtrips; copy the user-visible // Redis key name instead of the internal transport value. @@ -91,6 +110,307 @@ function compareRedisTreeNodes(a: RedisKeyTreeNode, b: RedisKeyTreeNode): number return a.label.localeCompare(b.label); } +function defaultYieldControl(): Promise { + return new Promise((resolve) => { + if (typeof requestAnimationFrame === "function") { + requestAnimationFrame(() => resolve()); + return; + } + setTimeout(resolve, 0); + }); +} + +interface MutableTreeBuildState { + index: RedisKeyTreeIndex; + touchedLevels: Set; + addedGroupIds: Set; +} + +class CooperativeWorkController { + private completedUnits = 0; + + constructor( + readonly chunkSize: number, + private readonly yieldControl: () => Promise, + private readonly shouldContinue: () => boolean, + ) {} + + isActive(): boolean { + return this.shouldContinue(); + } + + recordCompleted(units: number) { + this.completedUnits += units; + } + + needsYield(): boolean { + return this.completedUnits >= this.chunkSize; + } + + async yieldIfNeeded(): Promise { + if (!this.isActive()) return false; + if (!this.needsYield()) return true; + this.completedUnits = 0; + await this.yieldControl(); + return this.isActive(); + } +} + +function appendRedisKeyUnsorted(state: MutableTreeBuildState, key: RedisKeyInfo, db: number, separator: string) { + const { index, touchedLevels, addedGroupIds } = state; + if (index.leafByKeyRaw.has(key.key_raw)) return; + + const pathSegments = separator ? key.key_display.split(separator) : [key.key_display]; + const ancestorGroupIds: string[] = []; + let currentLevel = index.root; + + if (pathSegments.length > 1) { + const groupSegments: string[] = []; + for (const segment of pathSegments.slice(0, -1)) { + groupSegments.push(segment); + const groupId = buildGroupId(db, groupSegments); + let group = index.groupById.get(groupId); + if (!group) { + group = { + kind: "group", + id: groupId, + label: segment, + pathSegments: [...groupSegments], + children: [], + loadedLeafCount: 0, + }; + index.groupById.set(groupId, group); + currentLevel.push(group); + touchedLevels.add(currentLevel); + addedGroupIds.add(groupId); + } + group.loadedLeafCount++; + ancestorGroupIds.push(groupId); + currentLevel = group.children; + } + } + + const leaf: RedisKeyTreeLeafNode = { + kind: "leaf", + id: buildLeafId(db, key.key_raw), + label: pathSegments[pathSegments.length - 1], + fullKeyDisplay: key.key_display, + keyRaw: key.key_raw, + db, + keyType: key.key_type ?? "", + ttl: key.ttl ?? -2, + size: key.size ?? 0, + valuePreview: key.value_preview ?? "", + pathSegments, + }; + currentLevel.push(leaf); + touchedLevels.add(currentLevel); + index.leafByKeyRaw.set(key.key_raw, leaf); + index.ancestorGroupIdsByKeyRaw.set(key.key_raw, ancestorGroupIds); +} + +async function sortRedisTreeLevelCooperatively(nodes: RedisKeyTreeNode[], controller: CooperativeWorkController): Promise { + if (nodes.length < 2) { + controller.recordCompleted(nodes.length); + return controller.isActive(); + } + + let runs: RedisKeyTreeNode[][] = []; + for (let offset = 0; offset < nodes.length; offset += controller.chunkSize) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return false; + const run = nodes.slice(offset, offset + controller.chunkSize); + run.sort(compareRedisTreeNodes); + runs.push(run); + controller.recordCompleted(run.length); + } + + while (runs.length > 1) { + const mergedRuns: RedisKeyTreeNode[][] = []; + for (let runIndex = 0; runIndex < runs.length; runIndex += 2) { + const left = runs[runIndex]; + const right = runs[runIndex + 1]; + if (!right) { + mergedRuns.push(left); + continue; + } + + const merged: RedisKeyTreeNode[] = []; + merged.length = left.length + right.length; + let leftIndex = 0; + let rightIndex = 0; + let outputIndex = 0; + while (leftIndex < left.length || rightIndex < right.length) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return false; + const start = outputIndex; + const end = Math.min(outputIndex + controller.chunkSize, merged.length); + while (outputIndex < end) { + if (rightIndex >= right.length || (leftIndex < left.length && compareRedisTreeNodes(left[leftIndex], right[rightIndex]) <= 0)) { + merged[outputIndex++] = left[leftIndex++]; + } else { + merged[outputIndex++] = right[rightIndex++]; + } + } + controller.recordCompleted(outputIndex - start); + } + mergedRuns.push(merged); + } + runs = mergedRuns; + } + + const sorted = runs[0]; + for (let offset = 0; offset < sorted.length; offset += controller.chunkSize) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return false; + const end = Math.min(offset + controller.chunkSize, sorted.length); + for (let index = offset; index < end; index++) nodes[index] = sorted[index]; + controller.recordCompleted(end - offset); + } + return controller.isActive(); +} + +async function flattenVisibleRedisKeyTreeCooperatively(nodes: RedisKeyTreeNode[], expandedGroupIds: ReadonlySet, controller: CooperativeWorkController): Promise { + const rows: RedisKeyTreeRow[] = []; + const stack: Array<{ nodes: RedisKeyTreeNode[]; nextIndex: number; depth: number }> = [{ nodes, nextIndex: 0, depth: 0 }]; + + while (stack.length > 0) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return null; + const start = rows.length; + const end = rows.length + controller.chunkSize; + while (stack.length > 0 && rows.length < end) { + const frame = stack[stack.length - 1]; + if (frame.nextIndex >= frame.nodes.length) { + stack.pop(); + continue; + } + const node = frame.nodes[frame.nextIndex++]; + rows.push({ id: node.id, node, depth: frame.depth }); + if (node.kind === "group" && expandedGroupIds.has(node.id) && node.children.length > 0) { + // Keep only a cursor for the child collection. Pushing every child of + // one expanded million-key group would itself be an unbounded O(N) + // task before the outer cooperative loop gets another yield point. + stack.push({ nodes: node.children, nextIndex: 0, depth: frame.depth + 1 }); + } + } + controller.recordCompleted(rows.length - start); + } + return controller.isActive() ? rows : null; +} + +/** + * Builds the complete Fetch All snapshot without exposing partially-built + * structures to Vue. Every linear stage is divided into bounded work units so + * the browser can paint and dispatch input between them. + */ +export async function buildRedisKeySnapshotCooperatively( + keyBatches: readonly (readonly RedisKeyInfo[])[], + config: { + db: number; + separator?: string; + flatRows: boolean; + expandAll: boolean; + expandedGroupIds: ReadonlySet; + noExpiryOnly?: boolean; + }, + options: CooperativeRedisKeySnapshotOptions = {}, +): Promise { + const chunkSize = Math.max(1, options.workChunkSize ?? 25_000); + const yieldControl = options.yieldControl ?? defaultYieldControl; + const shouldContinue = options.shouldContinue ?? (() => true); + const controller = new CooperativeWorkController(chunkSize, yieldControl, shouldContinue); + const separator = config.separator ?? ":"; + if (!shouldContinue()) return null; + + const flatKeys: RedisKeyInfo[] = []; + const flatKeyByRaw = new Map(); + const loadedKeyRaws = new Set(); + let filteredKeyCount = 0; + const treeState: MutableTreeBuildState | null = config.flatRows + ? null + : { + index: { root: [], groupById: new Map(), leafByKeyRaw: new Map(), ancestorGroupIdsByKeyRaw: new Map() }, + touchedLevels: new Set(), + addedGroupIds: new Set(), + }; + const filteredTreeState: MutableTreeBuildState | null = + !config.flatRows && config.noExpiryOnly + ? { + index: { root: [], groupById: new Map(), leafByKeyRaw: new Map(), ancestorGroupIdsByKeyRaw: new Map() }, + touchedLevels: new Set(), + addedGroupIds: new Set(), + } + : null; + + for (const batch of keyBatches) { + for (let offset = 0; offset < batch.length; offset += chunkSize) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return null; + const end = Math.min(offset + chunkSize, batch.length); + for (let index = offset; index < end; index++) { + const key = batch[index]; + if (flatKeyByRaw.has(key.key_raw)) continue; + flatKeyByRaw.set(key.key_raw, key); + loadedKeyRaws.add(key.key_raw); + flatKeys.push(key); + if (!config.noExpiryOnly || key.ttl === -1) filteredKeyCount++; + if (treeState) appendRedisKeyUnsorted(treeState, key, config.db, separator); + if (filteredTreeState && key.ttl === -1) appendRedisKeyUnsorted(filteredTreeState, key, config.db, separator); + } + controller.recordCompleted(end - offset); + } + } + + for (const state of [treeState, filteredTreeState]) { + if (!state) continue; + for (const level of state.touchedLevels) { + if (level.length <= controller.chunkSize) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return null; + if (level.length > 1) level.sort(compareRedisTreeNodes); + controller.recordCompleted(level.length); + continue; + } + if (!(await sortRedisTreeLevelCooperatively(level, controller))) return null; + } + } + + const treeIndex = treeState?.index ?? null; + const expandedGroupIds = new Set(); + if (treeIndex) { + if (config.expandAll) { + for (const id of treeIndex.groupById.keys()) { + if (controller.needsYield() && !(await controller.yieldIfNeeded())) return null; + expandedGroupIds.add(id); + controller.recordCompleted(1); + } + } else { + for (const id of config.expandedGroupIds) { + if (controller.needsYield() && !(await controller.yieldIfNeeded())) return null; + if (treeIndex.groupById.has(id)) expandedGroupIds.add(id); + controller.recordCompleted(1); + } + } + } + + let visibleRows: RedisKeyTreeRow[]; + if (config.flatRows) { + visibleRows = []; + for (let offset = 0; offset < flatKeys.length; offset += chunkSize) { + if (!controller.isActive() || (controller.needsYield() && !(await controller.yieldIfNeeded()))) return null; + const end = Math.min(offset + chunkSize, flatKeys.length); + for (let index = offset; index < end; index++) { + const key = flatKeys[index]; + if (!config.noExpiryOnly || key.ttl === -1) visibleRows.push(redisKeyToFlatTreeRow(key, config.db)); + } + controller.recordCompleted(end - offset); + } + } else { + const visibleTree = filteredTreeState?.index.root ?? treeIndex?.root ?? []; + const rows = await flattenVisibleRedisKeyTreeCooperatively(visibleTree, expandedGroupIds, controller); + if (!rows) return null; + visibleRows = rows; + } + + if (!shouldContinue()) return null; + return { flatKeys, flatKeyByRaw, loadedKeyRaws, filteredKeyCount, treeIndex, expandedGroupIds, visibleRows }; +} + function redisKeyInfoFromLeaf(node: RedisKeyTreeLeafNode): RedisKeyInfo { return { key_display: node.fullKeyDisplay, @@ -118,63 +438,10 @@ export function createRedisKeyTreeIndex(keys: readonly RedisKeyInfo[], db: numbe * batch. This keeps repeated "load more" merges from re-sorting the full tree. */ export function appendRedisKeysToTreeIndex(index: RedisKeyTreeIndex, keys: readonly RedisKeyInfo[], db: number, separator = ":"): AppendRedisKeysResult { - const touchedLevels = new Set(); - const addedGroupIds = new Set(); - - for (const key of keys) { - if (index.leafByKeyRaw.has(key.key_raw)) continue; - - const pathSegments = separator ? key.key_display.split(separator) : [key.key_display]; - const ancestorGroupIds: string[] = []; - let currentLevel = index.root; - - if (pathSegments.length > 1) { - const groupSegments: string[] = []; - for (const segment of pathSegments.slice(0, -1)) { - groupSegments.push(segment); - const groupId = buildGroupId(db, groupSegments); - let group = index.groupById.get(groupId); - if (!group) { - group = { - kind: "group", - id: groupId, - label: segment, - pathSegments: [...groupSegments], - children: [], - loadedLeafCount: 0, - }; - index.groupById.set(groupId, group); - currentLevel.push(group); - touchedLevels.add(currentLevel); - addedGroupIds.add(groupId); - } - group.loadedLeafCount++; - ancestorGroupIds.push(groupId); - currentLevel = group.children; - } - } - - const leaf: RedisKeyTreeLeafNode = { - kind: "leaf", - id: buildLeafId(db, key.key_raw), - label: pathSegments[pathSegments.length - 1], - fullKeyDisplay: key.key_display, - keyRaw: key.key_raw, - db, - keyType: key.key_type ?? "", - ttl: key.ttl ?? -2, - size: key.size ?? 0, - valuePreview: key.value_preview ?? "", - pathSegments, - }; - currentLevel.push(leaf); - touchedLevels.add(currentLevel); - index.leafByKeyRaw.set(key.key_raw, leaf); - index.ancestorGroupIdsByKeyRaw.set(key.key_raw, ancestorGroupIds); - } - - for (const nodes of touchedLevels) nodes.sort(compareRedisTreeNodes); - return { addedGroupIds }; + const state: MutableTreeBuildState = { index, touchedLevels: new Set(), addedGroupIds: new Set() }; + for (const key of keys) appendRedisKeyUnsorted(state, key, db, separator); + for (const nodes of state.touchedLevels) nodes.sort(compareRedisTreeNodes); + return { addedGroupIds: state.addedGroupIds }; } /** diff --git a/packages/app-tests/redisKeyTree.test.ts b/packages/app-tests/redisKeyTree.test.ts index 518926791f..20ea6303af 100644 --- a/packages/app-tests/redisKeyTree.test.ts +++ b/packages/app-tests/redisKeyTree.test.ts @@ -2,6 +2,7 @@ import { test } from "vitest"; import assert from "node:assert/strict"; import { appendRedisKeysToTreeIndex, + buildRedisKeySnapshotCooperatively, buildRedisKeyTree, canBuildRedisFuzzyTree, collectRedisGroupKeyRaws, @@ -115,6 +116,75 @@ test("flattenVisibleRedisKeyTree handles very large expanded groups without stac assert.ok(rows.every((row) => row.id === row.node.id)); }); +test("cooperative snapshot matches the synchronous hierarchy, ordering, rows, and indexes", async () => { + const keys = [makeKey("team:web:home", "k2"), makeKey("zulu", "k5"), makeKey("team:api:v2", "k3"), makeKey("alpha", "k4"), makeKey("team:api:v1", "k1"), makeKey("team:api:v1", "k1")]; + let yields = 0; + + const snapshot = await buildRedisKeySnapshotCooperatively([keys.slice(0, 2), keys.slice(2)], { db: 3, flatRows: false, expandAll: true, expandedGroupIds: new Set() }, { workChunkSize: 2, yieldControl: async () => void yields++ }); + + assert.ok(snapshot); + const referenceIndex = createRedisKeyTreeIndex(keys, 3); + const referenceRows = flattenVisibleRedisKeyTree(referenceIndex.root, collectExpandedGroupIds(referenceIndex.root)); + assert.deepEqual(snapshot.treeIndex?.root, referenceIndex.root); + assert.deepEqual(snapshot.visibleRows, referenceRows); + assert.deepEqual( + snapshot.flatKeys.map((key) => key.key_raw), + ["k2", "k5", "k3", "k4", "k1"], + ); + assert.equal(snapshot.flatKeyByRaw.size, 5); + assert.equal(snapshot.treeIndex?.leafByKeyRaw.size, 5); + assert.deepEqual(snapshot.treeIndex?.ancestorGroupIdsByKeyRaw.get("k1"), referenceIndex.ancestorGroupIdsByKeyRaw.get("k1")); + assert.ok(yields > 3); +}); + +test("cooperative snapshot sorts a sibling collection across work slices", async () => { + const keys = Array.from({ length: 101 }, (_, index) => { + const reversed = String(100 - index).padStart(3, "0"); + return makeKey(`bucket:${reversed}`, `raw-${reversed}`); + }); + + const snapshot = await buildRedisKeySnapshotCooperatively([keys], { db: 0, flatRows: false, expandAll: true, expandedGroupIds: new Set() }, { workChunkSize: 7, yieldControl: async () => undefined }); + + assert.ok(snapshot?.treeIndex); + const bucket = findGroup(snapshot.treeIndex.root, "bucket"); + assert.deepEqual( + bucket.children.map((node) => node.label), + Array.from({ length: 101 }, (_, index) => String(index).padStart(3, "0")), + ); +}); + +test("cooperative snapshot supports flat filtered rows without building a hierarchy", async () => { + const snapshot = await buildRedisKeySnapshotCooperatively([[makeKey("a:b", "k1", "string", -1), makeKey("c:d", "k2", "string", 60)]], { db: 2, flatRows: true, expandAll: false, expandedGroupIds: new Set(), noExpiryOnly: true }, { workChunkSize: 1, yieldControl: async () => undefined }); + + assert.ok(snapshot); + assert.equal(snapshot.treeIndex, null); + assert.deepEqual( + snapshot.visibleRows.map((row) => row.node.label), + ["a:b"], + ); + assert.equal(snapshot.flatKeys.length, 2); +}); + +test("cooperative snapshot aborts between bounded work slices", async () => { + let active = true; + let yields = 0; + const snapshot = await buildRedisKeySnapshotCooperatively( + [Array.from({ length: 20 }, (_, index) => makeKey(`key:${index}`, `raw-${index}`))], + { db: 0, flatRows: false, expandAll: true, expandedGroupIds: new Set() }, + { + workChunkSize: 3, + shouldContinue: () => active, + yieldControl: async () => { + yields++; + active = yields < 2; + }, + }, + ); + + assert.equal(snapshot, null); + assert.equal(yields, 2); +}); + test("redisKeyToFlatTreeRow keeps search results flat with the full key label", () => { const row = redisKeyToFlatTreeRow(makeKey("user:profile:1", "user:profile:1", ""), 0);