feat(redis): add fuzzy key tree search
This commit is contained in:
1 parent
589fbb02c9
commit
4c57520a12
13 files changed
+901
-172
No files matched your search
@@ -1,5 +1,8 @@
|
||||
import type { RedisKeyInfo } from "@/lib/backend/api";
|
||||
|
||||
// Keep desktop IPC and Redis command payloads bounded for large group deletes.
|
||||
export const REDIS_DELETE_KEY_BATCH_SIZE = 1_000;
|
||||
|
||||
export function collectUniqueRedisKeys(keys: RedisKeyInfo[], loadedKeyRaws: Set<string>): RedisKeyInfo[] {
|
||||
const uniqueKeys: RedisKeyInfo[] = [];
|
||||
|
||||
@@ -11,3 +14,9 @@ export function collectUniqueRedisKeys(keys: RedisKeyInfo[], loadedKeyRaws: Set<
|
||||
|
||||
return uniqueKeys;
|
||||
}
|
||||
|
||||
export function* chunkRedisKeyRaws(keyRaws: readonly string[], batchSize = REDIS_DELETE_KEY_BATCH_SIZE): Generator<string[]> {
|
||||
for (let start = 0; start < keyRaws.length; start += batchSize) {
|
||||
yield keyRaws.slice(start, start + batchSize);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,13 @@
|
||||
import type { RedisKeyInfo } from "@/lib/backend/api";
|
||||
|
||||
// A hierarchy duplicates key metadata and indexes; above this limit keep
|
||||
// fuzzy results virtualized as flat rows instead of exhausting desktop memory.
|
||||
export const REDIS_FUZZY_TREE_MAX_KEYS = 200_000;
|
||||
|
||||
export function canBuildRedisFuzzyTree(loadedKeyCount: number): boolean {
|
||||
return loadedKeyCount <= REDIS_FUZZY_TREE_MAX_KEYS;
|
||||
}
|
||||
|
||||
export interface RedisKeyTreeLeafNode {
|
||||
kind: "leaf";
|
||||
id: string;
|
||||
@@ -20,6 +28,8 @@ export interface RedisKeyTreeGroupNode {
|
||||
label: string;
|
||||
pathSegments: string[];
|
||||
children: RedisKeyTreeNode[];
|
||||
// The rendered count must not recursively walk an entire visible subtree.
|
||||
loadedLeafCount: number;
|
||||
}
|
||||
|
||||
export type RedisKeyTreeNode = RedisKeyTreeLeafNode | RedisKeyTreeGroupNode;
|
||||
@@ -30,6 +40,17 @@ export interface RedisKeyTreeRow {
|
||||
depth: number;
|
||||
}
|
||||
|
||||
export interface RedisKeyTreeIndex {
|
||||
root: RedisKeyTreeNode[];
|
||||
groupById: Map<string, RedisKeyTreeGroupNode>;
|
||||
keyRaws: Set<string>;
|
||||
ancestorGroupIdsByKeyRaw: Map<string, readonly string[]>;
|
||||
}
|
||||
|
||||
export interface AppendRedisKeysResult {
|
||||
addedGroupIds: Set<string>;
|
||||
}
|
||||
|
||||
export function redisKeyNameCopyText(node: RedisKeyTreeNode): string | null {
|
||||
// keyRaw is base64-encoded for backend roundtrips; copy the user-visible
|
||||
// Redis key name instead of the internal transport value.
|
||||
@@ -37,7 +58,7 @@ export function redisKeyNameCopyText(node: RedisKeyTreeNode): string | null {
|
||||
}
|
||||
|
||||
function buildGroupId(db: number, pathSegments: string[]): string {
|
||||
return `group:${db}:${pathSegments.join("\u0000")}`;
|
||||
return `group:${db}:${JSON.stringify(pathSegments)}`;
|
||||
}
|
||||
|
||||
function buildLeafId(db: number, keyRaw: string): string {
|
||||
@@ -70,35 +91,73 @@ function compareRedisTreeNodes(a: RedisKeyTreeNode, b: RedisKeyTreeNode): number
|
||||
return a.label.localeCompare(b.label);
|
||||
}
|
||||
|
||||
function sortRedisTreeNodes(nodes: RedisKeyTreeNode[]): RedisKeyTreeNode[] {
|
||||
return [...nodes].sort(compareRedisTreeNodes).map((node) =>
|
||||
node.kind === "group"
|
||||
? {
|
||||
...node,
|
||||
children: sortRedisTreeNodes(node.children),
|
||||
}
|
||||
: node,
|
||||
);
|
||||
function redisKeyInfoFromLeaf(node: RedisKeyTreeLeafNode): RedisKeyInfo {
|
||||
return {
|
||||
key_display: node.fullKeyDisplay,
|
||||
key_raw: node.keyRaw,
|
||||
key_type: node.keyType,
|
||||
ttl: node.ttl,
|
||||
size: node.size,
|
||||
value_preview: node.valuePreview,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildRedisKeyTree(keys: RedisKeyInfo[], db: number, separator = ":"): RedisKeyTreeNode[] {
|
||||
const root: RedisKeyTreeNode[] = [];
|
||||
const groupMap = new Map<string, RedisKeyTreeGroupNode>();
|
||||
export function createRedisKeyTreeIndex(keys: readonly RedisKeyInfo[], db: number, separator = ":"): RedisKeyTreeIndex {
|
||||
const index: RedisKeyTreeIndex = {
|
||||
root: [],
|
||||
groupById: new Map(),
|
||||
keyRaws: new Set(),
|
||||
ancestorGroupIdsByKeyRaw: new Map(),
|
||||
};
|
||||
appendRedisKeysToTreeIndex(index, keys, db, separator);
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts one SCAN batch while sorting only sibling arrays changed by that
|
||||
* batch. This keeps repeated "load more" merges from re-sorting the full tree.
|
||||
*/
|
||||
export function appendRedisKeysToTreeIndex(index: RedisKeyTreeIndex, keys: readonly RedisKeyInfo[], db: number, separator = ":"): AppendRedisKeysResult {
|
||||
const touchedLevels = new Set<RedisKeyTreeNode[]>();
|
||||
const addedGroupIds = new Set<string>();
|
||||
|
||||
for (const key of keys) {
|
||||
insertKeyIntoTree(root, groupMap, key, db, separator);
|
||||
}
|
||||
if (index.keyRaws.has(key.key_raw)) continue;
|
||||
|
||||
return sortRedisTreeNodes(root);
|
||||
}
|
||||
const pathSegments = separator ? key.key_display.split(separator) : [key.key_display];
|
||||
const ancestorGroupIds: string[] = [];
|
||||
let currentLevel = index.root;
|
||||
|
||||
function insertKeyIntoTree(root: RedisKeyTreeNode[], groupMap: Map<string, RedisKeyTreeGroupNode>, key: RedisKeyInfo, db: number, separator: string): void {
|
||||
const pathSegments = separator ? key.key_display.split(separator) : [key.key_display];
|
||||
if (pathSegments.length === 1) {
|
||||
root.push({
|
||||
if (pathSegments.length > 1) {
|
||||
const groupSegments: string[] = [];
|
||||
for (const segment of pathSegments.slice(0, -1)) {
|
||||
groupSegments.push(segment);
|
||||
const groupId = buildGroupId(db, groupSegments);
|
||||
let group = index.groupById.get(groupId);
|
||||
if (!group) {
|
||||
group = {
|
||||
kind: "group",
|
||||
id: groupId,
|
||||
label: segment,
|
||||
pathSegments: [...groupSegments],
|
||||
children: [],
|
||||
loadedLeafCount: 0,
|
||||
};
|
||||
index.groupById.set(groupId, group);
|
||||
currentLevel.push(group);
|
||||
touchedLevels.add(currentLevel);
|
||||
addedGroupIds.add(groupId);
|
||||
}
|
||||
group.loadedLeafCount++;
|
||||
ancestorGroupIds.push(groupId);
|
||||
currentLevel = group.children;
|
||||
}
|
||||
}
|
||||
|
||||
currentLevel.push({
|
||||
kind: "leaf",
|
||||
id: buildLeafId(db, key.key_raw),
|
||||
label: pathSegments[0],
|
||||
label: pathSegments[pathSegments.length - 1],
|
||||
fullKeyDisplay: key.key_display,
|
||||
keyRaw: key.key_raw,
|
||||
db,
|
||||
@@ -108,113 +167,62 @@ function insertKeyIntoTree(root: RedisKeyTreeNode[], groupMap: Map<string, Redis
|
||||
valuePreview: key.value_preview ?? "",
|
||||
pathSegments,
|
||||
});
|
||||
return;
|
||||
touchedLevels.add(currentLevel);
|
||||
index.keyRaws.add(key.key_raw);
|
||||
index.ancestorGroupIdsByKeyRaw.set(key.key_raw, ancestorGroupIds);
|
||||
}
|
||||
|
||||
let currentLevel = root;
|
||||
const groupSegments: string[] = [];
|
||||
for (const segment of pathSegments.slice(0, -1)) {
|
||||
groupSegments.push(segment);
|
||||
const groupId = buildGroupId(db, groupSegments);
|
||||
let group = groupMap.get(groupId);
|
||||
if (!group) {
|
||||
group = {
|
||||
kind: "group",
|
||||
id: groupId,
|
||||
label: segment,
|
||||
pathSegments: [...groupSegments],
|
||||
children: [],
|
||||
};
|
||||
groupMap.set(groupId, group);
|
||||
currentLevel.push(group);
|
||||
}
|
||||
currentLevel = group.children;
|
||||
}
|
||||
|
||||
currentLevel.push({
|
||||
kind: "leaf",
|
||||
id: buildLeafId(db, key.key_raw),
|
||||
label: pathSegments[pathSegments.length - 1],
|
||||
fullKeyDisplay: key.key_display,
|
||||
keyRaw: key.key_raw,
|
||||
db,
|
||||
keyType: key.key_type ?? "",
|
||||
ttl: key.ttl ?? -2,
|
||||
size: key.size ?? 0,
|
||||
valuePreview: key.value_preview ?? "",
|
||||
pathSegments,
|
||||
});
|
||||
for (const nodes of touchedLevels) nodes.sort(compareRedisTreeNodes);
|
||||
return { addedGroupIds };
|
||||
}
|
||||
|
||||
function rebuildGroupMap(tree: RedisKeyTreeNode[]): Map<string, RedisKeyTreeGroupNode> {
|
||||
const groupMap = new Map<string, RedisKeyTreeGroupNode>();
|
||||
|
||||
const walk = (nodes: RedisKeyTreeNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (node.kind === "group") {
|
||||
groupMap.set(node.id, node);
|
||||
walk(node.children);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
walk(tree);
|
||||
return groupMap;
|
||||
export function buildRedisKeyTree(keys: RedisKeyInfo[], db: number, separator = ":"): RedisKeyTreeNode[] {
|
||||
return createRedisKeyTreeIndex(keys, db, separator).root;
|
||||
}
|
||||
|
||||
export function mergeKeysIntoRedisKeyTree(existingTree: RedisKeyTreeNode[], newKeys: RedisKeyInfo[], db: number, separator = ":"): RedisKeyTreeNode[] {
|
||||
if (existingTree.length === 0) return buildRedisKeyTree(newKeys, db, separator);
|
||||
|
||||
const groupMap = rebuildGroupMap(existingTree);
|
||||
const existingKeyIds = new Set<string>();
|
||||
const collectKeys = (nodes: RedisKeyTreeNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (node.kind === "leaf") {
|
||||
existingKeyIds.add(node.keyRaw);
|
||||
} else {
|
||||
collectKeys(node.children);
|
||||
}
|
||||
const existingKeys: RedisKeyInfo[] = [];
|
||||
const stack = [...existingTree];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop()!;
|
||||
if (node.kind === "leaf") {
|
||||
existingKeys.push(redisKeyInfoFromLeaf(node));
|
||||
} else {
|
||||
for (const child of node.children) stack.push(child);
|
||||
}
|
||||
};
|
||||
collectKeys(existingTree);
|
||||
|
||||
for (const key of newKeys) {
|
||||
if (existingKeyIds.has(key.key_raw)) continue;
|
||||
insertKeyIntoTree(existingTree, groupMap, key, db, separator);
|
||||
}
|
||||
|
||||
return sortRedisTreeNodes(existingTree);
|
||||
const index = createRedisKeyTreeIndex(existingKeys, db, separator);
|
||||
appendRedisKeysToTreeIndex(index, newKeys, db, separator);
|
||||
return index.root;
|
||||
}
|
||||
|
||||
export function collectExpandedGroupIds(nodes: RedisKeyTreeNode[]): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
|
||||
const visit = (entries: RedisKeyTreeNode[]) => {
|
||||
for (const node of entries) {
|
||||
if (node.kind !== "group") continue;
|
||||
ids.add(node.id);
|
||||
visit(node.children);
|
||||
}
|
||||
};
|
||||
|
||||
visit(nodes);
|
||||
const stack = [...nodes];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop()!;
|
||||
if (node.kind !== "group") continue;
|
||||
ids.add(node.id);
|
||||
for (const child of node.children) stack.push(child);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
export function collectRedisGroupKeyRaws(group: RedisKeyTreeGroupNode): string[] {
|
||||
const keyRaws: string[] = [];
|
||||
|
||||
const visit = (nodes: RedisKeyTreeNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (node.kind === "leaf") {
|
||||
keyRaws.push(node.keyRaw);
|
||||
} else {
|
||||
visit(node.children);
|
||||
const stack = [...group.children].reverse();
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop()!;
|
||||
if (node.kind === "leaf") {
|
||||
keyRaws.push(node.keyRaw);
|
||||
} else {
|
||||
// Push in reverse so the iterative walk keeps the rendered tree order.
|
||||
for (let index = node.children.length - 1; index >= 0; index--) {
|
||||
stack.push(node.children[index]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
visit(group.children);
|
||||
}
|
||||
return keyRaws;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user