From de39c1662db75143893c170279257e3b2378c852 Mon Sep 17 00:00:00 2001 From: miracle Date: Sat, 27 Jun 2026 15:02:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20ZK=20=E8=8A=82=E7=82=B9=E6=A0=91?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E7=9C=9F=E6=AD=A3=E6=87=92=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?=20(#1987)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: lazy load zookeeper nodes * fix: preserve zookeeper focused path hierarchy --- .../agent/zookeeper/ZooKeeperAgentTest.java | 23 ++ .../src/components/kv/KvKeyBrowser.vue | 294 +++++++++++++++--- .../zookeeper/ZooKeeperKeyBrowser.vue | 2 +- .../__tests__/zookeeperApiContract.spec.ts | 6 +- .../__tests__/zookeeperLazyKeyTree.spec.ts | 45 +++ apps/desktop/src/lib/api.ts | 1 + apps/desktop/src/lib/http.ts | 5 +- apps/desktop/src/lib/tauri.ts | 8 +- apps/desktop/src/lib/zookeeperLazyKeyTree.ts | 187 +++++++++++ crates/dbx-core/src/agent_kv.rs | 44 ++- crates/dbx-web/src/routes/zookeeper.rs | 4 +- src-tauri/src/commands/zookeeper_cmd.rs | 11 +- 12 files changed, 571 insertions(+), 59 deletions(-) create mode 100644 apps/desktop/src/lib/__tests__/zookeeperLazyKeyTree.spec.ts create mode 100644 apps/desktop/src/lib/zookeeperLazyKeyTree.ts diff --git a/agents/drivers/zookeeper/src/test/java/com/dbx/agent/zookeeper/ZooKeeperAgentTest.java b/agents/drivers/zookeeper/src/test/java/com/dbx/agent/zookeeper/ZooKeeperAgentTest.java index 94ba12250b..83ea7d8dd4 100644 --- a/agents/drivers/zookeeper/src/test/java/com/dbx/agent/zookeeper/ZooKeeperAgentTest.java +++ b/agents/drivers/zookeeper/src/test/java/com/dbx/agent/zookeeper/ZooKeeperAgentTest.java @@ -465,6 +465,29 @@ final class ZooKeeperAgentTest { } } + @Test + void listPrefixPaginatesDirectChildrenWhenRecursiveIsFalse() throws Exception { + try (TestingServer server = new TestingServer()) { + connect(server); + result(request(2, "kv_put", "{\"key\":\"/app/c/deep\",\"value\":{\"encoding\":\"utf8\",\"data\":\"c\"}}")); + result(request(3, "kv_put", "{\"key\":\"/app/a/deep\",\"value\":{\"encoding\":\"utf8\",\"data\":\"a\"}}")); + result(request(4, "kv_put", "{\"key\":\"/app/b/deep\",\"value\":{\"encoding\":\"utf8\",\"data\":\"b\"}}")); + + JsonObject first = result(request(5, "kv_list_prefix", "{\"prefix\":\"/app\",\"recursive\":false,\"limit\":2}")); + String continuation = first.get("continuation").getAsString(); + JsonObject second = result(request( + 6, + "kv_list_prefix", + "{\"prefix\":\"/app\",\"recursive\":false,\"limit\":2,\"continuation\":\"" + continuation + "\"}" + )); + + Assertions.assertEquals(List.of("/app/a", "/app/b"), listedKeys(first)); + Assertions.assertFalse(continuation.isBlank()); + Assertions.assertEquals(List.of("/app/c"), listedKeys(second)); + Assertions.assertTrue(second.get("continuation").isJsonNull()); + } + } + @Test void listPrefixRejectsContinuationForDifferentRequest() throws Exception { try (TestingServer server = new TestingServer()) { diff --git a/apps/desktop/src/components/kv/KvKeyBrowser.vue b/apps/desktop/src/components/kv/KvKeyBrowser.vue index 581bbecedd..ee9de8c7b4 100644 --- a/apps/desktop/src/components/kv/KvKeyBrowser.vue +++ b/apps/desktop/src/components/kv/KvKeyBrowser.vue @@ -1,5 +1,5 @@ diff --git a/apps/desktop/src/lib/__tests__/zookeeperApiContract.spec.ts b/apps/desktop/src/lib/__tests__/zookeeperApiContract.spec.ts index 3954c0b1f6..419b673be5 100644 --- a/apps/desktop/src/lib/__tests__/zookeeperApiContract.spec.ts +++ b/apps/desktop/src/lib/__tests__/zookeeperApiContract.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { zookeeperDelete, zookeeperGet, zookeeperListPrefix, zookeeperPut, type KvPutOptions, type KvPutResponse } from "../api"; +import { zookeeperDelete, zookeeperGet, zookeeperListPrefix, zookeeperPut, type KvListPrefixOptions, type KvPutOptions, type KvPutResponse } from "../api"; import type { DatabaseType } from "@/types/database"; describe("zookeeper frontend API contract", () => { @@ -13,9 +13,13 @@ describe("zookeeper frontend API contract", () => { key: "/sessions/member-0000000001", createdKey: "/sessions/member-0000000001", }; + const listOptions: KvListPrefixOptions = { + recursive: false, + }; expect(dbType).toBe("zookeeper"); expect(options.createMode).toBe("ephemeral_sequential"); + expect(listOptions.recursive).toBe(false); expect(response.createdKey).toBe(response.key); expect(typeof zookeeperListPrefix).toBe("function"); expect(typeof zookeeperGet).toBe("function"); diff --git a/apps/desktop/src/lib/__tests__/zookeeperLazyKeyTree.spec.ts b/apps/desktop/src/lib/__tests__/zookeeperLazyKeyTree.spec.ts new file mode 100644 index 0000000000..363401240a --- /dev/null +++ b/apps/desktop/src/lib/__tests__/zookeeperLazyKeyTree.spec.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { createLazyKvKeyTreeState, flattenLazyKvKeyTree, replaceLazyKvChildren, resetLazyKvKeyTree } from "../zookeeperLazyKeyTree"; + +describe("zookeeper lazy key tree", () => { + it("stores only the current path direct children on reset", () => { + const state = createLazyKvKeyTreeState(); + + resetLazyKvKeyTree(state, "/app"); + replaceLazyKvChildren( + state, + null, + [ + { key: "/app/a", numChildren: 0, valueSize: 1 }, + { key: "/app/folder", numChildren: 2, valueSize: 0 }, + ], + null, + ); + + expect(state.rootPath).toBe("/app"); + expect(state.roots.map((node) => node.key)).toEqual(["/app/a", "/app/folder"]); + expect(flattenLazyKvKeyTree(state, new Set()).map((row) => `${row.depth}:${row.node.label}`)).toEqual(["0:a", "0:folder"]); + }); + + it("does not expose grandchildren until their parent is loaded", () => { + const state = createLazyKvKeyTreeState("/"); + replaceLazyKvChildren(state, null, [{ key: "/app", numChildren: 1 }], null); + + expect(flattenLazyKvKeyTree(state, new Set(["lazy:/app"])).map((row) => row.node.key)).toEqual(["/app"]); + + replaceLazyKvChildren(state, "/app", [{ key: "/app/name", numChildren: 0 }], null); + + expect(flattenLazyKvKeyTree(state, new Set(["lazy:/app"])).map((row) => row.node.key)).toEqual(["/app", "/app/name"]); + }); + + it("keeps child pagination continuation on the owning node", () => { + const state = createLazyKvKeyTreeState("/"); + replaceLazyKvChildren(state, null, [{ key: "/app", numChildren: 3 }], "root-next"); + replaceLazyKvChildren(state, "/app", [{ key: "/app/a", numChildren: 0 }], "child-next"); + + const app = state.nodeByKey.get("/app"); + + expect(state.rootContinuation).toBe("root-next"); + expect(app?.continuation).toBe("child-next"); + }); +}); diff --git a/apps/desktop/src/lib/api.ts b/apps/desktop/src/lib/api.ts index 2e73bad3e3..87e56c3d22 100644 --- a/apps/desktop/src/lib/api.ts +++ b/apps/desktop/src/lib/api.ts @@ -443,6 +443,7 @@ export type { KvKeyMetadata, KvKeySummary, KvListPrefixResponse, + KvListPrefixOptions, KvGetResponse, KvWriteMode, KvCreateMode, diff --git a/apps/desktop/src/lib/http.ts b/apps/desktop/src/lib/http.ts index a25b3383bd..9c4aa01e1a 100644 --- a/apps/desktop/src/lib/http.ts +++ b/apps/desktop/src/lib/http.ts @@ -58,6 +58,7 @@ import type { RedisNodeEndpoint, KvValue, KvListPrefixResponse, + KvListPrefixOptions, KvGetResponse, KvPutOptions, KvPutResponse, @@ -1602,8 +1603,8 @@ export async function etcdDelete(connectionId: string, key: string): Promise { - return post("/api/zookeeper/list-prefix", { connectionId, prefix, limit, continuation }); +export async function zookeeperListPrefix(connectionId: string, prefix: string, limit: number, continuation?: string | null, options?: KvListPrefixOptions | null): Promise { + return post("/api/zookeeper/list-prefix", { connectionId, prefix, limit, continuation, recursive: options?.recursive ?? null }); } export async function zookeeperGet(connectionId: string, key: string): Promise { diff --git a/apps/desktop/src/lib/tauri.ts b/apps/desktop/src/lib/tauri.ts index a44769a393..779f2f2259 100644 --- a/apps/desktop/src/lib/tauri.ts +++ b/apps/desktop/src/lib/tauri.ts @@ -1346,6 +1346,10 @@ export interface KvListPrefixResponse { revision?: number | null; } +export interface KvListPrefixOptions { + recursive?: boolean | null; +} + export interface KvGetResponse { found: boolean; key?: string | null; @@ -1391,8 +1395,8 @@ export async function etcdDelete(connectionId: string, key: string): Promise { - return invoke("zookeeper_list_prefix", { connectionId, prefix, limit, continuation }); +export async function zookeeperListPrefix(connectionId: string, prefix: string, limit: number, continuation?: string | null, options?: KvListPrefixOptions | null): Promise { + return invoke("zookeeper_list_prefix", { connectionId, prefix, limit, continuation, recursive: options?.recursive ?? null }); } export async function zookeeperGet(connectionId: string, key: string): Promise { diff --git a/apps/desktop/src/lib/zookeeperLazyKeyTree.ts b/apps/desktop/src/lib/zookeeperLazyKeyTree.ts new file mode 100644 index 0000000000..b4b6a746f4 --- /dev/null +++ b/apps/desktop/src/lib/zookeeperLazyKeyTree.ts @@ -0,0 +1,187 @@ +import type { KvKeyMetadata, KvKeySummary } from "./api"; + +export interface LazyKvKeyTreeNode extends KvKeyMetadata { + kind: "lazy"; + id: string; + label: string; + key: string; + pathSegments: string[]; + hasChildren: boolean; + children: LazyKvKeyTreeNode[]; + loaded: boolean; + loading: boolean; + continuation: string | null; +} + +export interface LazyKvKeyTreeState { + rootPath: string; + roots: LazyKvKeyTreeNode[]; + rootContinuation: string | null; + nodeByKey: Map; +} + +export type LazyKvKeyTreeRow = + | { + type: "node"; + node: LazyKvKeyTreeNode; + depth: number; + } + | { + type: "loadMore"; + id: string; + parentKey: string | null; + depth: number; + loading: boolean; + }; + +export function normalizeZooKeeperPath(path: string): string { + const trimmed = path.trim(); + if (!trimmed || trimmed === "/") return "/"; + const withSlash = trimmed.startsWith("/") ? trimmed : `/${trimmed}`; + return withSlash.replace(/\/+$/, "") || "/"; +} + +export function parentZooKeeperPath(path: string): string { + const normalized = normalizeZooKeeperPath(path); + if (normalized === "/") return "/"; + const parent = normalized.slice(0, normalized.lastIndexOf("/")); + return parent || "/"; +} + +export function createLazyKvKeyTreeState(rootPath = "/"): LazyKvKeyTreeState { + return { + rootPath: normalizeZooKeeperPath(rootPath), + roots: [], + rootContinuation: null, + nodeByKey: new Map(), + }; +} + +export function resetLazyKvKeyTree(state: LazyKvKeyTreeState, rootPath = "/") { + state.rootPath = normalizeZooKeeperPath(rootPath); + state.roots = []; + state.rootContinuation = null; + state.nodeByKey.clear(); +} + +export function replaceLazyKvChildren(state: LazyKvKeyTreeState, parentKey: string | null, keys: KvKeySummary[], continuation: string | null | undefined, options: { append?: boolean } = {}) { + const previous = new Map(state.nodeByKey); + const nextChildren = keys.map((key) => lazyNodeFromSummary(key, previous.get(key.key))); + const children = options.append ? mergeLazyChildren(parentKey ? state.nodeByKey.get(parentKey)?.children || [] : state.roots, nextChildren) : nextChildren; + + if (parentKey) { + const parent = state.nodeByKey.get(parentKey); + if (!parent) return; + parent.children = children; + parent.loaded = true; + parent.continuation = continuation || null; + } else { + state.roots = children; + state.rootContinuation = continuation || null; + } + + rebuildLazyNodeIndex(state); +} + +export function replaceLazyKvFocusedRoot(state: LazyKvKeyTreeState, root: KvKeySummary, keys: KvKeySummary[], continuation: string | null | undefined) { + const previous = new Map(state.nodeByKey); + const focusedPath = normalizeZooKeeperPath(root.key); + const childNodes = keys.map((key) => lazyNodeFromSummary(key, previous.get(key.key))); + const chain = focusedPathChain(focusedPath); + let rootNode: LazyKvKeyTreeNode | null = null; + let parentNode: LazyKvKeyTreeNode | null = null; + + for (const path of chain) { + const node = lazyNodeFromSummary(path === focusedPath ? { ...root, key: focusedPath } : { key: path, numChildren: 1 }, previous.get(path)); + node.hasChildren = true; + node.loaded = true; + node.continuation = null; + node.children = []; + + if (parentNode) parentNode.children = [node]; + else rootNode = node; + parentNode = node; + } + + if (parentNode) { + parentNode.children = childNodes; + parentNode.hasChildren = parentNode.hasChildren || childNodes.length > 0 || !!continuation; + parentNode.continuation = continuation || null; + } + + state.roots = rootNode ? [rootNode] : []; + state.rootContinuation = null; + rebuildLazyNodeIndex(state); +} + +export function flattenLazyKvKeyTree(state: LazyKvKeyTreeState, expandedIds: ReadonlySet): LazyKvKeyTreeRow[] { + const rows = flattenLazyNodes(state.roots, expandedIds, 0); + if (state.rootContinuation) { + rows.push({ type: "loadMore", id: "lazy-load-more:root", parentKey: null, depth: 0, loading: false }); + } + return rows; +} + +export function lazyExpandedKeyFromId(id: string): string | null { + return id.startsWith("lazy:") ? id.slice("lazy:".length) : null; +} + +function flattenLazyNodes(nodes: LazyKvKeyTreeNode[], expandedIds: ReadonlySet, depth: number): LazyKvKeyTreeRow[] { + const rows: LazyKvKeyTreeRow[] = []; + for (const node of nodes) { + rows.push({ type: "node", node, depth }); + if (node.hasChildren && expandedIds.has(node.id)) { + rows.push(...flattenLazyNodes(node.children, expandedIds, depth + 1)); + if (node.continuation) { + rows.push({ type: "loadMore", id: `lazy-load-more:${node.key}`, parentKey: node.key, depth: depth + 1, loading: node.loading }); + } + } + } + return rows; +} + +function lazyNodeFromSummary(summary: KvKeySummary, previous?: LazyKvKeyTreeNode): LazyKvKeyTreeNode { + const pathSegments = keySegments(summary.key); + return { + ...summary, + kind: "lazy", + id: `lazy:${summary.key}`, + label: pathSegments[pathSegments.length - 1] || summary.key || "/", + key: summary.key, + pathSegments, + hasChildren: (summary.numChildren ?? 0) > 0, + children: previous?.children || [], + loaded: previous?.loaded || false, + loading: previous?.loading || false, + continuation: previous?.continuation || null, + }; +} + +function mergeLazyChildren(existing: LazyKvKeyTreeNode[], incoming: LazyKvKeyTreeNode[]): LazyKvKeyTreeNode[] { + const seen = new Set(existing.map((node) => node.key)); + return [...existing, ...incoming.filter((node) => !seen.has(node.key))]; +} + +function rebuildLazyNodeIndex(state: LazyKvKeyTreeState) { + state.nodeByKey.clear(); + const visit = (nodes: LazyKvKeyTreeNode[]) => { + for (const node of nodes) { + state.nodeByKey.set(node.key, node); + visit(node.children); + } + }; + visit(state.roots); +} + +function keySegments(key: string): string[] { + return key.split("/").filter(Boolean); +} + +function focusedPathChain(path: string): string[] { + const segments = keySegments(path); + const chain: string[] = []; + for (let index = 0; index < segments.length; index++) { + chain.push(`/${segments.slice(0, index + 1).join("/")}`); + } + return chain; +} diff --git a/crates/dbx-core/src/agent_kv.rs b/crates/dbx-core/src/agent_kv.rs index 530b391d9d..8b279c65a5 100644 --- a/crates/dbx-core/src/agent_kv.rs +++ b/crates/dbx-core/src/agent_kv.rs @@ -52,6 +52,8 @@ pub struct KvListPrefixRequest { pub limit: usize, #[serde(skip_serializing_if = "Option::is_none")] pub continuation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub recursive: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -140,10 +142,20 @@ pub struct KvDeleteResponse { } pub fn kv_list_prefix_params(prefix: &str, limit: usize, continuation: Option<&str>) -> serde_json::Value { + kv_list_prefix_params_with_options(prefix, limit, continuation, None) +} + +pub fn kv_list_prefix_params_with_options( + prefix: &str, + limit: usize, + continuation: Option<&str>, + recursive: Option, +) -> serde_json::Value { serde_json::to_value(KvListPrefixRequest { prefix: prefix.to_string(), limit, continuation: continuation.map(str::to_string), + recursive, }) .expect("KV list prefix request should serialize") } @@ -178,8 +190,24 @@ pub async fn kv_list_prefix_core( limit: usize, continuation: Option<&str>, ) -> Result { - call_agent_kv(state, connection_id, AgentKvMethod::ListPrefix, kv_list_prefix_params(prefix, limit, continuation)) - .await + kv_list_prefix_core_with_options(state, connection_id, prefix, limit, continuation, None).await +} + +pub async fn kv_list_prefix_core_with_options( + state: &AppState, + connection_id: &str, + prefix: &str, + limit: usize, + continuation: Option<&str>, + recursive: Option, +) -> Result { + call_agent_kv( + state, + connection_id, + AgentKvMethod::ListPrefix, + kv_list_prefix_params_with_options(prefix, limit, continuation, recursive), + ) + .await } pub async fn kv_get_core(state: &AppState, connection_id: &str, key: &str) -> Result { @@ -259,6 +287,18 @@ mod tests { ); } + #[test] + fn serializes_kv_list_prefix_params_with_recursive_false() { + assert_eq!( + kv_list_prefix_params_with_options("/app", 200, None, Some(false)), + serde_json::json!({ + "prefix": "/app", + "limit": 200, + "recursive": false + }) + ); + } + #[test] fn serializes_kv_get_put_delete_params() { assert_eq!(kv_get_params("/app/name"), serde_json::json!({ "key": "/app/name" })); diff --git a/crates/dbx-web/src/routes/zookeeper.rs b/crates/dbx-web/src/routes/zookeeper.rs index f2e175530d..bfac64dbed 100644 --- a/crates/dbx-web/src/routes/zookeeper.rs +++ b/crates/dbx-web/src/routes/zookeeper.rs @@ -28,6 +28,7 @@ pub struct ZooKeeperListPrefixRequest { pub prefix: String, pub limit: usize, pub continuation: Option, + pub recursive: Option, } #[derive(Deserialize)] @@ -50,12 +51,13 @@ pub async fn list_prefix( State(state): State>, Json(req): Json, ) -> Result, AppError> { - let result = dbx_core::agent_kv::kv_list_prefix_core( + let result = dbx_core::agent_kv::kv_list_prefix_core_with_options( &state.app, &req.connection_id, &req.prefix, req.limit, req.continuation.as_deref(), + req.recursive, ) .await .map_err(AppError)?; diff --git a/src-tauri/src/commands/zookeeper_cmd.rs b/src-tauri/src/commands/zookeeper_cmd.rs index da3a324e9f..8956a2ada6 100644 --- a/src-tauri/src/commands/zookeeper_cmd.rs +++ b/src-tauri/src/commands/zookeeper_cmd.rs @@ -12,8 +12,17 @@ pub async fn zookeeper_list_prefix( prefix: String, limit: usize, continuation: Option, + recursive: Option, ) -> Result { - dbx_core::agent_kv::kv_list_prefix_core(&state, &connection_id, &prefix, limit, continuation.as_deref()).await + dbx_core::agent_kv::kv_list_prefix_core_with_options( + &state, + &connection_id, + &prefix, + limit, + continuation.as_deref(), + recursive, + ) + .await } #[tauri::command]