From c7c266c309f0e0d278f54882d10e5b5340372d3f Mon Sep 17 00:00:00 2001 From: Lan-zk <68137514+Lan-zk@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:17:58 +0800 Subject: [PATCH] perf(search): batch column fetch and multi-table probe --- .../components/search/DatabaseSearchPanel.vue | 126 ++++++++- .../DatabaseSearchPanel.flow.spec.ts | 261 +++++++++++++++++ apps/desktop/src/lib/backend/api.ts | 2 + apps/desktop/src/lib/backend/http.ts | 8 + apps/desktop/src/lib/backend/tauri.ts | 8 + .../databaseSearchCapabilities.spec.ts | 18 ++ .../__tests__/databaseSearchColumns.spec.ts | 119 ++++++++ .../__tests__/databaseSearchProbe.spec.ts | 106 +++++++ .../src/lib/database/databaseSearch.ts | 11 +- .../database/databaseSearchCapabilities.ts | 14 + .../src/lib/database/databaseSearchColumns.ts | 97 +++++++ .../src/lib/database/databaseSearchProbe.ts | 95 +++++++ crates/dbx-core/src/lib.rs | 2 + .../src/database_search_columns_sql.rs | 248 ++++++++++++++++ .../src/database_search_probe_sql.rs | 267 ++++++++++++++++++ .../dbx-sql-data/src/database_search_sql.rs | 130 ++++++++- .../src/database_structure_search_sql.rs | 6 +- crates/dbx-sql-data/src/lib.rs | 2 + crates/dbx-sql/src/lib.rs | 3 +- crates/dbx-web/src/main.rs | 2 + crates/dbx-web/src/routes/query.rs | 24 ++ src-tauri/src/commands/query.rs | 14 + src-tauri/src/lib.rs | 2 + 23 files changed, 1533 insertions(+), 32 deletions(-) create mode 100644 apps/desktop/src/components/search/__tests__/DatabaseSearchPanel.flow.spec.ts create mode 100644 apps/desktop/src/lib/database/__tests__/databaseSearchCapabilities.spec.ts create mode 100644 apps/desktop/src/lib/database/__tests__/databaseSearchColumns.spec.ts create mode 100644 apps/desktop/src/lib/database/__tests__/databaseSearchProbe.spec.ts create mode 100644 apps/desktop/src/lib/database/databaseSearchCapabilities.ts create mode 100644 apps/desktop/src/lib/database/databaseSearchColumns.ts create mode 100644 apps/desktop/src/lib/database/databaseSearchProbe.ts create mode 100644 crates/dbx-sql-data/src/database_search_columns_sql.rs create mode 100644 crates/dbx-sql-data/src/database_search_probe_sql.rs diff --git a/apps/desktop/src/components/search/DatabaseSearchPanel.vue b/apps/desktop/src/components/search/DatabaseSearchPanel.vue index 1946d86b42..084c3a1286 100644 --- a/apps/desktop/src/components/search/DatabaseSearchPanel.vue +++ b/apps/desktop/src/components/search/DatabaseSearchPanel.vue @@ -12,7 +12,10 @@ import { useConnectionStore } from "@/stores/connectionStore"; import * as api from "@/lib/backend/api"; import { buildDatabaseSearchSql, buildSearchResultWhere, findMatchedSearchColumns } from "@/lib/database/databaseSearch"; import { databaseSearchBatchRange, databaseSearchNextBatchSize } from "@/lib/database/databaseSearchBatch"; -import type { DatabaseSearchResultItem, DatabaseSearchScope, DatabaseSearchTableTask, DatabaseSearchTabState, DatabaseType, TableInfo } from "@/types/database"; +import { fetchDatabaseSearchColumns, findSearchColumns, type DatabaseSearchColumnLite, type SearchColumnsByTable } from "@/lib/database/databaseSearchColumns"; +import { nextProbeChunkEnd, probeTables } from "@/lib/database/databaseSearchProbe"; +import { supportsBulkDatabaseSearch } from "@/lib/database/databaseSearchCapabilities"; +import type { ColumnInfo, DatabaseSearchResultItem, DatabaseSearchScope, DatabaseSearchTableTask, DatabaseSearchTabState, DatabaseType, TableInfo } from "@/types/database"; import { buildDatabaseStructureSearchSql, effectiveDatabaseSearchScope, runDatabaseStructureSearch, structureHitPreview, supportsDatabaseStructureSearchFromManifest } from "@/lib/database/databaseStructureSearch"; import { usesDocumentStructureSearch, usesSqlCatalogStructureSearch } from "@/lib/database/databaseStructureSearchCapabilities"; import type { NavigationTarget } from "@/composables/useNavigationTargets"; @@ -60,6 +63,8 @@ const generalError = ref(props.initialState?.generalError ?? ""); const currentExecutionId = ref(""); const tableTasks = ref(props.initialState?.tableTasks ? [...props.initialState.tableTasks] : []); const nextTableIndex = ref(props.initialState?.nextTableIndex ?? 0); +// 批量取列结果(键小写)。不进持久化 tab state:组件存活期间复用,重建后为空走逐表回退。 +const columnsByTable = ref(new Map()); const activeKeyword = ref(props.initialState?.activeKeyword ?? ""); const activePerTableLimit = ref(props.initialState?.activePerTableLimit ?? 20); let runId = 0; @@ -132,6 +137,7 @@ function resetSearchState() { currentExecutionId.value = ""; tableTasks.value = []; nextTableIndex.value = 0; + columnsByTable.value = new Map(); activeKeyword.value = ""; activePerTableLimit.value = 20; emitState(); @@ -226,6 +232,8 @@ async function startSearch() { progressTotal.value = tableTasks.value.length; loadingTables.value = false; emitState(); + await loadBulkColumns(currentRun); + if (currentRun !== runId || cancelled.value) return; await scanTableRange(currentRun, false); } catch (error: any) { if (currentRun === runId && !cancelled.value) { @@ -333,25 +341,125 @@ async function runStructureSearch(currentRun: number) { } } +/** 批量取列:按任务表涉及的 schema 各发一条目录查询,失败静默(空 Map → 逐表回退)。 */ +async function loadBulkColumns(currentRun: number) { + const databaseType = connection.value?.db_type; + if (!databaseType || !supportsBulkDatabaseSearch(databaseType)) return; + const schemas = [...new Set(tableTasks.value.map((task) => task.schema))]; + const merged: SearchColumnsByTable = new Map(); + for (const schema of schemas) { + if (currentRun !== runId || cancelled.value) return; + const executionId = makeExecutionId(); + currentExecutionId.value = executionId; + const partial = await fetchDatabaseSearchColumns({ + databaseType, + database: props.database, + schema, + executeQuery: (sql, runExecutionId) => api.executeQuery(props.connectionId, props.database, sql, undefined, runExecutionId), + executionId, + }); + if (currentExecutionId.value === executionId) currentExecutionId.value = ""; + if (currentRun !== runId || cancelled.value) return; + partial.forEach((columns, key) => merged.set(key, columns)); + } + columnsByTable.value = merged; +} + async function scanTableRange(currentRun: number, scanAllRemaining: boolean) { const range = databaseSearchBatchRange(nextTableIndex.value, tableTasks.value.length, scanAllRemaining); + const databaseType = connection.value!.db_type; + const probeReady = supportsBulkDatabaseSearch(databaseType) && columnsByTable.value.size > 0; + // 每张任务的预取列(可能为 undefined——该表无目录信息,批内走逐表回退) + const taskColumns = tableTasks.value.map((task) => findSearchColumns(columnsByTable.value, task.schema, props.database, task.table.name)); + while (nextTableIndex.value < range.end) { if (currentRun !== runId || cancelled.value) break; - const task = tableTasks.value[nextTableIndex.value]; - if (!task) break; - await searchTable(task, connection.value!.db_type, currentRun); + const chunkStart = nextTableIndex.value; + const chunkEnd = probeReady ? nextProbeChunkEnd(chunkStart, range.end, taskColumns, activeKeyword.value) : Math.min(chunkStart + 1, range.end); + const chunk = tableTasks.value.slice(chunkStart, chunkEnd); + if (!chunk.length) break; + + let hitChunkIndexes: Set | undefined; + if (probeReady) { + hitChunkIndexes = await probeChunk(chunk, taskColumns.slice(chunkStart, chunkEnd), databaseType, currentRun); + } if (currentRun !== runId || cancelled.value) break; - nextTableIndex.value += 1; - progressDone.value = nextTableIndex.value; - emitState(); + + for (let offset = 0; offset < chunk.length; offset++) { + if (currentRun !== runId || cancelled.value) break; + const taskIndex = chunkStart + offset; + const task = chunk[offset]; + const prefetched = taskColumns[taskIndex]; + // 探测路径下:未命中的表跳过取数;无目录信息的表恒走逐表 + if (hitChunkIndexes && prefetched && !hitChunkIndexes.has(taskIndex)) { + // skip: 探测未命中 + } else { + await searchTable(task, databaseType, currentRun, prefetched); + } + nextTableIndex.value = taskIndex + 1; + progressDone.value = nextTableIndex.value; + emitState(); + } } } -async function searchTable(task: DatabaseSearchTableTask, databaseType: DatabaseType, currentRun: number) { +/** 对一个探测批执行多表探测,返回命中表的全局下标集合;语句失败整批回退(全部视为待取数)。 */ +async function probeChunk(chunk: DatabaseSearchTableTask[], chunkColumns: Array, databaseType: DatabaseType, currentRun: number): Promise> { + const chunkStart = nextTableIndex.value; + const probeable: number[] = []; + chunkColumns.forEach((columns, offset) => { + if (columns?.length) probeable.push(offset); + }); + + if (!probeable.length) return new Set(); + + const executionId = makeExecutionId(); + currentExecutionId.value = executionId; + let hitProbeIndexes: Set; + try { + hitProbeIndexes = await probeTables({ + databaseType, + driverProfile: connection.value?.driver_profile, + term: activeKeyword.value, + limit: activePerTableLimit.value, + tables: probeable.map((offset) => ({ + schema: chunk[offset].schema, + tableName: chunk[offset].table.name, + columns: chunkColumns[offset]!, + })), + executeQuery: (sql, runExecutionId) => api.executeQuery(props.connectionId, props.database, sql, undefined, runExecutionId), + executionId, + }); + } catch { + // 探测批失败(单表权限/中途 DDL 等):整批回退逐表,错误重新隔离到单表 + hitProbeIndexes = new Set(probeable.map((_, index) => index)); + } + if (currentExecutionId.value === executionId) currentExecutionId.value = ""; + if (currentRun !== runId || cancelled.value) return new Set(); + + const hitTaskIndexes = new Set(); + hitProbeIndexes.forEach((probeIndex) => { + const offset = probeable[probeIndex]; + if (offset !== undefined) hitTaskIndexes.add(chunkStart + offset); + }); + return hitTaskIndexes; +} + +async function searchTable(task: DatabaseSearchTableTask, databaseType: DatabaseType, currentRun: number, prefetchedColumns?: DatabaseSearchColumnLite[]) { const tableLabel = task.schema ? `${task.schema}.${task.table.name}` : task.table.name; try { const schema = task.schema || props.database; - const columns = await api.getColumns(props.connectionId, props.database, schema, task.table.name); + // 批量取列命中时跳过逐表 getColumns;数据搜索单表查询只消费这三个字段 + const columns: ColumnInfo[] = prefetchedColumns?.length + ? prefetchedColumns.map((column) => ({ + name: column.name, + data_type: column.data_type, + is_primary_key: column.is_primary_key, + is_nullable: false, + column_default: null, + extra: null, + })) + : await api.getColumns(props.connectionId, props.database, schema, task.table.name); const query = await buildDatabaseSearchSql({ databaseType, driverProfile: connection.value?.driver_profile, diff --git a/apps/desktop/src/components/search/__tests__/DatabaseSearchPanel.flow.spec.ts b/apps/desktop/src/components/search/__tests__/DatabaseSearchPanel.flow.spec.ts new file mode 100644 index 0000000000..cc53ee14b9 --- /dev/null +++ b/apps/desktop/src/components/search/__tests__/DatabaseSearchPanel.flow.spec.ts @@ -0,0 +1,261 @@ +// @vitest-environment happy-dom + +import { createApp, defineComponent, h, nextTick, ref, type App } from "vue"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createPinia, setActivePinia } from "pinia"; +import i18n from "@/i18n"; +import * as api from "@/lib/backend/api"; +import DatabaseSearchPanel from "@/components/search/DatabaseSearchPanel.vue"; + +vi.mock("@/lib/backend/api", () => ({ + listSchemas: vi.fn(), + listTables: vi.fn(), + getColumns: vi.fn(), + executeQuery: vi.fn(), + buildDatabaseSearchSql: vi.fn(), + buildDatabaseSearchColumnsSql: vi.fn(), + buildDatabaseSearchProbeSql: vi.fn(), + buildSearchResultWhere: vi.fn(), + cancelQuery: vi.fn(), +})); + +const connectionConfig: { id: string; name: string; db_type: string; driver_profile: string | null } = { + id: "conn-1", + name: "test-conn", + db_type: "dameng", + driver_profile: null, +}; + +vi.mock("@/stores/connectionStore", () => ({ + useConnectionStore: () => ({ + getConfig: (id: string) => (connectionConfig.id === id ? connectionConfig : undefined), + ensureConnected: vi.fn(async () => {}), + }), +})); + +vi.mock("@/components/ui/button", async () => { + const { defineComponent, h } = await import("vue"); + return { + Button: defineComponent({ + props: { disabled: Boolean, variant: String, size: String }, + emits: ["click"], + setup(props, { slots, emit }) { + return () => h("button", { disabled: props.disabled, onClick: (e: MouseEvent) => emit("click", e) }, slots.default?.()); + }, + }), + }; +}); + +vi.mock("@/components/ui/input", async () => { + const { defineComponent, h } = await import("vue"); + return { + Input: defineComponent({ + props: { modelValue: [String, Number], disabled: Boolean, placeholder: String, type: String }, + emits: ["update:modelValue", "keydown"], + setup(props, { emit }) { + return () => + h("input", { + value: props.modelValue, + disabled: props.disabled, + placeholder: props.placeholder, + type: props.type || "text", + onInput: (e: any) => emit("update:modelValue", e.target.value), + onKeydown: (e: any) => emit("keydown", e), + }); + }, + }), + }; +}); + +vi.mock("@/components/ui/badge", async () => { + const { defineComponent, h } = await import("vue"); + return { + Badge: defineComponent({ + setup(_props, { slots }) { + return () => h("span", slots.default?.()); + }, + }), + }; +}); + +vi.mock("@/components/ui/label", async () => { + const { defineComponent, h } = await import("vue"); + return { + Label: defineComponent({ + setup(_props, { slots }) { + return () => h("label", slots.default?.()); + }, + }), + }; +}); + +vi.mock("@/components/ui/select", async () => { + const { defineComponent, h } = await import("vue"); + const passthrough = defineComponent({ + setup(_props, { slots }) { + return () => h("div", slots.default?.()); + }, + }); + return { Select: passthrough, SelectContent: passthrough, SelectItem: passthrough, SelectTrigger: passthrough, SelectValue: passthrough }; +}); + +const mountedApps: App[] = []; + +beforeEach(() => { + setActivePinia(createPinia()); + connectionConfig.db_type = "dameng"; +}); + +afterEach(() => { + // 模块工厂里的 vi.fn 不受 restoreAllMocks 影响,必须显式清调用历史与实现 + vi.clearAllMocks(); + for (const app of mountedApps.splice(0)) app.unmount(); + document.body.innerHTML = ""; +}); + +const TABLES = [ + { name: "USERS", table_type: "TABLE" }, + { name: "FILES", table_type: "TABLE" }, + { name: "V_ORDERS", table_type: "VIEW" }, +]; + +const BULK_COLUMN_ROWS = { + columns: ["table_name", "column_name", "data_type", "is_primary_key"], + rows: [ + ["USERS", "ID", "NUMBER", 1], + ["USERS", "EMAIL", "VARCHAR2", 0], + ["FILES", "DATA", "BLOB", 0], + ], +}; + +async function mountPanel() { + vi.mocked(api.listTables).mockResolvedValue(TABLES as any); + vi.mocked(api.getColumns).mockResolvedValue([]); + vi.mocked(api.buildSearchResultWhere).mockResolvedValue('"ID" = 1'); + vi.mocked(api.buildDatabaseSearchSql).mockResolvedValue({ sql: "SELECT_USERS", searchableColumns: ["EMAIL"] }); + vi.mocked(api.buildDatabaseSearchColumnsSql).mockResolvedValue({ sql: "COLS_SQL" }); + vi.mocked(api.buildDatabaseSearchProbeSql).mockResolvedValue({ sql: "PROBE_SQL", includedIndices: [0] }); + + const states: any[] = []; + const panelRef = ref(null); + const container = document.createElement("div"); + document.body.append(container); + const app = createApp( + defineComponent({ + setup() { + return () => + h(DatabaseSearchPanel, { + ref: panelRef, + connectionId: "conn-1", + database: "shop", + schema: "DLWEB", + initialState: { keyword: "admin" }, + inDialog: true, + "onUpdate:state": (state: any) => states.push(state), + }); + }, + }), + ); + app.use(i18n); + mountedApps.push(app); + app.mount(container); + await nextTick(); + return { panelRef, states }; +} + +describe("DatabaseSearchPanel scan flow", () => { + it("bulk path: one columns query + one probe, fetch rows only for hit tables", async () => { + vi.mocked(api.executeQuery).mockImplementation(async (_conn: string, _db: string, sql: string) => { + if (sql === "COLS_SQL") return BULK_COLUMN_ROWS as any; + if (sql === "PROBE_SQL") return { columns: ["TABLE_LABEL", "HITS"], rows: [["t0", 2]] } as any; + if (sql === "SELECT_USERS") return { columns: ["ID", "EMAIL"], rows: [[1, "admin@x.com"]] } as any; + throw new Error(`unexpected sql: ${sql}`); + }); + + const { panelRef, states } = await mountPanel(); + await panelRef.value.startSearch(); + await vi.waitFor(() => { + expect(states.at(-1)?.progressDone).toBe(2); + }); + + // 批量取列:一条目录查询,零次逐表 getColumns + expect(api.buildDatabaseSearchColumnsSql).toHaveBeenCalledTimes(1); + expect(api.getColumns).not.toHaveBeenCalled(); + + // 探测:两个任务表(视图已被排除),只有 USERS 命中并取数 + expect(api.buildDatabaseSearchProbeSql).toHaveBeenCalledTimes(1); + const probeArg = vi.mocked(api.buildDatabaseSearchProbeSql).mock.calls[0][0]; + expect(probeArg.tables.map((t: any) => t.tableName)).toEqual(["USERS", "FILES"]); + expect(api.buildDatabaseSearchSql).toHaveBeenCalledTimes(1); + expect(vi.mocked(api.buildDatabaseSearchSql).mock.calls[0][0].tableName).toBe("USERS"); + // 预取列被传入单表查询(证明不再走 getColumns) + expect(vi.mocked(api.buildDatabaseSearchSql).mock.calls[0][0].columns.map((c: any) => c.name)).toEqual(["ID", "EMAIL"]); + + // 结果与回表条件一致 + const finalState = states.at(-1); + expect(finalState.results).toHaveLength(1); + expect(finalState.results[0].tableName).toBe("USERS"); + expect(finalState.results[0].whereInput).toBe('"ID" = 1'); + expect(finalState.progressTotal).toBe(2); + }); + + it("probe failure falls back to per-table scan for the whole batch", async () => { + vi.mocked(api.executeQuery).mockImplementation(async (_conn: string, _db: string, sql: string) => { + if (sql === "COLS_SQL") return BULK_COLUMN_ROWS as any; + if (sql === "PROBE_SQL") throw new Error("ORA-00942: table or view does not exist"); + if (sql === "SELECT_USERS") return { columns: ["ID", "EMAIL"], rows: [[1, "admin@x.com"]] } as any; + throw new Error(`unexpected sql: ${sql}`); + }); + + const { panelRef, states } = await mountPanel(); + await panelRef.value.startSearch(); + await vi.waitFor(() => { + expect(states.at(-1)?.progressDone).toBe(2); + }); + + // 整批回退:两张表都执行了逐表取数查询,列信息仍来自批量 Map + expect(api.buildDatabaseSearchSql).toHaveBeenCalledTimes(2); + expect(vi.mocked(api.buildDatabaseSearchSql).mock.calls.map((c) => c[0].tableName)).toEqual(["USERS", "FILES"]); + expect(api.getColumns).not.toHaveBeenCalled(); + // 失败被吞掉,不产生 generalError + expect(states.at(-1)?.generalError).toBe(""); + }); + + it("unsupported dialect silently falls back to per-table getColumns", async () => { + connectionConfig.db_type = "sqlite"; + vi.mocked(api.getColumns).mockResolvedValue([{ name: "EMAIL", data_type: "varchar", is_nullable: true, column_default: null, is_primary_key: false, extra: null }] as any); + vi.mocked(api.executeQuery).mockImplementation(async (_conn: string, _db: string, sql: string) => { + if (sql === "SELECT_USERS") return { columns: ["EMAIL"], rows: [["admin@x.com"]] } as any; + throw new Error(`unexpected sql: ${sql}`); + }); + + const { panelRef, states } = await mountPanel(); + await panelRef.value.startSearch(); + await vi.waitFor(() => { + expect(states.at(-1)?.progressDone).toBe(2); + }); + + expect(api.buildDatabaseSearchColumnsSql).not.toHaveBeenCalled(); + expect(api.buildDatabaseSearchProbeSql).not.toHaveBeenCalled(); + expect(api.getColumns).toHaveBeenCalledTimes(2); + expect(api.buildDatabaseSearchSql).toHaveBeenCalledTimes(2); + }); + + it("tables without probe hits advance progress without any fetch query", async () => { + vi.mocked(api.executeQuery).mockImplementation(async (_conn: string, _db: string, sql: string) => { + if (sql === "COLS_SQL") return BULK_COLUMN_ROWS as any; + throw new Error(`unexpected sql: ${sql}`); + }); + + const { panelRef, states } = await mountPanel(); + // mountPanel 会设置默认探测 mock,这里在启动前覆盖为 null(全部无可搜列等价场景) + vi.mocked(api.buildDatabaseSearchProbeSql).mockResolvedValue(null); + await panelRef.value.startSearch(); + await vi.waitFor(() => { + expect(states.at(-1)?.progressDone).toBe(2); + }); + + expect(api.buildDatabaseSearchSql).not.toHaveBeenCalled(); + expect(states.at(-1)?.results).toHaveLength(0); + }); +}); diff --git a/apps/desktop/src/lib/backend/api.ts b/apps/desktop/src/lib/backend/api.ts index 2a9f39b7ea..c67b84c96d 100644 --- a/apps/desktop/src/lib/backend/api.ts +++ b/apps/desktop/src/lib/backend/api.ts @@ -304,6 +304,8 @@ export const buildCreateUserSql = forward("buildCreateUserSql"); export const buildDroppedFilePreviewSql = forward("buildDroppedFilePreviewSql"); export const buildTableSelectSql = forward("buildTableSelectSql"); export const buildDatabaseSearchSql = forward("buildDatabaseSearchSql"); +export const buildDatabaseSearchColumnsSql = forward("buildDatabaseSearchColumnsSql"); +export const buildDatabaseSearchProbeSql = forward("buildDatabaseSearchProbeSql"); export const buildDatabaseStructureSearchSql = forward("buildDatabaseStructureSearchSql"); export const supportsDatabaseStructureSearch = forward("supportsDatabaseStructureSearch"); export const buildSearchResultWhere = forward("buildSearchResultWhere"); diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts index 9d813dfa49..299fd6bbb3 100644 --- a/apps/desktop/src/lib/backend/http.ts +++ b/apps/desktop/src/lib/backend/http.ts @@ -1873,6 +1873,14 @@ export async function buildDatabaseSearchSql(options: DatabaseSearchSqlOptions): return post("/api/query/build-database-search-sql", { options }); } +export async function buildDatabaseSearchColumnsSql(options: import("@/lib/database/databaseSearchColumns").DatabaseSearchColumnsSqlOptions): Promise<{ sql: string } | null> { + return post("/api/query/build-database-search-columns-sql", { options }); +} + +export async function buildDatabaseSearchProbeSql(options: import("@/lib/database/databaseSearchProbe").DatabaseSearchProbeSqlOptions): Promise<{ sql: string; includedIndices: number[] } | null> { + return post("/api/query/build-database-search-probe-sql", { options }); +} + export async function buildDatabaseStructureSearchSql(options: DatabaseStructureSearchSqlOptions): Promise { return post("/api/query/build-database-structure-search-sql", { options }); } diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 5cf2e62826..24c821b52d 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -2310,6 +2310,14 @@ export async function buildDatabaseSearchSql(options: DatabaseSearchSqlOptions): return invoke("build_database_search_sql", { options }); } +export async function buildDatabaseSearchColumnsSql(options: import("@/lib/database/databaseSearchColumns").DatabaseSearchColumnsSqlOptions): Promise<{ sql: string } | null> { + return invoke("build_database_search_columns_sql", { options }); +} + +export async function buildDatabaseSearchProbeSql(options: import("@/lib/database/databaseSearchProbe").DatabaseSearchProbeSqlOptions): Promise<{ sql: string; includedIndices: number[] } | null> { + return invoke("build_database_search_probe_sql", { options }); +} + export async function buildDatabaseStructureSearchSql(options: DatabaseStructureSearchSqlOptions): Promise { return invoke("build_database_structure_search_sql", { options }); } diff --git a/apps/desktop/src/lib/database/__tests__/databaseSearchCapabilities.spec.ts b/apps/desktop/src/lib/database/__tests__/databaseSearchCapabilities.spec.ts new file mode 100644 index 0000000000..0e40f8a957 --- /dev/null +++ b/apps/desktop/src/lib/database/__tests__/databaseSearchCapabilities.spec.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { BULK_DATABASE_SEARCH_TYPES, supportsBulkDatabaseSearch } from "@/lib/database/databaseSearchCapabilities"; + +// 镜像 Rust database_search_columns_sql.rs / database_search_probe_sql.rs 的家族 match 臂: +// 两边同步修改时必须一并更新此快照,防止前端预判与后端判定漂移。 +const RUST_BULK_SEARCH_FAMILIES = ["oracle", "oceanbase-oracle", "dameng", "mysql", "doris", "starrocks", "manticoresearch", "goldendb", "postgres", "kingbase", "highgo", "uxdb", "vastbase", "opengauss", "gaussdb", "kwdb", "yashandb", "oscar", "redshift", "sqlserver"]; + +describe("databaseSearchCapabilities", () => { + it("mirrors the Rust bulk search family dispatch", () => { + expect([...BULK_DATABASE_SEARCH_TYPES].sort()).toEqual([...RUST_BULK_SEARCH_FAMILIES].sort()); + }); + + it("gates unsupported dialects", () => { + expect(supportsBulkDatabaseSearch("dameng")).toBe(true); + expect(supportsBulkDatabaseSearch("sqlite")).toBe(false); + expect(supportsBulkDatabaseSearch(undefined)).toBe(false); + }); +}); diff --git a/apps/desktop/src/lib/database/__tests__/databaseSearchColumns.spec.ts b/apps/desktop/src/lib/database/__tests__/databaseSearchColumns.spec.ts new file mode 100644 index 0000000000..f88542e41a --- /dev/null +++ b/apps/desktop/src/lib/database/__tests__/databaseSearchColumns.spec.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as api from "@/lib/backend/api.ts"; +import { fetchDatabaseSearchColumns, findSearchColumns, searchColumnsTableKey } from "@/lib/database/databaseSearchColumns"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function rowsResult(columns: string[], rows: unknown[][]) { + return { columns, rows }; +} + +describe("databaseSearchColumns", () => { + it("groups catalog rows by table with lowercase keys", async () => { + const buildSpy = vi.spyOn(api, "buildDatabaseSearchColumnsSql").mockResolvedValue({ sql: "SELECT 1" }); + const executeQuery = vi.fn().mockResolvedValue( + rowsResult( + ["TABLE_NAME", "COLUMN_NAME", "DATA_TYPE", "IS_PRIMARY_KEY"], + [ + ["USERS", "ID", "NUMBER", 1], + ["USERS", "EMAIL", "VARCHAR2", 0], + ["ORDERS", "ID", "NUMBER", 1], + ], + ), + ); + + const map = await fetchDatabaseSearchColumns({ + databaseType: "dameng", + database: "app", + schema: "DLWEB", + executeQuery, + executionId: "exec-1", + }); + + expect(buildSpy).toHaveBeenCalledWith({ databaseType: "dameng", database: "app", schema: "DLWEB" }); + const users = findSearchColumns(map, "DLWEB", "app", "USERS"); + expect(users).toHaveLength(2); + expect(users?.[0]).toEqual({ name: "ID", data_type: "NUMBER", is_primary_key: true }); + expect(users?.[1]?.is_primary_key).toBe(false); + expect(findSearchColumns(map, "dlweb", "app", "users")).toHaveLength(2); + expect(findSearchColumns(map, "DLWEB", "app", "ORDERS")).toHaveLength(1); + }); + + it("normalizes primary key flag variants", async () => { + vi.spyOn(api, "buildDatabaseSearchColumnsSql").mockResolvedValue({ sql: "SELECT 1" }); + const executeQuery = vi.fn().mockResolvedValue( + rowsResult( + ["table_name", "column_name", "data_type", "is_primary_key"], + [ + ["t1", "a", "int", true], + ["t1", "b", "int", false], + ["t2", "a", "int", "1"], + ["t2", "b", "int", "0"], + ["t3", "a", "int", 1], + ["t3", "b", "int", 0], + ], + ), + ); + + const map = await fetchDatabaseSearchColumns({ databaseType: "postgres", database: "app", executeQuery, executionId: "e" }); + const t1 = findSearchColumns(map, undefined, "app", "t1")!; + const t2 = findSearchColumns(map, undefined, "app", "t2")!; + const t3 = findSearchColumns(map, undefined, "app", "t3")!; + expect(t1.map((c) => c.is_primary_key)).toEqual([true, false]); + expect(t2.map((c) => c.is_primary_key)).toEqual([true, false]); + expect(t3.map((c) => c.is_primary_key)).toEqual([true, false]); + }); + + it("returns empty map when builder yields no sql", async () => { + const buildSpy = vi.spyOn(api, "buildDatabaseSearchColumnsSql").mockResolvedValue(null); + const executeQuery = vi.fn(); + const map = await fetchDatabaseSearchColumns({ databaseType: "dameng", database: "app", executeQuery, executionId: "e" }); + expect(map.size).toBe(0); + expect(buildSpy).toHaveBeenCalled(); + expect(executeQuery).not.toHaveBeenCalled(); + }); + + it("returns empty map when the catalog result is truncated so callers fall back per-table", async () => { + vi.spyOn(api, "buildDatabaseSearchColumnsSql").mockResolvedValue({ sql: "SELECT 1" }); + const executeQuery = vi.fn().mockResolvedValue({ + columns: ["TABLE_NAME", "COLUMN_NAME", "DATA_TYPE", "IS_PRIMARY_KEY"], + rows: [ + ["USERS", "ID", "NUMBER", 1], + ["USERS", "EMAIL", "VARCHAR2", 0], + ], + truncated: true, + }); + + const map = await fetchDatabaseSearchColumns({ + databaseType: "dameng", + database: "app", + schema: "DLWEB", + executeQuery, + executionId: "exec-1", + }); + + expect(map.size).toBe(0); + }); + + it("returns empty map when the catalog query fails", async () => { + vi.spyOn(api, "buildDatabaseSearchColumnsSql").mockResolvedValue({ sql: "SELECT 1" }); + const executeQuery = vi.fn().mockRejectedValue(new Error("permission denied")); + const map = await fetchDatabaseSearchColumns({ databaseType: "mysql", database: "app", executeQuery, executionId: "e" }); + expect(map.size).toBe(0); + }); + + it("skips bulk fetch for unsupported dialects without calling the builder", async () => { + const buildSpy = vi.spyOn(api, "buildDatabaseSearchColumnsSql"); + const map = await fetchDatabaseSearchColumns({ databaseType: "sqlite", database: "app", executeQuery: vi.fn(), executionId: "e" }); + expect(map.size).toBe(0); + expect(buildSpy).not.toHaveBeenCalled(); + }); + + it("keys use schema when present and database otherwise", () => { + expect(searchColumnsTableKey("public", "app", "Users")).toBe("public\nusers"); + expect(searchColumnsTableKey(undefined, "app", "Users")).toBe("app\nusers"); + expect(searchColumnsTableKey(" ", "app", "Users")).toBe("app\nusers"); + }); +}); diff --git a/apps/desktop/src/lib/database/__tests__/databaseSearchProbe.spec.ts b/apps/desktop/src/lib/database/__tests__/databaseSearchProbe.spec.ts new file mode 100644 index 0000000000..30f7829534 --- /dev/null +++ b/apps/desktop/src/lib/database/__tests__/databaseSearchProbe.spec.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as api from "@/lib/backend/api.ts"; +import type { DatabaseSearchColumnLite } from "@/lib/database/databaseSearchColumns"; +import { countProbeConditions, DATABASE_SEARCH_PROBE_CONDITION_LIMIT, DATABASE_SEARCH_PROBE_TABLE_BATCH_SIZE, nextProbeChunkEnd, probeTables } from "@/lib/database/databaseSearchProbe"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function col(name: string, data_type: string): DatabaseSearchColumnLite { + return { name, data_type, is_primary_key: false }; +} + +describe("databaseSearchProbe", () => { + it("counts text columns and numeric columns only for numeric terms", () => { + const columns = [col("a", "varchar"), col("b", "int"), col("c", "text"), col("d", "decimal")]; + expect(countProbeConditions(columns, "alice")).toBe(2); + expect(countProbeConditions(columns, "42")).toBe(4); + expect(countProbeConditions([], "alice")).toBe(0); + }); + + it("caps chunks at the table batch size", () => { + const tasks = Array.from({ length: 30 }, () => [col("a", "varchar")]); + expect(nextProbeChunkEnd(0, tasks.length, tasks, "x")).toBe(DATABASE_SEARCH_PROBE_TABLE_BATCH_SIZE); + expect(nextProbeChunkEnd(15, tasks.length, tasks, "x")).toBe(30); + }); + + it("splits before the table that would exceed the condition limit", () => { + // 每表 60 条件:第 8 张起累计 480,第 9 张会到 540 > 500 → 批在第 8 张后切分 + const heavy = Array.from({ length: 60 }, (_, i) => col(`c${i}`, "varchar")); + const tasks = Array.from({ length: 12 }, () => heavy); + expect(nextProbeChunkEnd(0, tasks.length, tasks, "x")).toBe(8); + + // 无列信息的表按 0 条件计,不触顶 + const unknown = [heavy, undefined, heavy, undefined]; + expect(nextProbeChunkEnd(0, 4, unknown, "x")).toBe(4); + }); + + it("always takes at least one table even beyond the condition limit", () => { + const huge = Array.from({ length: DATABASE_SEARCH_PROBE_CONDITION_LIMIT + 10 }, (_, i) => col(`c${i}`, "varchar")); + expect(nextProbeChunkEnd(0, 3, [huge, huge, huge], "x")).toBe(1); + }); + + it("maps label rows back to hit table indexes via includedIndices", async () => { + const buildSpy = vi.spyOn(api, "buildDatabaseSearchProbeSql").mockResolvedValue({ + sql: "SELECT 1", + // 输入 3 张表,第 0、2 张进探测分支(t0、t1),第 1 张被剔除 + includedIndices: [0, 2], + }); + const executeQuery = vi.fn().mockResolvedValue({ + columns: ["TABLE_LABEL", "HITS"], + rows: [ + ["t0", 3], + ["t1", 0], + ["tX", 9], + ], + }); + + const hits = await probeTables({ + databaseType: "dameng", + term: "alice", + limit: 20, + tables: [ + { tableName: "a", columns: [col("x", "varchar2")] }, + { tableName: "b", columns: [col("x", "blob")] }, + { tableName: "c", columns: [col("y", "text")] }, + ], + executeQuery, + executionId: "e", + }); + + expect(buildSpy).toHaveBeenCalled(); + // t0 命中(hits=3)→ 输入下标 0;t1 未命中(hits=0);未知标签 tX 忽略;被剔除的输入下标 1 不出现 + expect([...hits]).toEqual([0]); + expect(hits.has(1)).toBe(false); + expect(hits.has(2)).toBe(false); + }); + + it("returns empty set when the builder yields no sql", async () => { + vi.spyOn(api, "buildDatabaseSearchProbeSql").mockResolvedValue(null); + const hits = await probeTables({ + databaseType: "mysql", + term: "alice", + limit: 20, + tables: [{ tableName: "a", columns: [col("x", "varchar")] }], + executeQuery: vi.fn(), + executionId: "e", + }); + expect(hits.size).toBe(0); + }); + + it("propagates statement failures so callers fall back per table", async () => { + vi.spyOn(api, "buildDatabaseSearchProbeSql").mockResolvedValue({ sql: "SELECT 1", includedIndices: [0] }); + const executeQuery = vi.fn().mockRejectedValue(new Error("table dropped")); + await expect( + probeTables({ + databaseType: "mysql", + term: "alice", + limit: 20, + tables: [{ tableName: "a", columns: [col("x", "varchar")] }], + executeQuery, + executionId: "e", + }), + ).rejects.toThrow("table dropped"); + }); +}); diff --git a/apps/desktop/src/lib/database/databaseSearch.ts b/apps/desktop/src/lib/database/databaseSearch.ts index ba3c23091e..a77df280a7 100644 --- a/apps/desktop/src/lib/database/databaseSearch.ts +++ b/apps/desktop/src/lib/database/databaseSearch.ts @@ -30,17 +30,20 @@ const NUMBER_TYPES = ["int", "serial", "number", "numeric", "decimal", "float", const SKIPPED_TYPES = ["blob", "binary", "bytea", "image", "geometry", "geography"]; -function normalizedType(column: ColumnInfo): string { +/** 文本/数值列判定只读 data_type,兼容批量取列的最小列结构。 */ +type SearchableColumn = Pick; + +function normalizedType(column: SearchableColumn): string { return column.data_type.toLowerCase(); } -export function isTextSearchColumn(column: ColumnInfo): boolean { +export function isTextSearchColumn(column: SearchableColumn): boolean { const type = normalizedType(column); if (SKIPPED_TYPES.some((skipped) => type.includes(skipped))) return false; return TEXT_TYPES.some((textType) => type.includes(textType)); } -export function isNumericSearchColumn(column: ColumnInfo): boolean { +export function isNumericSearchColumn(column: SearchableColumn): boolean { const type = normalizedType(column); if (SKIPPED_TYPES.some((skipped) => type.includes(skipped))) return false; return NUMBER_TYPES.some((numberType) => type.includes(numberType)); @@ -50,7 +53,7 @@ function cleanTerm(term: string): string { return term.trim(); } -function parseNumericTerm(term: string): string | null { +export function parseNumericTerm(term: string): string | null { const trimmed = cleanTerm(term); if (!/^[+-]?(?:\d+|\d+\.\d+|\.\d+)$/.test(trimmed)) return null; return trimmed; diff --git a/apps/desktop/src/lib/database/databaseSearchCapabilities.ts b/apps/desktop/src/lib/database/databaseSearchCapabilities.ts new file mode 100644 index 0000000000..f552b89356 --- /dev/null +++ b/apps/desktop/src/lib/database/databaseSearchCapabilities.ts @@ -0,0 +1,14 @@ +import type { DatabaseType } from "@/types/database"; + +/** + * 「在数据库中查找」批量路径(批量取列 + 多表探测)支持的方言, + * 镜像 Rust `database_search_columns_sql` / `database_search_probe_sql` + * 的家族判定;两处集合必须同步修改(有单测锁形状)。 + */ +const BULK_DATABASE_SEARCH_TYPES = new Set(["oracle", "oceanbase-oracle", "dameng", "mysql", "doris", "starrocks", "manticoresearch", "goldendb", "postgres", "kingbase", "highgo", "uxdb", "vastbase", "opengauss", "gaussdb", "kwdb", "yashandb", "oscar", "redshift", "sqlserver"]); + +export function supportsBulkDatabaseSearch(dbType?: DatabaseType): boolean { + return !!dbType && BULK_DATABASE_SEARCH_TYPES.has(dbType); +} + +export { BULK_DATABASE_SEARCH_TYPES }; diff --git a/apps/desktop/src/lib/database/databaseSearchColumns.ts b/apps/desktop/src/lib/database/databaseSearchColumns.ts new file mode 100644 index 0000000000..09744d5edc --- /dev/null +++ b/apps/desktop/src/lib/database/databaseSearchColumns.ts @@ -0,0 +1,97 @@ +import type { DatabaseType } from "@/types/database.ts"; +import * as api from "@/lib/backend/api.ts"; +import { supportsBulkDatabaseSearch } from "@/lib/database/databaseSearchCapabilities.ts"; + +/** 批量取列的最小列形状:数据搜索路径只消费 name / data_type / is_primary_key。 */ +export interface DatabaseSearchColumnLite { + name: string; + data_type: string; + is_primary_key: boolean; +} + +export interface DatabaseSearchColumnsSqlOptions { + databaseType?: DatabaseType; + database: string; + schema?: string; +} + +/** (schema, 表名) → 列列表。键统一小写兜底:listTables 与目录行大小写偶有差异。 */ +export type SearchColumnsByTable = Map; + +export function searchColumnsTableKey(schema: string | undefined, database: string, tableName: string): string { + const owner = (schema && schema.trim()) || database; + return `${owner.toLowerCase()}\n${tableName.toLowerCase()}`; +} + +export function findSearchColumns(map: SearchColumnsByTable, schema: string | undefined, database: string, tableName: string): DatabaseSearchColumnLite[] | undefined { + return map.get(searchColumnsTableKey(schema, database, tableName)); +} + +function isTruthyPrimaryKey(value: unknown): boolean { + if (value === true) return true; + if (typeof value === "number") return value !== 0; + if (typeof value === "string") return value.trim().toLowerCase() === "1" || value.trim().toLowerCase() === "true"; + return false; +} + +/** + * 一次目录查询取回整个 schema 的列信息并按表分组。 + * 任何失败(方言不支持、目录视图不可见、执行出错)都返回空 Map, + * 调用方对缺列的表逐表回退 getColumns——绝不抛错中断搜索。 + */ +export async function fetchDatabaseSearchColumns(options: { + databaseType?: DatabaseType; + database: string; + schema?: string; + buildSql?: typeof api.buildDatabaseSearchColumnsSql; + executeQuery: (sql: string, executionId: string) => Promise<{ columns: string[]; rows: unknown[][]; truncated?: boolean }>; + executionId: string; +}): Promise { + const map: SearchColumnsByTable = new Map(); + if (!supportsBulkDatabaseSearch(options.databaseType)) return map; + + try { + const buildSql = options.buildSql ?? api.buildDatabaseSearchColumnsSql; + const built = await buildSql({ + databaseType: options.databaseType, + database: options.database, + schema: options.schema, + }); + if (!built?.sql) return map; + + const result = await options.executeQuery(built.sql, options.executionId); + // The catalog query rides the backend's default row cap (10k). A truncation + // can land mid-table, and a table seen with only part of its columns would + // make the probe miss keywords in the dropped columns — silently skipping + // that table. Drop the whole catalog instead: an empty map sends every + // table down the per-table getColumns fallback, which is always complete. + if (result.truncated) return map; + const headerIndex = new Map(); + result.columns.forEach((name, index) => headerIndex.set(name.toLowerCase(), index)); + const tableIndex = headerIndex.get("table_name"); + const columnIndex = headerIndex.get("column_name"); + const typeIndex = headerIndex.get("data_type"); + const primaryKeyIndex = headerIndex.get("is_primary_key"); + if (tableIndex === undefined || columnIndex === undefined || typeIndex === undefined) return map; + + for (const row of result.rows) { + const tableName = String(row[tableIndex] ?? "").trim(); + const columnName = String(row[columnIndex] ?? "").trim(); + if (!tableName || !columnName) continue; + const key = searchColumnsTableKey(options.schema, options.database, tableName); + let columns = map.get(key); + if (!columns) { + columns = []; + map.set(key, columns); + } + columns.push({ + name: columnName, + data_type: String(row[typeIndex] ?? ""), + is_primary_key: primaryKeyIndex !== undefined && isTruthyPrimaryKey(row[primaryKeyIndex]), + }); + } + } catch { + // 目录查询失败——返回已收集部分,未命中的表由调用方逐表回退 + } + return map; +} diff --git a/apps/desktop/src/lib/database/databaseSearchProbe.ts b/apps/desktop/src/lib/database/databaseSearchProbe.ts new file mode 100644 index 0000000000..8608be7e0c --- /dev/null +++ b/apps/desktop/src/lib/database/databaseSearchProbe.ts @@ -0,0 +1,95 @@ +import type { DatabaseType } from "@/types/database.ts"; +import * as api from "@/lib/backend/api.ts"; +import { isNumericSearchColumn, isTextSearchColumn, parseNumericTerm } from "@/lib/database/databaseSearch.ts"; +import type { DatabaseSearchColumnLite } from "@/lib/database/databaseSearchColumns.ts"; + +/** 单条探测语句最多合并的表数与条件数(双上限任一触顶切批,防巨型语句拖垮解析器)。 */ +export const DATABASE_SEARCH_PROBE_TABLE_BATCH_SIZE = 20; +export const DATABASE_SEARCH_PROBE_CONDITION_LIMIT = 500; + +export interface DatabaseSearchProbeSqlOptions { + databaseType?: DatabaseType; + driverProfile?: string; + tables: Array<{ + schema?: string; + tableName: string; + columns: DatabaseSearchColumnLite[]; + }>; + term: string; + limit?: number; +} + +export function countProbeConditions(columns: DatabaseSearchColumnLite[], term: string): number { + const textCount = columns.filter(isTextSearchColumn).length; + const numericCount = parseNumericTerm(term) ? columns.filter(isNumericSearchColumn).length : 0; + return textCount + numericCount; +} + +/** + * 计算下一个探测批的结束下标(不含):从 nextIndex 起,受表数/条件数双上限约束; + * 至少包含一张表,避免零条件批死循环(无列信息的表按 0 条件计,走批内逐表回退)。 + * tasks 直接传每张表的预取列数组(undefined = 无目录信息)。 + */ +export function nextProbeChunkEnd(nextIndex: number, endIndex: number, tasks: Array, term: string, options?: { tableBatchSize?: number; conditionLimit?: number }): number { + const tableBatchSize = options?.tableBatchSize ?? DATABASE_SEARCH_PROBE_TABLE_BATCH_SIZE; + const conditionLimit = options?.conditionLimit ?? DATABASE_SEARCH_PROBE_CONDITION_LIMIT; + let end = Math.min(nextIndex, endIndex); + let conditions = 0; + let tables = 0; + while (end < endIndex && tables < tableBatchSize) { + const taskConditions = tasks[end] ? countProbeConditions(tasks[end]!, term) : 0; + if (tables > 0 && conditions + taskConditions > conditionLimit) break; + conditions += taskConditions; + tables += 1; + end += 1; + } + return Math.max(end, nextIndex + 1 <= endIndex ? nextIndex + 1 : nextIndex); +} + +/** + * 执行一条多表探测语句,返回命中的表下标集合(相对传入 tables)。 + * 语句失败向上抛出,由调用方将该批回退逐表路径。 + */ +export async function probeTables(options: { + databaseType?: DatabaseType; + driverProfile?: string; + term: string; + limit: number; + tables: DatabaseSearchProbeSqlOptions["tables"]; + buildSql?: typeof api.buildDatabaseSearchProbeSql; + executeQuery: (sql: string, executionId: string) => Promise<{ columns: string[]; rows: unknown[][] }>; + executionId: string; +}): Promise> { + const buildSql = options.buildSql ?? api.buildDatabaseSearchProbeSql; + const probe = await buildSql({ + databaseType: options.databaseType, + driverProfile: options.driverProfile, + tables: options.tables, + term: options.term, + limit: options.limit, + }); + if (!probe?.sql) return new Set(); + + const result = await options.executeQuery(probe.sql, options.executionId); + const headerIndex = new Map(); + result.columns.forEach((name, index) => headerIndex.set(name.toLowerCase(), index)); + const labelIndex = headerIndex.get("table_label"); + const hitsIndex = headerIndex.get("hits"); + if (labelIndex === undefined) return new Set(); + + // includedIndices[k] 是第 k 个分支(t{k})对应的输入表下标 + const labelToIndex = new Map(); + probe.includedIndices.forEach((tableIndex, branch) => { + labelToIndex.set(`t${branch}`, tableIndex); + }); + + const hits = new Set(); + for (const row of result.rows) { + const label = String(row[labelIndex] ?? "").trim(); + const tableIndex = labelToIndex.get(label); + if (tableIndex === undefined) continue; + const hitCount = hitsIndex === undefined ? 1 : Number(row[hitsIndex] ?? 0); + if (Number.isFinite(hitCount) && hitCount > 0) hits.add(tableIndex); + } + return hits; +} diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 718c2c9b15..b31aaba264 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -44,6 +44,8 @@ pub use data::database_export; pub use dbx_drivers::database_capabilities; pub use dbx_sql::data_grid_extractors; pub use dbx_sql::data_grid_sql; +pub use dbx_sql::database_search_columns_sql; +pub use dbx_sql::database_search_probe_sql; pub use dbx_sql::database_search_sql; pub use dbx_sql::database_structure_search_sql; pub use dbx_types::database_manifest; diff --git a/crates/dbx-sql-data/src/database_search_columns_sql.rs b/crates/dbx-sql-data/src/database_search_columns_sql.rs new file mode 100644 index 0000000000..782f84670a --- /dev/null +++ b/crates/dbx-sql-data/src/database_search_columns_sql.rs @@ -0,0 +1,248 @@ +use serde::{Deserialize, Serialize}; + +use crate::models::connection::DatabaseType; + +/// 「在数据库中查找」的批量列信息查询：每个 schema 一条目录 SQL， +/// 一次取回该 schema 全部普通表的列名/类型/主键标记，替代逐表 getColumns 往返。 +/// 行形状统一为 table_name / column_name / data_type / is_primary_key， +/// 主键来源以 LEFT JOIN 挂接——权限不足时主键标记缺失但列信息仍可用。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseSearchColumnsSqlOptions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_type: Option, + pub database: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseSearchColumnsSql { + pub sql: String, +} + +pub fn build_database_search_columns_sql(options: DatabaseSearchColumnsSqlOptions) -> Option { + let database_type = options.database_type?; + let sql = match database_type { + DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => { + oracle_family_columns_sql(options.schema.as_deref(), &options.database) + } + DatabaseType::Mysql + | DatabaseType::Doris + | DatabaseType::StarRocks + | DatabaseType::ManticoreSearch + | DatabaseType::Goldendb => mysql_family_columns_sql(options.schema.as_deref(), &options.database), + DatabaseType::Postgres + | DatabaseType::Kingbase + | DatabaseType::Highgo + | DatabaseType::Uxdb + | DatabaseType::Gbase8c + | DatabaseType::Vastbase + | DatabaseType::OpenGauss + | DatabaseType::Gaussdb + | DatabaseType::Kwdb + | DatabaseType::Yashandb + | DatabaseType::Oscar + | DatabaseType::Redshift => postgres_family_columns_sql(options.schema.as_deref()), + DatabaseType::SqlServer => sqlserver_columns_sql(options.schema.as_deref()), + // 其余方言（含 SQLite 本地家族）不支持批量取列，调用方逐表回退 getColumns。 + _ => return None, + }; + Some(DatabaseSearchColumnsSql { sql }) +} + +fn schema_or_database<'a>(schema: Option<&'a str>, database: &'a str) -> &'a str { + schema.filter(|value| !value.trim().is_empty()).unwrap_or(database) +} + +// Oracle 家族用带引号的小写别名，否则无引号别名会被统一大写。 +fn oracle_family_columns_sql(schema: Option<&str>, database: &str) -> String { + let owner = sql_string_literal(schema_or_database(schema, database)); + format!( + "SELECT c.TABLE_NAME AS \"table_name\", c.COLUMN_NAME AS \"column_name\", c.DATA_TYPE AS \"data_type\" \ + , CASE WHEN pc.COLUMN_NAME IS NULL THEN 0 ELSE 1 END AS \"is_primary_key\" \ + FROM ALL_TAB_COLUMNS c \ + LEFT JOIN ( \ + SELECT cc.OWNER, cc.TABLE_NAME, cc.COLUMN_NAME \ + FROM ALL_CONSTRAINTS cn \ + JOIN ALL_CONS_COLUMNS cc ON cc.OWNER = cn.OWNER AND cc.TABLE_NAME = cn.TABLE_NAME AND cc.CONSTRAINT_NAME = cn.CONSTRAINT_NAME \ + WHERE cn.CONSTRAINT_TYPE = 'P' ) pc \ + ON pc.OWNER = c.OWNER AND pc.TABLE_NAME = c.TABLE_NAME AND pc.COLUMN_NAME = c.COLUMN_NAME \ + WHERE c.OWNER = {owner} \ + ORDER BY c.TABLE_NAME, c.COLUMN_ID" + ) +} + +fn mysql_family_columns_sql(schema: Option<&str>, database: &str) -> String { + let table_schema = sql_string_literal(schema_or_database(schema, database)); + format!( + "SELECT c.TABLE_NAME AS table_name, c.COLUMN_NAME AS column_name, c.DATA_TYPE AS data_type \ + , CASE WHEN k.COLUMN_NAME IS NULL THEN 0 ELSE 1 END AS is_primary_key \ + FROM information_schema.COLUMNS c \ + LEFT JOIN information_schema.KEY_COLUMN_USAGE k \ + ON k.TABLE_SCHEMA = c.TABLE_SCHEMA AND k.TABLE_NAME = c.TABLE_NAME AND k.COLUMN_NAME = c.COLUMN_NAME \ + AND k.CONSTRAINT_NAME = 'PRIMARY' \ + WHERE c.TABLE_SCHEMA = {table_schema} \ + ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION" + ) +} + +// PG 家族走 pg_catalog（与结构搜索同路线）：对 PG 系分支（Kingbase/OpenGauss 等）兼容性 +// 优于 information_schema；data_type 取 t.typname 裸类型名，与逐表 getColumns 返回的类型 +// 字符串在文本/数值列子串匹配上等价（varchar/character varying 均命中 "char"）。 +fn postgres_family_columns_sql(schema: Option<&str>) -> String { + let schema_filter = match schema.filter(|value| !value.trim().is_empty()) { + Some(schema) => format!("n.nspname = {}", sql_string_literal(schema)), + None => "n.nspname NOT IN ('pg_catalog', 'information_schema')".to_string(), + }; + format!( + "SELECT c.relname AS table_name, a.attname AS column_name, t.typname AS data_type \ + , (pk.attnum IS NOT NULL) AS is_primary_key \ + FROM pg_catalog.pg_attribute a \ + JOIN pg_catalog.pg_class c ON c.oid = a.attrelid \ + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ + JOIN pg_catalog.pg_type t ON t.oid = a.atttypid \ + LEFT JOIN ( \ + SELECT i.indrelid AS relid, a2.attnum AS attnum \ + FROM pg_catalog.pg_index i \ + JOIN pg_catalog.pg_attribute a2 ON a2.attrelid = i.indrelid AND a2.attnum = ANY(i.indkey) \ + WHERE i.indisprimary ) pk ON pk.relid = c.oid AND pk.attnum = a.attnum \ + WHERE a.attnum > 0 AND NOT a.attisdropped AND c.relkind IN ('r', 'p') \ + AND {schema_filter} \ + ORDER BY c.relname, a.attnum" + ) +} + +fn sqlserver_columns_sql(schema: Option<&str>) -> String { + let schema_filter = match schema.filter(|value| !value.trim().is_empty()) { + Some(schema) => format!("c.TABLE_SCHEMA = {}", sql_string_literal(schema)), + None => "c.TABLE_SCHEMA NOT IN ('sys', 'INFORMATION_SCHEMA')".to_string(), + }; + format!( + "SELECT c.TABLE_NAME AS table_name, c.COLUMN_NAME AS column_name, c.DATA_TYPE AS data_type \ + , CASE WHEN pk.COLUMN_NAME IS NULL THEN 0 ELSE 1 END AS is_primary_key \ + FROM INFORMATION_SCHEMA.COLUMNS c \ + LEFT JOIN ( \ + SELECT kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME \ + FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc \ + JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu \ + ON kcu.CONSTRAINT_SCHEMA = tc.CONSTRAINT_SCHEMA AND kcu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME \ + AND kcu.TABLE_SCHEMA = tc.TABLE_SCHEMA \ + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' ) pk \ + ON pk.TABLE_SCHEMA = c.TABLE_SCHEMA AND pk.TABLE_NAME = c.TABLE_NAME AND pk.COLUMN_NAME = c.COLUMN_NAME \ + WHERE {schema_filter} \ + ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION" + ) +} + +fn sql_string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn oracle_family_uses_all_tab_columns_with_pk_join() { + let query = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Dameng), + database: "dlweb".to_string(), + schema: Some("DLWEB".to_string()), + }) + .unwrap(); + + assert!(query.sql.contains("FROM ALL_TAB_COLUMNS c")); + assert!(query.sql.contains("ALL_CONSTRAINTS cn")); + assert!(query.sql.contains("cn.CONSTRAINT_TYPE = 'P'")); + assert!(query.sql.contains("LEFT JOIN")); + assert!(query.sql.contains("WHERE c.OWNER = 'DLWEB'")); + // 带引号的小写别名保证 Oracle 驱动返回小写列头 + assert!(query.sql.contains("AS \"table_name\"")); + assert!(query.sql.contains("AS \"is_primary_key\"")); + assert!(query.sql.contains("ORDER BY c.TABLE_NAME, c.COLUMN_ID")); + } + + #[test] + fn oracle_family_falls_back_to_database_when_schema_missing() { + let query = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Oracle), + database: "APP".to_string(), + schema: None, + }) + .unwrap(); + assert!(query.sql.contains("WHERE c.OWNER = 'APP'")); + } + + #[test] + fn mysql_family_uses_information_schema_with_primary_key_usage() { + let query = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Mysql), + database: "shop".to_string(), + schema: None, + }) + .unwrap(); + + assert!(query.sql.contains("FROM information_schema.COLUMNS c")); + assert!(query.sql.contains("information_schema.KEY_COLUMN_USAGE k")); + assert!(query.sql.contains("k.CONSTRAINT_NAME = 'PRIMARY'")); + assert!(query.sql.contains("WHERE c.TABLE_SCHEMA = 'shop'")); + // 复合主键：join 按列名对齐，多行主键列各自命中 + assert!(query.sql.contains("k.COLUMN_NAME = c.COLUMN_NAME")); + assert!(query.sql.contains("ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION")); + } + + #[test] + fn postgres_family_uses_pg_catalog_and_excludes_system_schemas_by_default() { + let with_schema = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Postgres), + database: "app".to_string(), + schema: Some("public".to_string()), + }) + .unwrap(); + assert!(with_schema.sql.contains("pg_catalog.pg_attribute a")); + assert!(with_schema.sql.contains("i.indisprimary")); + assert!(with_schema.sql.contains("n.nspname = 'public'")); + assert!(with_schema.sql.contains("c.relkind IN ('r', 'p')")); + + let without_schema = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Postgres), + database: "app".to_string(), + schema: None, + }) + .unwrap(); + assert!(without_schema.sql.contains("n.nspname NOT IN ('pg_catalog', 'information_schema')")); + } + + #[test] + fn sqlserver_excludes_system_schemas_when_schema_missing() { + let query = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::SqlServer), + database: "master".to_string(), + schema: None, + }) + .unwrap(); + assert!(query.sql.contains("INFORMATION_SCHEMA.COLUMNS c")); + assert!(query.sql.contains("tc.CONSTRAINT_TYPE = 'PRIMARY KEY'")); + assert!(query.sql.contains("c.TABLE_SCHEMA NOT IN ('sys', 'INFORMATION_SCHEMA')")); + + let with_schema = build_database_search_columns_sql(DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::SqlServer), + database: "master".to_string(), + schema: Some("dbo".to_string()), + }) + .unwrap(); + assert!(with_schema.sql.contains("c.TABLE_SCHEMA = 'dbo'")); + } + + #[test] + fn unsupported_dialects_return_none() { + let options = DatabaseSearchColumnsSqlOptions { + database_type: Some(DatabaseType::Sqlite), + database: "local".to_string(), + schema: None, + }; + assert_eq!(build_database_search_columns_sql(options), None); + } +} diff --git a/crates/dbx-sql-data/src/database_search_probe_sql.rs b/crates/dbx-sql-data/src/database_search_probe_sql.rs new file mode 100644 index 0000000000..7276672a9b --- /dev/null +++ b/crates/dbx-sql-data/src/database_search_probe_sql.rs @@ -0,0 +1,267 @@ +use serde::{Deserialize, Serialize}; + +use crate::models::connection::DatabaseType; +use crate::sql_dialect::{ + firebird_rows_clause, pagination_strategy, qualified_table_name, PaginationContext, TablePaginationStrategy, +}; + +use crate::database_search_sql::{build_table_search_conditions, clamp_limit, DatabaseSearchColumn}; + +/// 多表探测查询：把一批表的搜索条件合并为一条 UNION ALL 语句， +/// 每个分支返回 (序号标签, 命中数)——分支内套与单表查询一致的 WHERE 条件和 +/// 该方言的行数限额，COUNT 结果即"该表最多有 limit 条命中"。 +/// 标签用分支序号 t0..tN 而非表名字面量，彻底绕开标识符转义； +/// included_indices[k] 是第 k 个分支对应的输入表下标。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseSearchProbeTable { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub table_name: String, + #[serde(default)] + pub columns: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseSearchProbeSqlOptions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_profile: Option, + #[serde(default)] + pub tables: Vec, + pub term: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseSearchProbeSql { + pub sql: String, + pub included_indices: Vec, +} + +pub fn build_database_search_probe_sql(options: DatabaseSearchProbeSqlOptions) -> Option { + let database_type = options.database_type; + let driver_profile = options.driver_profile.as_deref(); + let limit = clamp_limit(options.limit); + + let mut branches = Vec::new(); + let mut included_indices = Vec::new(); + for (index, table) in options.tables.iter().enumerate() { + // 无可搜列的表剔除出探测批，不中断其余表（进度由调用方按已处理推进）。 + let Some(conditions) = + build_table_search_conditions(database_type, driver_profile, &table.columns, &options.term) + else { + continue; + }; + let qualified = qualified_table_name(database_type, table.schema.as_deref(), &table.table_name); + let label = format!("t{}", branches.len()); + branches.push(probe_branch_sql(database_type, &qualified, &conditions.where_clause, &label, limit)); + included_indices.push(index); + } + + if branches.is_empty() { + return None; + } + Some(DatabaseSearchProbeSql { sql: branches.join("\nUNION ALL\n"), included_indices }) +} + +fn probe_branch_sql( + database_type: Option, + qualified_table: &str, + where_clause: &str, + label: &str, + limit: usize, +) -> String { + let limited_inner = match pagination_strategy(database_type, PaginationContext::BoundedRead) { + TablePaginationStrategy::SqlServerTop | TablePaginationStrategy::IrisTop => { + format!("SELECT TOP {limit} 1 FROM {qualified_table} WHERE ({where_clause})") + } + TablePaginationStrategy::InformixFirst => { + format!("SELECT FIRST {limit} 1 FROM {qualified_table} WHERE ({where_clause})") + } + TablePaginationStrategy::FirebirdRows => { + format!("SELECT 1 FROM {qualified_table} WHERE ({where_clause}) {}", firebird_rows_clause(limit, 0)) + } + TablePaginationStrategy::Db2FetchFirst | TablePaginationStrategy::FetchFirst => { + format!("SELECT 1 FROM {qualified_table} WHERE ({where_clause}) FETCH FIRST {limit} ROWS ONLY") + } + TablePaginationStrategy::Rownum => { + format!("SELECT 1 FROM {qualified_table} WHERE ({where_clause}) AND ROWNUM <= {limit}") + } + // 这些策略没有服务端限额可用，COUNT 会数完所有命中行——正确性不变，代价交给调用方批次控制。 + TablePaginationStrategy::AgentMaxRows + | TablePaginationStrategy::QuestDbLimit + | TablePaginationStrategy::LimitOffset + | TablePaginationStrategy::Unbounded => { + format!("SELECT 1 FROM {qualified_table} WHERE ({where_clause}) LIMIT {limit}") + } + }; + format!("SELECT '{label}' AS table_label, COUNT(*) AS hits FROM ({limited_inner}) probe_branch") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::database_search_sql::build_database_search_sql; + + fn col(name: &str, data_type: &str, is_primary_key: bool) -> DatabaseSearchColumn { + DatabaseSearchColumn { name: name.to_string(), data_type: data_type.to_string(), is_primary_key } + } + + fn probe_table(schema: Option<&str>, table: &str, columns: Vec) -> DatabaseSearchProbeTable { + DatabaseSearchProbeTable { schema: schema.map(str::to_string), table_name: table.to_string(), columns } + } + + fn probe_options( + database_type: DatabaseType, + tables: Vec, + ) -> DatabaseSearchProbeSqlOptions { + DatabaseSearchProbeSqlOptions { + database_type: Some(database_type), + driver_profile: None, + tables, + term: "alice".to_string(), + limit: Some(20), + } + } + + #[test] + fn oracle_family_branch_uses_rownum_limited_subquery() { + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::Dameng, + vec![probe_table(Some("DLWEB"), "USERS", vec![col("NAME", "varchar2", false), col("PIC", "blob", false)])], + )) + .unwrap(); + + assert_eq!(probe.included_indices, vec![0]); + assert_eq!( + probe.sql, + "SELECT 't0' AS table_label, COUNT(*) AS hits FROM \ + (SELECT 1 FROM \"DLWEB\".\"USERS\" WHERE (LOWER(CAST(\"NAME\" AS TEXT)) LIKE '%alice%' ESCAPE '~') AND ROWNUM <= 20) probe_branch" + ); + } + + #[test] + fn mysql_family_branch_uses_limit_subquery() { + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::Mysql, + vec![ + probe_table(None, "users", vec![col("email", "varchar", false)]), + probe_table(None, "orders", vec![col("note", "text", false)]), + ], + )) + .unwrap(); + + assert_eq!(probe.included_indices, vec![0, 1]); + assert!(probe.sql.contains("FROM `users` WHERE")); + assert!(probe.sql.contains("LIMIT 20")); + assert_eq!(probe.sql.matches("UNION ALL").count(), 1); + assert!(probe.sql.contains("'t0' AS table_label")); + assert!(probe.sql.contains("'t1' AS table_label")); + } + + #[test] + fn sqlserver_branch_uses_top_subquery() { + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::SqlServer, + vec![probe_table(Some("dbo"), "users", vec![col("name", "nvarchar", false)])], + )) + .unwrap(); + assert!(probe.sql.contains("SELECT TOP 20 1 FROM [dbo].[users]")); + } + + #[test] + fn postgres_branch_uses_limit_subquery() { + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::Postgres, + vec![probe_table(Some("public"), "orders", vec![col("note", "text", false)])], + )) + .unwrap(); + assert!(probe.sql.contains("FROM \"public\".\"orders\"")); + assert!(probe.sql.contains("LIMIT 20) probe_branch")); + } + + #[test] + fn tables_without_searchable_columns_are_excluded_and_indices_preserved() { + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::Mysql, + vec![ + probe_table(None, "files", vec![col("payload", "blob", false)]), + probe_table(None, "users", vec![col("email", "varchar", false)]), + probe_table(None, "empty", vec![]), + ], + )) + .unwrap(); + + // 无可搜列表被剔除，included_indices 指向原输入下标 + assert_eq!(probe.included_indices, vec![1]); + assert!(probe.sql.contains("'t0' AS table_label")); + assert!(probe.sql.contains("FROM `users`")); + assert!(!probe.sql.contains("`files`")); + assert!(!probe.sql.contains("`empty`")); + } + + #[test] + fn empty_or_all_excluded_input_returns_none() { + assert_eq!( + build_database_search_probe_sql(DatabaseSearchProbeSqlOptions { + database_type: Some(DatabaseType::Mysql), + driver_profile: None, + tables: vec![], + term: "alice".to_string(), + limit: Some(20), + }), + None + ); + assert_eq!( + build_database_search_probe_sql(probe_options( + DatabaseType::Mysql, + vec![probe_table(None, "files", vec![col("payload", "blob", false)])], + )), + None + ); + } + + #[test] + fn probe_branch_conditions_match_single_table_query_exactly() { + let columns = vec![col("email", "varchar", false), col("id", "bigint", true)]; + + let single = build_database_search_sql(crate::database_search_sql::DatabaseSearchSqlOptions { + database_type: Some(DatabaseType::Dameng), + driver_profile: None, + schema: Some("DLWEB".to_string()), + table_name: "USERS".to_string(), + columns: columns.clone(), + term: "alice".to_string(), + limit: Some(20), + }) + .unwrap(); + let probe = build_database_search_probe_sql(probe_options( + DatabaseType::Dameng, + vec![probe_table(Some("DLWEB"), "USERS", columns)], + )) + .unwrap(); + + // 探测分支内的 WHERE 条件与单表查询完全一致（含数值词场景由共享拼装函数保证） + let single_conds = single.sql.split(" WHERE (").nth(1).unwrap().split(")) WHERE ROWNUM").next().unwrap(); + assert!(probe.sql.contains(&format!("WHERE ({single_conds}) AND ROWNUM <= 20) probe_branch"))); + } + + #[test] + fn numeric_term_adds_equality_predicates_in_probe_branches() { + let probe = build_database_search_probe_sql(DatabaseSearchProbeSqlOptions { + database_type: Some(DatabaseType::Mysql), + driver_profile: None, + tables: vec![probe_table(None, "orders", vec![col("id", "bigint", true), col("note", "text", false)])], + term: "42".to_string(), + limit: Some(20), + }) + .unwrap(); + assert!(probe.sql.contains("`note`")); + assert!(probe.sql.contains("`id` = 42")); + } +} diff --git a/crates/dbx-sql-data/src/database_search_sql.rs b/crates/dbx-sql-data/src/database_search_sql.rs index a2a4d2aaad..bb85cd295b 100644 --- a/crates/dbx-sql-data/src/database_search_sql.rs +++ b/crates/dbx-sql-data/src/database_search_sql.rs @@ -75,42 +75,68 @@ pub fn is_numeric_search_column(column: &DatabaseSearchColumn) -> bool { NUMBER_TYPES.iter().any(|number_type| data_type.contains(number_type)) } -pub fn build_database_search_sql(options: DatabaseSearchSqlOptions) -> Option { - let term = clean_term(&options.term); +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableSearchConditions { + pub where_clause: String, + pub searchable_columns: Vec, +} + +/// 单表搜索条件拼装：数据搜索单表查询与多表探测分支共用， +/// 保证两条路径对同一表同一关键词产出完全一致的 WHERE 条件。 +pub fn build_table_search_conditions( + database_type: Option, + driver_profile: Option<&str>, + columns: &[DatabaseSearchColumn], + term: &str, +) -> Option { + let term = clean_term(term); if term.is_empty() { return None; } - let text_columns: Vec<_> = options.columns.iter().filter(|column| is_text_search_column(column)).collect(); + let text_columns: Vec<_> = columns.iter().filter(|column| is_text_search_column(column)).collect(); let numeric_term = parse_numeric_term(&term); let numeric_columns: Vec<_> = if numeric_term.is_some() { - options.columns.iter().filter(|column| is_numeric_search_column(column)).collect() + columns.iter().filter(|column| is_numeric_search_column(column)).collect() } else { Vec::new() }; let mut conditions = Vec::new(); for column in &text_columns { - let identifier = quote_table_identifier(options.database_type, &column.name); - conditions.push(format!( - "{} LIKE {} ESCAPE '~'", - text_cast_expression(options.database_type, options.driver_profile.as_deref(), &identifier), - sql_string_literal(&like_pattern(&term)) - )); + let identifier = quote_table_identifier(database_type, &column.name); + conditions.push(text_search_condition(database_type, driver_profile, column, &identifier, &term)); } for column in &numeric_columns { - let identifier = quote_table_identifier(options.database_type, &column.name); + let identifier = quote_table_identifier(database_type, &column.name); conditions.push(format!("{identifier} = {}", numeric_term.as_deref().unwrap_or_default())); } if conditions.is_empty() { return None; } + Some(TableSearchConditions { + where_clause: conditions.join(" OR "), + searchable_columns: text_columns + .iter() + .chain(numeric_columns.iter()) + .map(|column| column.name.clone()) + .collect(), + }) +} + +pub fn build_database_search_sql(options: DatabaseSearchSqlOptions) -> Option { + let conditions = build_table_search_conditions( + options.database_type, + options.driver_profile.as_deref(), + &options.columns, + &options.term, + )?; let table = qualified_table_name(options.database_type, options.schema.as_deref(), &options.table_name); - let where_clause = conditions.join(" OR "); + let where_clause = conditions.where_clause; let limit = clamp_limit(options.limit); - let searchable_columns = - text_columns.iter().chain(numeric_columns.iter()).map(|column| column.name.clone()).collect(); + let searchable_columns = conditions.searchable_columns; let sql = match pagination_strategy(options.database_type, PaginationContext::BoundedRead) { TablePaginationStrategy::SqlServerTop => format!("SELECT TOP {limit} * FROM {table} WHERE ({where_clause})"), @@ -179,7 +205,7 @@ fn clean_term(term: &str) -> String { term.trim().to_string() } -fn clamp_limit(limit: Option) -> usize { +pub(crate) fn clamp_limit(limit: Option) -> usize { limit.unwrap_or(20).clamp(1, 100) } @@ -248,6 +274,35 @@ fn text_cast_expression(database_type: Option, driver_profile: Opt } } +/// Oracle 家族的 CLOB 列不能走 CAST AS VARCHAR2(4000)：内容超 4000 字符时直接抛 +/// ORA-22835（整表不可搜）。改用 DBMS_LOB.INSTR 全量搜索——不截断、不报错； +/// LOWER(CLOB) 返回 CLOB，INSTR 对 CLOB 直接可用。DM8/OB Oracle 模式均提供 DBMS_LOB。 +/// INSTR 按字面子串匹配，与 LIKE '%term%' ESCAPE（通配符已被转义为字面量）语义一致。 +fn is_oracle_family_clob(database_type: Option, data_type: &str) -> bool { + matches!(database_type, Some(DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng)) + && data_type.to_ascii_lowercase().contains("clob") +} + +fn text_search_condition( + database_type: Option, + driver_profile: Option<&str>, + column: &DatabaseSearchColumn, + identifier: &str, + term: &str, +) -> String { + if is_oracle_family_clob(database_type, &column.data_type) { + return format!( + "DBMS_LOB.INSTR(LOWER({identifier}), {}) > 0", + sql_string_literal(&clean_term(term).to_ascii_lowercase()) + ); + } + format!( + "{} LIKE {} ESCAPE '~'", + text_cast_expression(database_type, driver_profile, identifier), + sql_string_literal(&like_pattern(term)) + ) +} + fn sql_value_literal(database_type: Option, column: &DatabaseSearchColumn, value: &Value) -> String { match value { Value::Null => "NULL".to_string(), @@ -281,6 +336,51 @@ mod tests { DatabaseSearchColumn { name: name.to_string(), data_type: data_type.to_string(), is_primary_key } } + #[test] + fn oracle_family_clob_columns_use_dbms_lob_instr() { + let query = build_database_search_sql(DatabaseSearchSqlOptions { + database_type: Some(DatabaseType::Oracle), + driver_profile: None, + schema: Some("ZOEDM".to_string()), + table_name: "CLOSED_LOOP_VIEW_LOG".to_string(), + columns: vec![col("ID", "number", true), col("TITLE", "varchar2", false), col("CONTENT", "clob", false)], + term: "closed_loop".to_string(), + limit: Some(20), + }) + .unwrap(); + + // CLOB 列走 DBMS_LOB.INSTR(全量、不抛 ORA-22835),varchar2 列保持 CAST+LIKE + assert!(query.sql.contains("DBMS_LOB.INSTR(LOWER(\"CONTENT\"), 'closed_loop') > 0")); + // LIKE 侧通配符照常转义(_ → ~_) + assert!(query.sql.contains("LOWER(CAST(\"TITLE\" AS VARCHAR2(4000))) LIKE '%closed~_loop%' ESCAPE '~'")); + + // Dameng / OceanBase Oracle 同为 Oracle 家族 + let dameng = build_database_search_sql(DatabaseSearchSqlOptions { + database_type: Some(DatabaseType::Dameng), + driver_profile: None, + schema: None, + table_name: "t".to_string(), + columns: vec![col("BODY", "clob", false)], + term: "x".to_string(), + limit: None, + }) + .unwrap(); + assert!(dameng.sql.contains("DBMS_LOB.INSTR(LOWER(\"BODY\"), 'x') > 0")); + + // 其他方言的 clob/text 列不受影响,仍走 CAST+LIKE + let mysql = build_database_search_sql(DatabaseSearchSqlOptions { + database_type: Some(DatabaseType::Mysql), + driver_profile: None, + schema: None, + table_name: "t".to_string(), + columns: vec![col("body", "longtext", false)], + term: "x".to_string(), + limit: None, + }) + .unwrap(); + assert!(mysql.sql.contains("LIKE '%x%'")); + } + #[test] fn builds_table_search_query_over_text_columns() { let query = build_database_search_sql(DatabaseSearchSqlOptions { diff --git a/crates/dbx-sql-data/src/database_structure_search_sql.rs b/crates/dbx-sql-data/src/database_structure_search_sql.rs index 41da857d6f..b2ba541829 100644 --- a/crates/dbx-sql-data/src/database_structure_search_sql.rs +++ b/crates/dbx-sql-data/src/database_structure_search_sql.rs @@ -285,7 +285,7 @@ fn oracle_family_structure_search_sql( LEFT JOIN ALL_COL_COMMENTS m ON c.OWNER = m.OWNER AND c.TABLE_NAME = m.TABLE_NAME AND c.COLUMN_NAME = m.COLUMN_NAME \ WHERE c.OWNER = {owner} \ AND ({name_expr} LIKE LOWER({pattern}) ESCAPE '~' OR LOWER(COALESCE(m.COMMENTS, '')) LIKE LOWER({pattern}) ESCAPE '~') \ - ORDER BY object_schema, object_name, column_name) WHERE ROWNUM <= {limit} \ + ORDER BY 1, 2, 3) WHERE ROWNUM <= {limit} \ UNION ALL \ SELECT * FROM ( \ SELECT t.OWNER AS object_schema, t.TABLE_NAME AS object_name, CAST(NULL AS VARCHAR2(128)) AS column_name \ @@ -293,8 +293,8 @@ fn oracle_family_structure_search_sql( FROM ALL_TAB_COMMENTS t \ WHERE t.OWNER = {owner} \ AND LOWER(COALESCE(t.COMMENTS, '')) LIKE LOWER({pattern}) ESCAPE '~' \ - ORDER BY object_schema, object_name) WHERE ROWNUM <= {table_limit} \ - ORDER BY object_schema, object_name, column_name) WHERE ROWNUM <= {total_limit}", + ORDER BY 1, 2) WHERE ROWNUM <= {table_limit} \ + ORDER BY 1, 2, 3) WHERE ROWNUM <= {total_limit}", owner = owner_literal, name_expr = name_expr, pattern = pattern_literal, diff --git a/crates/dbx-sql-data/src/lib.rs b/crates/dbx-sql-data/src/lib.rs index e052dd5789..517332da8b 100644 --- a/crates/dbx-sql-data/src/lib.rs +++ b/crates/dbx-sql-data/src/lib.rs @@ -6,6 +6,8 @@ pub use dbx_types::{database_manifest, models, types}; pub mod data_grid_extractors; pub mod data_grid_sql; +pub mod database_search_columns_sql; +pub mod database_search_probe_sql; pub mod database_search_sql; pub mod database_structure_search_sql; pub mod query_result_sql; diff --git a/crates/dbx-sql/src/lib.rs b/crates/dbx-sql/src/lib.rs index f37c4f06e3..346c6c046a 100644 --- a/crates/dbx-sql/src/lib.rs +++ b/crates/dbx-sql/src/lib.rs @@ -6,7 +6,8 @@ pub use dbx_sql_core::{ value_literals, }; pub use dbx_sql_data::{ - data_grid_extractors, data_grid_sql, database_search_sql, database_structure_search_sql, query_result_sql, + data_grid_extractors, data_grid_sql, database_search_columns_sql, database_search_probe_sql, database_search_sql, + database_structure_search_sql, query_result_sql, }; pub use dbx_sql_dialect::{dml_binding, postgres_index_key, sql_dialect}; pub use dbx_sql_schema::{db_admin_sql, schema_diff, sql_parser, table_structure_sql}; diff --git a/crates/dbx-web/src/main.rs b/crates/dbx-web/src/main.rs index 18ebb79470..3c1e99398e 100644 --- a/crates/dbx-web/src/main.rs +++ b/crates/dbx-web/src/main.rs @@ -867,6 +867,8 @@ async fn serve() -> Result<(), String> { .route("/query/build-create-user-sql", post(routes::query::build_create_user_sql)) .route("/query/build-table-select-sql", post(routes::query::build_table_select_sql)) .route("/query/build-database-search-sql", post(routes::query::build_database_search_sql)) + .route("/query/build-database-search-columns-sql", post(routes::query::build_database_search_columns_sql)) + .route("/query/build-database-search-probe-sql", post(routes::query::build_database_search_probe_sql)) .route("/query/build-database-structure-search-sql", post(routes::query::build_database_structure_search_sql)) .route("/query/supports-database-structure-search", post(routes::query::supports_database_structure_search)) .route("/query/build-search-result-where", post(routes::query::build_search_result_where)) diff --git a/crates/dbx-web/src/routes/query.rs b/crates/dbx-web/src/routes/query.rs index e048dba300..bfc945197d 100644 --- a/crates/dbx-web/src/routes/query.rs +++ b/crates/dbx-web/src/routes/query.rs @@ -152,6 +152,18 @@ pub struct BuildDatabaseStructureSearchSqlRequest { pub options: dbx_core::database_structure_search_sql::DatabaseStructureSearchSqlOptions, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildDatabaseSearchColumnsSqlRequest { + pub options: dbx_core::database_search_columns_sql::DatabaseSearchColumnsSqlOptions, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildDatabaseSearchProbeSqlRequest { + pub options: dbx_core::database_search_probe_sql::DatabaseSearchProbeSqlOptions, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct SupportsDatabaseStructureSearchRequest { @@ -1154,6 +1166,18 @@ pub async fn build_database_structure_search_sql( Json(dbx_core::database_structure_search_sql::build_database_structure_search_sql(req.options)) } +pub async fn build_database_search_columns_sql( + Json(req): Json, +) -> Json> { + Json(dbx_core::database_search_columns_sql::build_database_search_columns_sql(req.options)) +} + +pub async fn build_database_search_probe_sql( + Json(req): Json, +) -> Json> { + Json(dbx_core::database_search_probe_sql::build_database_search_probe_sql(req.options)) +} + pub async fn supports_database_structure_search(Json(req): Json) -> Json { Json(dbx_core::database_structure_search_sql::supports_database_structure_search( req.database_type, diff --git a/src-tauri/src/commands/query.rs b/src-tauri/src/commands/query.rs index 31b11d1b49..abd0f5aa85 100644 --- a/src-tauri/src/commands/query.rs +++ b/src-tauri/src/commands/query.rs @@ -704,6 +704,20 @@ pub fn build_database_structure_search_sql( Ok(dbx_core::database_structure_search_sql::build_database_structure_search_sql(options)) } +#[tauri::command] +pub fn build_database_search_columns_sql( + options: dbx_core::database_search_columns_sql::DatabaseSearchColumnsSqlOptions, +) -> Result, String> { + Ok(dbx_core::database_search_columns_sql::build_database_search_columns_sql(options)) +} + +#[tauri::command] +pub fn build_database_search_probe_sql( + options: dbx_core::database_search_probe_sql::DatabaseSearchProbeSqlOptions, +) -> Result, String> { + Ok(dbx_core::database_search_probe_sql::build_database_search_probe_sql(options)) +} + #[tauri::command] pub fn supports_database_structure_search( database_type: Option, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 76c98ba42c..f37a8ebd46 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -2365,6 +2365,8 @@ pub fn run() { commands::query::build_dropped_file_preview_sql, commands::query::build_table_select_sql, commands::query::build_database_search_sql, + commands::query::build_database_search_columns_sql, + commands::query::build_database_search_probe_sql, commands::query::build_database_structure_search_sql, commands::query::supports_database_structure_search, commands::query::build_search_result_where,