From 2e8ed4b485ecd2919da6c414f8cc532891fd35a9 Mon Sep 17 00:00:00 2001 From: zipg Date: Tue, 1 Sep 2026 12:55:48 +0800 Subject: [PATCH] feat(search): batch scan all tables in database search --- .../search/DatabaseSearchDialog.vue | 91 ++++++++++++++----- apps/desktop/src/i18n/locales/en.ts | 3 +- apps/desktop/src/i18n/locales/es.ts | 3 +- apps/desktop/src/i18n/locales/it.ts | 3 +- apps/desktop/src/i18n/locales/ja.ts | 3 +- apps/desktop/src/i18n/locales/ko.ts | 3 +- apps/desktop/src/i18n/locales/pt-BR.ts | 3 +- apps/desktop/src/i18n/locales/zh-CN.ts | 3 +- apps/desktop/src/i18n/locales/zh-TW.ts | 3 +- .../src/lib/database/databaseSearchBatch.ts | 21 +++++ packages/app-tests/databaseSearch.test.ts | 15 +++ .../app-tests/databaseSearchDialog.test.ts | 29 ++++++ 12 files changed, 149 insertions(+), 31 deletions(-) create mode 100644 apps/desktop/src/lib/database/databaseSearchBatch.ts create mode 100644 packages/app-tests/databaseSearchDialog.test.ts diff --git a/apps/desktop/src/components/search/DatabaseSearchDialog.vue b/apps/desktop/src/components/search/DatabaseSearchDialog.vue index 1da6a8aef2..896300c3c6 100644 --- a/apps/desktop/src/components/search/DatabaseSearchDialog.vue +++ b/apps/desktop/src/components/search/DatabaseSearchDialog.vue @@ -11,6 +11,7 @@ import { Label } from "@/components/ui/label"; import { useConnectionStore } from "@/stores/connectionStore"; import * as api from "@/lib/backend/api"; import { buildDatabaseSearchSql, buildSearchResultWhere, findMatchedSearchColumns } from "@/lib/database/databaseSearch"; +import { databaseSearchBatchRange, databaseSearchNextBatchSize } from "@/lib/database/databaseSearchBatch"; import type { DatabaseType, TableInfo } from "@/types/database"; import { isSchemaAware } from "@/lib/database/databaseCapabilities"; @@ -58,7 +59,6 @@ type SearchTableTask = { }; const SYSTEM_SCHEMAS = new Set(["information_schema", "pg_catalog", "sys", "system", "mysql", "performance_schema", "xdb", "outln", "dbsnmp"]); -const MAX_TABLES = 200; const keyword = ref(""); const perTableLimit = ref(20); @@ -70,14 +70,20 @@ const progressTotal = ref(0); const results = ref([]); const tableErrors = ref>([]); const generalError = ref(""); -const limitedTables = ref(false); const currentExecutionId = ref(""); +const tableTasks = ref([]); +const nextTableIndex = ref(0); +const activeKeyword = ref(""); +const activePerTableLimit = ref(20); let runId = 0; const connection = computed(() => (props.prefillConnectionId ? connectionStore.getConfig(props.prefillConnectionId) : undefined)); const scopeLabel = computed(() => [connection.value?.name, props.prefillDatabase, props.prefillSchema].filter(Boolean).join(" / ")); const canSearch = computed(() => Boolean(props.prefillConnectionId && props.prefillDatabase && keyword.value.trim() && !running.value)); const progressLabel = computed(() => t("databaseSearch.progress", { done: progressDone.value, total: progressTotal.value })); +const remainingTableCount = computed(() => Math.max(0, tableTasks.value.length - nextTableIndex.value)); +const nextBatchSize = computed(() => databaseSearchNextBatchSize(nextTableIndex.value, tableTasks.value.length)); +const canContinueSearch = computed(() => !running.value && !loadingTables.value && remainingTableCount.value > 0); watch( dialogOpen, @@ -100,8 +106,11 @@ function resetSearchState() { results.value = []; tableErrors.value = []; generalError.value = ""; - limitedTables.value = false; currentExecutionId.value = ""; + tableTasks.value = []; + nextTableIndex.value = 0; + activeKeyword.value = ""; + activePerTableLimit.value = 20; } function isSearchableTable(table: TableInfo): boolean { @@ -154,10 +163,6 @@ async function listSearchTables(): Promise { } } - if (tasks.length > MAX_TABLES) { - limitedTables.value = true; - return tasks.slice(0, MAX_TABLES); - } return tasks; } @@ -167,21 +172,18 @@ async function startSearch() { resetSearchState(); running.value = true; loadingTables.value = true; + activeKeyword.value = keyword.value.trim(); + activePerTableLimit.value = Math.min(100, Math.max(1, Math.trunc(Number(perTableLimit.value) || 20))); try { await connectionStore.ensureConnected(props.prefillConnectionId); - const tasks = await listSearchTables(); + tableTasks.value = await listSearchTables(); if (currentRun !== runId || cancelled.value) return; - progressTotal.value = tasks.length; + progressTotal.value = tableTasks.value.length; loadingTables.value = false; - - for (const task of tasks) { - if (currentRun !== runId || cancelled.value) break; - await searchTable(task, connection.value.db_type, currentRun); - progressDone.value += 1; - } + await scanTableRange(currentRun, false); } catch (error: any) { - if (!cancelled.value) generalError.value = error?.message || String(error); + if (currentRun === runId && !cancelled.value) generalError.value = error?.message || String(error); } finally { if (currentRun === runId) { running.value = false; @@ -191,6 +193,39 @@ async function startSearch() { } } +async function continueSearch(scanAllRemaining: boolean) { + if (!canContinueSearch.value || !connection.value) return; + const currentRun = ++runId; + cancelled.value = false; + running.value = true; + generalError.value = ""; + + try { + await connectionStore.ensureConnected(props.prefillConnectionId); + await scanTableRange(currentRun, scanAllRemaining); + } catch (error: any) { + if (currentRun === runId && !cancelled.value) generalError.value = error?.message || String(error); + } finally { + if (currentRun === runId) { + running.value = false; + currentExecutionId.value = ""; + } + } +} + +async function scanTableRange(currentRun: number, scanAllRemaining: boolean) { + const range = databaseSearchBatchRange(nextTableIndex.value, tableTasks.value.length, scanAllRemaining); + while (nextTableIndex.value < range.end) { + if (currentRun !== runId || cancelled.value) break; + const task = tableTasks.value[nextTableIndex.value]; + if (!task) break; + await searchTable(task, connection.value!.db_type, currentRun); + if (currentRun !== runId || cancelled.value) break; + nextTableIndex.value += 1; + progressDone.value = nextTableIndex.value; + } +} + async function searchTable(task: SearchTableTask, databaseType: DatabaseType, currentRun: number) { const tableLabel = task.schema ? `${task.schema}.${task.table.name}` : task.table.name; try { @@ -202,8 +237,8 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu schema: task.schema, tableName: task.table.name, columns, - term: keyword.value, - limit: perTableLimit.value, + term: activeKeyword.value, + limit: activePerTableLimit.value, }); if (!query || currentRun !== runId || cancelled.value) return; @@ -213,8 +248,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu if (currentExecutionId.value === executionId) currentExecutionId.value = ""; if (currentRun !== runId || cancelled.value) return; + const tableResults: SearchResultItem[] = []; for (const [rowIndex, row] of result.rows.entries()) { - const matchedColumns = findMatchedSearchColumns(result.columns, row, columns, keyword.value); + const matchedColumns = findMatchedSearchColumns(result.columns, row, columns, activeKeyword.value); const whereInput = await buildSearchResultWhere({ databaseType, columns, @@ -222,8 +258,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu row, matchedColumns, }); - results.value.push({ - id: `${tableLabel}:${rowIndex}:${results.value.length}`, + if (currentRun !== runId || cancelled.value) return; + tableResults.push({ + id: `${tableLabel}:${rowIndex}`, schema: task.schema, tableName: task.table.name, tableType: task.table.table_type, @@ -232,8 +269,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu whereInput, }); } + if (currentRun === runId && !cancelled.value) results.value.push(...tableResults); } catch (error: any) { - if (!cancelled.value) { + if (currentRun === runId && !cancelled.value) { tableErrors.value.push({ tableName: tableLabel, message: error?.message || String(error) }); } } @@ -312,7 +350,14 @@ function openResult(item: SearchResultItem) {
{{ loadingTables ? t("databaseSearch.loadingTables") : progressLabel }} {{ t("databaseSearch.resultCount", { count: results.length }) }} - {{ t("databaseSearch.limitedTables", { count: 200 }) }} +
+ + +
diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index 5ffd1403b1..92cf7aaa30 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -2434,7 +2434,8 @@ export default { loadingTables: "Reading table list...", progress: "Scanned {done}/{total} tables", resultCount: "{count} matches", - limitedTables: "Large schema detected; only the first {count} tables are scanned", + continueNextBatch: "Scan next {count}", + scanAllRemaining: "Scan remaining {count}", results: "Results", openResult: "Open", waiting: "Waiting for matches...", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index 4320ec9d7b..8d614e8382 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -2369,7 +2369,8 @@ export default withEnglishFallback({ loadingTables: "Leyendo lista de tablas...", progress: "Analizadas {done}/{total} tablas", resultCount: "{count} coincidencias", - limitedTables: "Esquema grande detectado; solo se analizan las primeras {count} tablas", + continueNextBatch: "Analizar las siguientes {count}", + scanAllRemaining: "Analizar las {count} restantes", results: "Resultados", openResult: "Abrir", waiting: "Esperando coincidencias...", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index 3bde919e42..8be0bbcc65 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -2367,7 +2367,8 @@ export default withEnglishFallback({ loadingTables: "Lettura dell'elenco delle tabelle...", progress: "Scansionate {done}/{total} tabelle", resultCount: "{count} corrispondenze", - limitedTables: "Rilevato schema di grandi dimensioni; vengono scansionate solo le prime {count} tabelle", + continueNextBatch: "Scansiona le prossime {count}", + scanAllRemaining: "Scansiona le {count} rimanenti", results: "Risultati", openResult: "Apri", waiting: "In attesa di corrispondenze...", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index fa177e01b2..98e0eedc1f 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -2394,7 +2394,8 @@ export default withEnglishFallback({ loadingTables: "テーブル一覧を読み取り中...", progress: "{done}/{total}テーブルをスキャンしました", resultCount: "{count}件の一致", - limitedTables: "大規模スキーマが検出されました。最初の{count}テーブルのみスキャンされます", + continueNextBatch: "次の{count}テーブルをスキャン", + scanAllRemaining: "残り{count}テーブルをスキャン", results: "結果", openResult: "開く", waiting: "一致を待機中...", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index 96c8cab252..9841398b41 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -2268,7 +2268,8 @@ export default withEnglishFallback({ loadingTables: "테이블 목록을 읽는 중...", progress: "테이블 {done}/{total}개 스캔", resultCount: "{count}개 일치", - limitedTables: "대규모 스키마가 감지되어 처음 {count}개 테이블만 스캔합니다", + continueNextBatch: "다음 {count}개 스캔", + scanAllRemaining: "남은 {count}개 스캔", results: "결과", openResult: "열기", waiting: "일치 항목을 기다리는 중...", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index 10b062b7a3..dfbc60e48b 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -2369,7 +2369,8 @@ export default withEnglishFallback({ loadingTables: "Lendo lista de tabelas...", progress: "{done}/{total} tabelas verificadas", resultCount: "{count} correspondências", - limitedTables: "Schema grande detectado; apenas as primeiras {count} tabelas são verificadas", + continueNextBatch: "Verificar próximas {count}", + scanAllRemaining: "Verificar {count} restantes", results: "Resultados", openResult: "Abrir", waiting: "Aguardando correspondências...", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index 4755356795..c471db4065 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -2357,7 +2357,8 @@ export default withEnglishFallback({ loadingTables: "正在读取表列表...", progress: "已扫描 {done}/{total} 张表", resultCount: "{count} 条命中", - limitedTables: "表数量较多，本次仅扫描前 {count} 张表", + continueNextBatch: "继续扫描下 {count} 张", + scanAllRemaining: "扫描剩余 {count} 张", results: "查找结果", openResult: "打开", waiting: "等待命中结果...", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index c3b865f238..3594a992d2 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -2367,7 +2367,8 @@ export default withEnglishFallback({ loadingTables: "正在讀取表清單……", progress: "已掃描 {done}/{total} 張資料表", resultCount: "{count} 筆結果", - limitedTables: "偵測到大型 schema；僅掃描前 {count} 張資料表", + continueNextBatch: "繼續掃描下 {count} 張", + scanAllRemaining: "掃描剩餘 {count} 張", results: "搜尋結果", openResult: "開啟", waiting: "搜尋中……", diff --git a/apps/desktop/src/lib/database/databaseSearchBatch.ts b/apps/desktop/src/lib/database/databaseSearchBatch.ts new file mode 100644 index 0000000000..550ee3d0d9 --- /dev/null +++ b/apps/desktop/src/lib/database/databaseSearchBatch.ts @@ -0,0 +1,21 @@ +export const DATABASE_SEARCH_TABLE_BATCH_SIZE = 200; + +export type DatabaseSearchBatchRange = { + start: number; + end: number; +}; + +export function databaseSearchBatchRange(nextIndex: number, total: number, scanAllRemaining = false, batchSize = DATABASE_SEARCH_TABLE_BATCH_SIZE): DatabaseSearchBatchRange { + const safeTotal = Math.max(0, Math.trunc(total)); + const start = Math.min(Math.max(0, Math.trunc(nextIndex)), safeTotal); + const safeBatchSize = Math.max(1, Math.trunc(batchSize)); + return { + start, + end: scanAllRemaining ? safeTotal : Math.min(start + safeBatchSize, safeTotal), + }; +} + +export function databaseSearchNextBatchSize(nextIndex: number, total: number, batchSize = DATABASE_SEARCH_TABLE_BATCH_SIZE): number { + const range = databaseSearchBatchRange(nextIndex, total, false, batchSize); + return range.end - range.start; +} diff --git a/packages/app-tests/databaseSearch.test.ts b/packages/app-tests/databaseSearch.test.ts index 55eed2c2c9..47388ce81e 100644 --- a/packages/app-tests/databaseSearch.test.ts +++ b/packages/app-tests/databaseSearch.test.ts @@ -2,6 +2,7 @@ import { test } from "vitest"; import assert from "node:assert/strict"; import type { ColumnInfo } from "../../apps/desktop/src/types/database.ts"; import { findMatchedSearchColumns, isNumericSearchColumn, isTextSearchColumn } from "../../apps/desktop/src/lib/database/databaseSearch.ts"; +import { DATABASE_SEARCH_TABLE_BATCH_SIZE, databaseSearchBatchRange, databaseSearchNextBatchSize } from "../../apps/desktop/src/lib/database/databaseSearchBatch.ts"; function col(name: string, dataType: string, primary = false): ColumnInfo { return { @@ -26,3 +27,17 @@ test("finds matched columns from returned rows", () => { assert.deepEqual(matches, ["email"]); }); + +test("treats 200 tables as a resumable batch instead of a search cap", () => { + assert.equal(DATABASE_SEARCH_TABLE_BATCH_SIZE, 200); + assert.deepEqual(databaseSearchBatchRange(0, 450), { start: 0, end: 200 }); + assert.deepEqual(databaseSearchBatchRange(200, 450), { start: 200, end: 400 }); + assert.deepEqual(databaseSearchBatchRange(400, 450), { start: 400, end: 450 }); + assert.equal(databaseSearchNextBatchSize(400, 450), 50); +}); + +test("can scan every remaining table without exceeding the discovered table list", () => { + assert.deepEqual(databaseSearchBatchRange(200, 450, true), { start: 200, end: 450 }); + assert.deepEqual(databaseSearchBatchRange(999, 450, true), { start: 450, end: 450 }); + assert.deepEqual(databaseSearchBatchRange(-10, 5, false, 0), { start: 0, end: 1 }); +}); diff --git a/packages/app-tests/databaseSearchDialog.test.ts b/packages/app-tests/databaseSearchDialog.test.ts new file mode 100644 index 0000000000..3f0130657b --- /dev/null +++ b/packages/app-tests/databaseSearchDialog.test.ts @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "vitest"; +import { compileScript, compileTemplate, parse } from "vue/compiler-sfc"; + +const dialogPath = "apps/desktop/src/components/search/DatabaseSearchDialog.vue"; +const dialogSource = readFileSync(dialogPath, "utf8"); + +test("database search dialog compiles", () => { + const { descriptor, errors } = parse(dialogSource, { filename: dialogPath }); + assert.deepEqual(errors, []); + assert.ok(descriptor.scriptSetup); + compileScript(descriptor, { id: dialogPath }); + assert.ok(descriptor.template); + const template = compileTemplate({ id: dialogPath, filename: dialogPath, source: descriptor.template!.content }); + assert.deepEqual(template.errors, []); +}); + +test("database search continues after each table batch without retaining a hard cap", () => { + assert.doesNotMatch(dialogSource, /MAX_TABLES|tasks\.slice\(0/); + assert.match(dialogSource, /progressTotal\.value = tableTasks\.value\.length/); + assert.match(dialogSource, /await scanTableRange\(currentRun, false\)/); + assert.match(dialogSource, /@click="continueSearch\(false\)"/); + assert.match(dialogSource, /@click="continueSearch\(true\)"/); + assert.match(dialogSource, /activeKeyword\.value/); + assert.match(dialogSource, /activePerTableLimit\.value/); + assert.match(dialogSource, /const tableResults: SearchResultItem\[\] = \[\]/); + assert.match(dialogSource, /currentRun === runId && !cancelled\.value/); +});