feat(search): batch scan all tables in database search

This commit is contained in:
zipg authored and GitHub committed 2026-09-01 12:55:48 +08:00
1 parent 8f21e7af9f
commit 2e8ed4b485
12 files changed
+149 -31

No files matched your search

@@ -11,6 +11,7 @@ import { Label } from "@/components/ui/label";
import { useConnectionStore } from "@/stores/connectionStore"; import { useConnectionStore } from "@/stores/connectionStore";
import * as api from "@/lib/backend/api"; import * as api from "@/lib/backend/api";
import { buildDatabaseSearchSql, buildSearchResultWhere, findMatchedSearchColumns } from "@/lib/database/databaseSearch"; import { buildDatabaseSearchSql, buildSearchResultWhere, findMatchedSearchColumns } from "@/lib/database/databaseSearch";
import { databaseSearchBatchRange, databaseSearchNextBatchSize } from "@/lib/database/databaseSearchBatch";
import type { DatabaseType, TableInfo } from "@/types/database"; import type { DatabaseType, TableInfo } from "@/types/database";
import { isSchemaAware } from "@/lib/database/databaseCapabilities"; import { isSchemaAware } from "@/lib/database/databaseCapabilities";
@@ -58,7 +59,6 @@ type SearchTableTask = {
}; };
const SYSTEM_SCHEMAS = new Set(["information_schema", "pg_catalog", "sys", "system", "mysql", "performance_schema", "xdb", "outln", "dbsnmp"]); const SYSTEM_SCHEMAS = new Set(["information_schema", "pg_catalog", "sys", "system", "mysql", "performance_schema", "xdb", "outln", "dbsnmp"]);
const MAX_TABLES = 200;
const keyword = ref(""); const keyword = ref("");
const perTableLimit = ref(20); const perTableLimit = ref(20);
@@ -70,14 +70,20 @@ const progressTotal = ref(0);
const results = ref<SearchResultItem[]>([]); const results = ref<SearchResultItem[]>([]);
const tableErrors = ref<Array<{ tableName: string; message: string }>>([]); const tableErrors = ref<Array<{ tableName: string; message: string }>>([]);
const generalError = ref(""); const generalError = ref("");
const limitedTables = ref(false);
const currentExecutionId = ref(""); const currentExecutionId = ref("");
const tableTasks = ref<SearchTableTask[]>([]);
const nextTableIndex = ref(0);
const activeKeyword = ref("");
const activePerTableLimit = ref(20);
let runId = 0; let runId = 0;
const connection = computed(() => (props.prefillConnectionId ? connectionStore.getConfig(props.prefillConnectionId) : undefined)); const connection = computed(() => (props.prefillConnectionId ? connectionStore.getConfig(props.prefillConnectionId) : undefined));
const scopeLabel = computed(() => [connection.value?.name, props.prefillDatabase, props.prefillSchema].filter(Boolean).join(" / ")); const scopeLabel = computed(() => [connection.value?.name, props.prefillDatabase, props.prefillSchema].filter(Boolean).join(" / "));
const canSearch = computed(() => Boolean(props.prefillConnectionId && props.prefillDatabase && keyword.value.trim() && !running.value)); const canSearch = computed(() => Boolean(props.prefillConnectionId && props.prefillDatabase && keyword.value.trim() && !running.value));
const progressLabel = computed(() => t("databaseSearch.progress", { done: progressDone.value, total: progressTotal.value })); const progressLabel = computed(() => t("databaseSearch.progress", { done: progressDone.value, total: progressTotal.value }));
const remainingTableCount = computed(() => Math.max(0, tableTasks.value.length - nextTableIndex.value));
const nextBatchSize = computed(() => databaseSearchNextBatchSize(nextTableIndex.value, tableTasks.value.length));
const canContinueSearch = computed(() => !running.value && !loadingTables.value && remainingTableCount.value > 0);
watch( watch(
dialogOpen, dialogOpen,
@@ -100,8 +106,11 @@ function resetSearchState() {
results.value = []; results.value = [];
tableErrors.value = []; tableErrors.value = [];
generalError.value = ""; generalError.value = "";
limitedTables.value = false;
currentExecutionId.value = ""; currentExecutionId.value = "";
tableTasks.value = [];
nextTableIndex.value = 0;
activeKeyword.value = "";
activePerTableLimit.value = 20;
} }
function isSearchableTable(table: TableInfo): boolean { function isSearchableTable(table: TableInfo): boolean {
@@ -154,10 +163,6 @@ async function listSearchTables(): Promise<SearchTableTask[]> {
} }
} }
if (tasks.length > MAX_TABLES) {
limitedTables.value = true;
return tasks.slice(0, MAX_TABLES);
}
return tasks; return tasks;
} }
@@ -167,21 +172,18 @@ async function startSearch() {
resetSearchState(); resetSearchState();
running.value = true; running.value = true;
loadingTables.value = true; loadingTables.value = true;
activeKeyword.value = keyword.value.trim();
activePerTableLimit.value = Math.min(100, Math.max(1, Math.trunc(Number(perTableLimit.value) || 20)));
try { try {
await connectionStore.ensureConnected(props.prefillConnectionId); await connectionStore.ensureConnected(props.prefillConnectionId);
const tasks = await listSearchTables(); tableTasks.value = await listSearchTables();
if (currentRun !== runId || cancelled.value) return; if (currentRun !== runId || cancelled.value) return;
progressTotal.value = tasks.length; progressTotal.value = tableTasks.value.length;
loadingTables.value = false; loadingTables.value = false;
await scanTableRange(currentRun, false);
for (const task of tasks) {
if (currentRun !== runId || cancelled.value) break;
await searchTable(task, connection.value.db_type, currentRun);
progressDone.value += 1;
}
} catch (error: any) { } catch (error: any) {
if (!cancelled.value) generalError.value = error?.message || String(error); if (currentRun === runId && !cancelled.value) generalError.value = error?.message || String(error);
} finally { } finally {
if (currentRun === runId) { if (currentRun === runId) {
running.value = false; running.value = false;
@@ -191,6 +193,39 @@ async function startSearch() {
} }
} }
async function continueSearch(scanAllRemaining: boolean) {
if (!canContinueSearch.value || !connection.value) return;
const currentRun = ++runId;
cancelled.value = false;
running.value = true;
generalError.value = "";
try {
await connectionStore.ensureConnected(props.prefillConnectionId);
await scanTableRange(currentRun, scanAllRemaining);
} catch (error: any) {
if (currentRun === runId && !cancelled.value) generalError.value = error?.message || String(error);
} finally {
if (currentRun === runId) {
running.value = false;
currentExecutionId.value = "";
}
}
}
async function scanTableRange(currentRun: number, scanAllRemaining: boolean) {
const range = databaseSearchBatchRange(nextTableIndex.value, tableTasks.value.length, scanAllRemaining);
while (nextTableIndex.value < range.end) {
if (currentRun !== runId || cancelled.value) break;
const task = tableTasks.value[nextTableIndex.value];
if (!task) break;
await searchTable(task, connection.value!.db_type, currentRun);
if (currentRun !== runId || cancelled.value) break;
nextTableIndex.value += 1;
progressDone.value = nextTableIndex.value;
}
}
async function searchTable(task: SearchTableTask, databaseType: DatabaseType, currentRun: number) { async function searchTable(task: SearchTableTask, databaseType: DatabaseType, currentRun: number) {
const tableLabel = task.schema ? `${task.schema}.${task.table.name}` : task.table.name; const tableLabel = task.schema ? `${task.schema}.${task.table.name}` : task.table.name;
try { try {
@@ -202,8 +237,8 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu
schema: task.schema, schema: task.schema,
tableName: task.table.name, tableName: task.table.name,
columns, columns,
term: keyword.value, term: activeKeyword.value,
limit: perTableLimit.value, limit: activePerTableLimit.value,
}); });
if (!query || currentRun !== runId || cancelled.value) return; if (!query || currentRun !== runId || cancelled.value) return;
@@ -213,8 +248,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu
if (currentExecutionId.value === executionId) currentExecutionId.value = ""; if (currentExecutionId.value === executionId) currentExecutionId.value = "";
if (currentRun !== runId || cancelled.value) return; if (currentRun !== runId || cancelled.value) return;
const tableResults: SearchResultItem[] = [];
for (const [rowIndex, row] of result.rows.entries()) { for (const [rowIndex, row] of result.rows.entries()) {
const matchedColumns = findMatchedSearchColumns(result.columns, row, columns, keyword.value); const matchedColumns = findMatchedSearchColumns(result.columns, row, columns, activeKeyword.value);
const whereInput = await buildSearchResultWhere({ const whereInput = await buildSearchResultWhere({
databaseType, databaseType,
columns, columns,
@@ -222,8 +258,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu
row, row,
matchedColumns, matchedColumns,
}); });
results.value.push({ if (currentRun !== runId || cancelled.value) return;
id: `${tableLabel}:${rowIndex}:${results.value.length}`, tableResults.push({
id: `${tableLabel}:${rowIndex}`,
schema: task.schema, schema: task.schema,
tableName: task.table.name, tableName: task.table.name,
tableType: task.table.table_type, tableType: task.table.table_type,
@@ -232,8 +269,9 @@ async function searchTable(task: SearchTableTask, databaseType: DatabaseType, cu
whereInput, whereInput,
}); });
} }
if (currentRun === runId && !cancelled.value) results.value.push(...tableResults);
} catch (error: any) { } catch (error: any) {
if (!cancelled.value) { if (currentRun === runId && !cancelled.value) {
tableErrors.value.push({ tableName: tableLabel, message: error?.message || String(error) }); tableErrors.value.push({ tableName: tableLabel, message: error?.message || String(error) });
} }
} }
@@ -312,7 +350,14 @@ function openResult(item: SearchResultItem) {
<div v-if="running || progressTotal" class="flex flex-wrap items-center gap-3 text-xs text-muted-foreground"> <div v-if="running || progressTotal" class="flex flex-wrap items-center gap-3 text-xs text-muted-foreground">
<span class="tabular-nums">{{ loadingTables ? t("databaseSearch.loadingTables") : progressLabel }}</span> <span class="tabular-nums">{{ loadingTables ? t("databaseSearch.loadingTables") : progressLabel }}</span>
<span class="tabular-nums">{{ t("databaseSearch.resultCount", { count: results.length }) }}</span> <span class="tabular-nums">{{ t("databaseSearch.resultCount", { count: results.length }) }}</span>
<span v-if="limitedTables" class="text-amber-600">{{ t("databaseSearch.limitedTables", { count: 200 }) }}</span> <div v-if="canContinueSearch" class="ml-auto flex flex-wrap items-center gap-2">
<Button variant="outline" size="sm" class="h-7 text-xs" @click="continueSearch(false)">
{{ t("databaseSearch.continueNextBatch", { count: nextBatchSize }) }}
</Button>
<Button variant="secondary" size="sm" class="h-7 text-xs" @click="continueSearch(true)">
{{ t("databaseSearch.scanAllRemaining", { count: remainingTableCount }) }}
</Button>
</div>
</div> </div>
<div v-if="generalError" class="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive"> <div v-if="generalError" class="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive">
+2 -1
View File
@@ -2434,7 +2434,8 @@ export default {
loadingTables: "Reading table list...", loadingTables: "Reading table list...",
progress: "Scanned {done}/{total} tables", progress: "Scanned {done}/{total} tables",
resultCount: "{count} matches", resultCount: "{count} matches",
limitedTables: "Large schema detected; only the first {count} tables are scanned", continueNextBatch: "Scan next {count}",
scanAllRemaining: "Scan remaining {count}",
results: "Results", results: "Results",
openResult: "Open", openResult: "Open",
waiting: "Waiting for matches...", waiting: "Waiting for matches...",
+2 -1
View File
@@ -2369,7 +2369,8 @@ export default withEnglishFallback({
loadingTables: "Leyendo lista de tablas...", loadingTables: "Leyendo lista de tablas...",
progress: "Analizadas {done}/{total} tablas", progress: "Analizadas {done}/{total} tablas",
resultCount: "{count} coincidencias", resultCount: "{count} coincidencias",
limitedTables: "Esquema grande detectado; solo se analizan las primeras {count} tablas", continueNextBatch: "Analizar las siguientes {count}",
scanAllRemaining: "Analizar las {count} restantes",
results: "Resultados", results: "Resultados",
openResult: "Abrir", openResult: "Abrir",
waiting: "Esperando coincidencias...", waiting: "Esperando coincidencias...",
+2 -1
View File
@@ -2367,7 +2367,8 @@ export default withEnglishFallback({
loadingTables: "Lettura dell'elenco delle tabelle...", loadingTables: "Lettura dell'elenco delle tabelle...",
progress: "Scansionate {done}/{total} tabelle", progress: "Scansionate {done}/{total} tabelle",
resultCount: "{count} corrispondenze", resultCount: "{count} corrispondenze",
limitedTables: "Rilevato schema di grandi dimensioni; vengono scansionate solo le prime {count} tabelle", continueNextBatch: "Scansiona le prossime {count}",
scanAllRemaining: "Scansiona le {count} rimanenti",
results: "Risultati", results: "Risultati",
openResult: "Apri", openResult: "Apri",
waiting: "In attesa di corrispondenze...", waiting: "In attesa di corrispondenze...",
+2 -1
View File
@@ -2394,7 +2394,8 @@ export default withEnglishFallback({
loadingTables: "テーブル一覧を読み取り中...", loadingTables: "テーブル一覧を読み取り中...",
progress: "{done}/{total}テーブルをスキャンしました", progress: "{done}/{total}テーブルをスキャンしました",
resultCount: "{count}件の一致", resultCount: "{count}件の一致",
limitedTables: "大規模スキーマが検出されました。最初の{count}テーブルのみスキャンされます", continueNextBatch: "次の{count}テーブルをスキャン",
scanAllRemaining: "残り{count}テーブルをスキャン",
results: "結果", results: "結果",
openResult: "開く", openResult: "開く",
waiting: "一致を待機中...", waiting: "一致を待機中...",
+2 -1
View File
@@ -2268,7 +2268,8 @@ export default withEnglishFallback({
loadingTables: "테이블 목록을 읽는 중...", loadingTables: "테이블 목록을 읽는 중...",
progress: "테이블 {done}/{total}개 스캔", progress: "테이블 {done}/{total}개 스캔",
resultCount: "{count}개 일치", resultCount: "{count}개 일치",
limitedTables: "대규모 스키마가 감지되어 처음 {count}개 테이블만 스캔합니다", continueNextBatch: "다음 {count}개 스캔",
scanAllRemaining: "남은 {count}개 스캔",
results: "결과", results: "결과",
openResult: "열기", openResult: "열기",
waiting: "일치 항목을 기다리는 중...", waiting: "일치 항목을 기다리는 중...",
+2 -1
View File
@@ -2369,7 +2369,8 @@ export default withEnglishFallback({
loadingTables: "Lendo lista de tabelas...", loadingTables: "Lendo lista de tabelas...",
progress: "{done}/{total} tabelas verificadas", progress: "{done}/{total} tabelas verificadas",
resultCount: "{count} correspondências", resultCount: "{count} correspondências",
limitedTables: "Schema grande detectado; apenas as primeiras {count} tabelas são verificadas", continueNextBatch: "Verificar próximas {count}",
scanAllRemaining: "Verificar {count} restantes",
results: "Resultados", results: "Resultados",
openResult: "Abrir", openResult: "Abrir",
waiting: "Aguardando correspondências...", waiting: "Aguardando correspondências...",
+2 -1
View File
@@ -2357,7 +2357,8 @@ export default withEnglishFallback({
loadingTables: "正在读取表列表...", loadingTables: "正在读取表列表...",
progress: "已扫描 {done}/{total} 张表", progress: "已扫描 {done}/{total} 张表",
resultCount: "{count} 条命中", resultCount: "{count} 条命中",
limitedTables: "表数量较多,本次仅扫描前 {count} 张表", continueNextBatch: "继续扫描下 {count} 张",
scanAllRemaining: "扫描剩余 {count} 张",
results: "查找结果", results: "查找结果",
openResult: "打开", openResult: "打开",
waiting: "等待命中结果...", waiting: "等待命中结果...",
+2 -1
View File
@@ -2367,7 +2367,8 @@ export default withEnglishFallback({
loadingTables: "正在讀取表清單……", loadingTables: "正在讀取表清單……",
progress: "已掃描 {done}/{total} 張資料表", progress: "已掃描 {done}/{total} 張資料表",
resultCount: "{count} 筆結果", resultCount: "{count} 筆結果",
limitedTables: "偵測到大型 schema;僅掃描前 {count} 張資料表", continueNextBatch: "繼續掃描下 {count} 張",
scanAllRemaining: "掃描剩餘 {count} 張",
results: "搜尋結果", results: "搜尋結果",
openResult: "開啟", openResult: "開啟",
waiting: "搜尋中……", waiting: "搜尋中……",
@@ -0,0 +1,21 @@
export const DATABASE_SEARCH_TABLE_BATCH_SIZE = 200;
export type DatabaseSearchBatchRange = {
start: number;
end: number;
};
export function databaseSearchBatchRange(nextIndex: number, total: number, scanAllRemaining = false, batchSize = DATABASE_SEARCH_TABLE_BATCH_SIZE): DatabaseSearchBatchRange {
const safeTotal = Math.max(0, Math.trunc(total));
const start = Math.min(Math.max(0, Math.trunc(nextIndex)), safeTotal);
const safeBatchSize = Math.max(1, Math.trunc(batchSize));
return {
start,
end: scanAllRemaining ? safeTotal : Math.min(start + safeBatchSize, safeTotal),
};
}
export function databaseSearchNextBatchSize(nextIndex: number, total: number, batchSize = DATABASE_SEARCH_TABLE_BATCH_SIZE): number {
const range = databaseSearchBatchRange(nextIndex, total, false, batchSize);
return range.end - range.start;
}
+15
View File
@@ -2,6 +2,7 @@ import { test } from "vitest";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import type { ColumnInfo } from "../../apps/desktop/src/types/database.ts"; import type { ColumnInfo } from "../../apps/desktop/src/types/database.ts";
import { findMatchedSearchColumns, isNumericSearchColumn, isTextSearchColumn } from "../../apps/desktop/src/lib/database/databaseSearch.ts"; import { findMatchedSearchColumns, isNumericSearchColumn, isTextSearchColumn } from "../../apps/desktop/src/lib/database/databaseSearch.ts";
import { DATABASE_SEARCH_TABLE_BATCH_SIZE, databaseSearchBatchRange, databaseSearchNextBatchSize } from "../../apps/desktop/src/lib/database/databaseSearchBatch.ts";
function col(name: string, dataType: string, primary = false): ColumnInfo { function col(name: string, dataType: string, primary = false): ColumnInfo {
return { return {
@@ -26,3 +27,17 @@ test("finds matched columns from returned rows", () => {
assert.deepEqual(matches, ["email"]); assert.deepEqual(matches, ["email"]);
}); });
test("treats 200 tables as a resumable batch instead of a search cap", () => {
assert.equal(DATABASE_SEARCH_TABLE_BATCH_SIZE, 200);
assert.deepEqual(databaseSearchBatchRange(0, 450), { start: 0, end: 200 });
assert.deepEqual(databaseSearchBatchRange(200, 450), { start: 200, end: 400 });
assert.deepEqual(databaseSearchBatchRange(400, 450), { start: 400, end: 450 });
assert.equal(databaseSearchNextBatchSize(400, 450), 50);
});
test("can scan every remaining table without exceeding the discovered table list", () => {
assert.deepEqual(databaseSearchBatchRange(200, 450, true), { start: 200, end: 450 });
assert.deepEqual(databaseSearchBatchRange(999, 450, true), { start: 450, end: 450 });
assert.deepEqual(databaseSearchBatchRange(-10, 5, false, 0), { start: 0, end: 1 });
});
+29
View File
@@ -0,0 +1,29 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { test } from "vitest";
import { compileScript, compileTemplate, parse } from "vue/compiler-sfc";
const dialogPath = "apps/desktop/src/components/search/DatabaseSearchDialog.vue";
const dialogSource = readFileSync(dialogPath, "utf8");
test("database search dialog compiles", () => {
const { descriptor, errors } = parse(dialogSource, { filename: dialogPath });
assert.deepEqual(errors, []);
assert.ok(descriptor.scriptSetup);
compileScript(descriptor, { id: dialogPath });
assert.ok(descriptor.template);
const template = compileTemplate({ id: dialogPath, filename: dialogPath, source: descriptor.template!.content });
assert.deepEqual(template.errors, []);
});
test("database search continues after each table batch without retaining a hard cap", () => {
assert.doesNotMatch(dialogSource, /MAX_TABLES|tasks\.slice\(0/);
assert.match(dialogSource, /progressTotal\.value = tableTasks\.value\.length/);
assert.match(dialogSource, /await scanTableRange\(currentRun, false\)/);
assert.match(dialogSource, /@click="continueSearch\(false\)"/);
assert.match(dialogSource, /@click="continueSearch\(true\)"/);
assert.match(dialogSource, /activeKeyword\.value/);
assert.match(dialogSource, /activePerTableLimit\.value/);
assert.match(dialogSource, /const tableResults: SearchResultItem\[\] = \[\]/);
assert.match(dialogSource, /currentRun === runId && !cancelled\.value/);
});