diff --git a/apps/desktop/src/components/diff/DataCompareDialog.vue b/apps/desktop/src/components/diff/DataCompareDialog.vue index 14f36ce06d..597db9ae13 100644 --- a/apps/desktop/src/components/diff/DataCompareDialog.vue +++ b/apps/desktop/src/components/diff/DataCompareDialog.vue @@ -3,16 +3,45 @@ import { computed, ref, watch } from "vue"; import { useI18n } from "vue-i18n"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useConnectionStore } from "@/stores/connectionStore"; import { useToast } from "@/composables/useToast"; import { isSchemaAware } from "@/lib/databaseCapabilities"; import { copyToClipboard } from "@/lib/clipboard"; -import type { DataCompareResult } from "@/lib/dataCompare"; import * as api from "@/lib/api"; import DatabaseIcon from "@/components/icons/DatabaseIcon.vue"; -import { Copy, GitCompareArrows, Loader2, Play, ArrowLeftRight } from "lucide-vue-next"; +import { ArrowLeftRight, CheckSquare, Copy, GitCompareArrows, Loader2, Play, Square } from "lucide-vue-next"; + +interface CompareColumn { + name: string; + is_primary_key?: boolean; +} + +interface DataCompareTableTask { + sourceTable: string; + targetTable: string; +} + +type DataCompareTableStatus = "different" | "same" | "error"; + +interface DataCompareTableResult { + sourceTable: string; + targetTable: string; + keyColumns: string[]; + status: DataCompareTableStatus; + added: number; + removed: number; + modified: number; + sourceRowCount: number; + targetRowCount: number; + sourceTruncated: boolean; + targetTruncated: boolean; + syncStatements: string[]; + syncSql: string; + error?: string; +} const { t } = useI18n(); const { toast } = useToast(); @@ -33,6 +62,8 @@ const sourceTable = ref(""); const sourceDatabases = ref([]); const sourceSchemas = ref([]); const sourceTables = ref([]); +const sourceTableSearch = ref(""); +const selectedSourceTables = ref>(new Set()); const targetConnectionId = ref(""); const targetDatabase = ref(""); @@ -44,12 +75,13 @@ const targetTables = ref([]); const keyColumnsText = ref(""); const rowLimit = ref("1000"); -const sourceRowCount = ref(null); -const targetRowCount = ref(null); -const result = ref(null); const syncSql = ref(""); const syncStatements = ref([]); +const batchResults = ref([]); const comparing = ref(false); +const compareProgressCurrent = ref(0); +const compareProgressTotal = ref(0); +const compareProgressTable = ref(""); const executing = ref(false); const executedCount = ref(0); const executeTotal = ref(0); @@ -59,16 +91,45 @@ const rowLimitOptions = [1000, 5000, 10000, 50000]; const sqlConnections = computed(() => store.connections.filter((connection) => !["redis", "mongodb", "elasticsearch"].includes(connection.db_type)), ); +const selectedSourceTableNames = computed(() => + sourceTables.value.filter((table) => selectedSourceTables.value.has(table)), +); +const isBatchCompare = computed(() => selectedSourceTableNames.value.length > 1); +const filteredSourceTables = computed(() => { + const query = sourceTableSearch.value.trim().toLowerCase(); + if (!query) return sourceTables.value; + return sourceTables.value.filter((table) => table.toLowerCase().includes(query)); +}); +const allFilteredTablesSelected = computed( + () => + filteredSourceTables.value.length > 0 && + filteredSourceTables.value.every((table) => selectedSourceTables.value.has(table)), +); +const compareTasksPreview = computed(() => + selectedSourceTableNames.value.map((table) => { + const target = isBatchCompare.value ? table : targetTable.value || table; + const matched = !!target && targetTables.value.includes(target); + return { + sourceTable: table, + targetTable: target, + matched, + }; + }), +); +const matchedTaskCount = computed(() => compareTasksPreview.value.filter((task) => task.matched).length); +const missingTargetTables = computed(() => + compareTasksPreview.value.filter((task) => !task.matched).map((task) => task.targetTable || task.sourceTable), +); const canCompare = computed( () => sourceConnectionId.value && sourceDatabase.value && sourceSchema.value && - sourceTable.value && + selectedSourceTableNames.value.length > 0 && targetConnectionId.value && targetDatabase.value && targetSchema.value && - targetTable.value, + (!isBatchCompare.value ? !!targetTable.value : true), ); const keyColumns = computed(() => keyColumnsText.value @@ -76,46 +137,123 @@ const keyColumns = computed(() => .map((value) => value.trim()) .filter(Boolean), ); +const rowLimitNumber = computed(() => Number(rowLimit.value) || 1000); +const comparedResults = computed(() => batchResults.value.filter((item) => item.status !== "error")); +const sameTableCount = computed(() => batchResults.value.filter((item) => item.status === "same").length); +const differentTableCount = computed(() => batchResults.value.filter((item) => item.status === "different").length); +const failedTableCount = computed(() => batchResults.value.filter((item) => item.status === "error").length); +const totalAdded = computed(() => batchResults.value.reduce((sum, item) => sum + item.added, 0)); +const totalRemoved = computed(() => batchResults.value.reduce((sum, item) => sum + item.removed, 0)); +const totalModified = computed(() => batchResults.value.reduce((sum, item) => sum + item.modified, 0)); +const hasAnyTruncated = computed(() => batchResults.value.some((item) => item.sourceTruncated || item.targetTruncated)); +const hasResults = computed(() => batchResults.value.length > 0); const summary = computed(() => { - const diff = result.value; - if (!diff) return ""; - return t("dataCompare.summary", { - added: diff.added.length, - removed: diff.removed.length, - modified: diff.modified.length, + if (!hasResults.value) return ""; + if (batchResults.value.length === 1 && batchResults.value[0]?.status !== "error") { + const item = batchResults.value[0]; + return t("dataCompare.summary", { + added: item.added, + removed: item.removed, + modified: item.modified, + }); + } + return t("dataCompare.batchSummary", { + tables: batchResults.value.length, + different: differentTableCount.value, + same: sameTableCount.value, + failed: failedTableCount.value, + added: totalAdded.value, + removed: totalRemoved.value, + modified: totalModified.value, + }); +}); +const compareProgressLabel = computed(() => { + if (!comparing.value || compareProgressTotal.value === 0) return ""; + return t("dataCompare.comparingTable", { + current: compareProgressCurrent.value, + total: compareProgressTotal.value, + table: compareProgressTable.value, }); }); -const rowLimitNumber = computed(() => Number(rowLimit.value) || 1000); -const isSourceTruncated = computed(() => sourceRowCount.value !== null && sourceRowCount.value > rowLimitNumber.value); -const isTargetTruncated = computed(() => targetRowCount.value !== null && targetRowCount.value > rowLimitNumber.value); function connectionIconType(connectionId: string) { const config = store.getConfig(connectionId); return config?.driver_profile || config?.db_type || "mysql"; } +function resetSelectedSourceTables(nextTables: Iterable) { + selectedSourceTables.value = new Set(nextTables); +} + +function toggleSourceTable(table: string) { + const next = new Set(selectedSourceTables.value); + if (next.has(table)) next.delete(table); + else next.add(table); + resetSelectedSourceTables(next); +} + +function toggleSelectAllSourceTables() { + const next = new Set(selectedSourceTables.value); + if (allFilteredTablesSelected.value) { + filteredSourceTables.value.forEach((table) => next.delete(table)); + } else { + filteredSourceTables.value.forEach((table) => next.add(table)); + } + resetSelectedSourceTables(next); +} + +function buildCompareTasks(): DataCompareTableTask[] { + if (!selectedSourceTableNames.value.length) return []; + if (!isBatchCompare.value) { + const table = selectedSourceTableNames.value[0]; + return targetTable.value ? [{ sourceTable: table, targetTable: targetTable.value }] : []; + } + return selectedSourceTableNames.value.map((table) => ({ + sourceTable: table, + targetTable: table, + })); +} + +function clearResult() { + batchResults.value = []; + syncSql.value = ""; + syncStatements.value = []; + syncErrors.value = []; + compareProgressCurrent.value = 0; + compareProgressTotal.value = 0; + compareProgressTable.value = ""; +} + function swapSourceTarget() { + const previousSelectedTables = [...selectedSourceTableNames.value]; + const nextSingleTarget = previousSelectedTables.length === 1 ? (previousSelectedTables[0] ?? "") : ""; + const nextSourceSelection = + previousSelectedTables.length <= 1 ? [targetTable.value].filter(Boolean) : previousSelectedTables; + const tmpConnId = sourceConnectionId.value; const tmpDb = sourceDatabase.value; const tmpDbs = sourceDatabases.value; const tmpSchema = sourceSchema.value; const tmpSchemas = sourceSchemas.value; - const tmpTable = sourceTable.value; const tmpTables = sourceTables.value; + sourceConnectionId.value = targetConnectionId.value; sourceDatabase.value = targetDatabase.value; sourceDatabases.value = targetDatabases.value; sourceSchema.value = targetSchema.value; sourceSchemas.value = targetSchemas.value; - sourceTable.value = targetTable.value; sourceTables.value = targetTables.value; + resetSelectedSourceTables(nextSourceSelection.filter((table) => targetTables.value.includes(table))); + targetConnectionId.value = tmpConnId; targetDatabase.value = tmpDb; targetDatabases.value = tmpDbs; targetSchema.value = tmpSchema; targetSchemas.value = tmpSchemas; - targetTable.value = tmpTable; targetTables.value = tmpTables; + targetTable.value = nextSingleTarget; + + sourceTable.value = selectedSourceTableNames.value.length === 1 ? selectedSourceTableNames.value[0] : ""; clearResult(); } @@ -173,6 +311,7 @@ async function loadDatabases(connectionId: string, side: "source" | "target") { sourceSchema.value = ""; sourceTables.value = []; sourceTable.value = ""; + resetSelectedSourceTables([]); } else { targetDatabases.value = names; targetDatabase.value = names.length === 1 ? names[0] : ""; @@ -194,97 +333,225 @@ async function loadTables(side: "source" | "target") { const tables = (await api.listTables(connectionId, database, schema)) .filter((table) => table.table_type !== "VIEW") .map((table) => table.name); + if (side === "source") { + const preferredSelection = + props.prefillTable && tables.includes(props.prefillTable) + ? [props.prefillTable] + : [...selectedSourceTables.value].filter((table) => tables.includes(table)); sourceSchema.value = schema; sourceTables.value = tables; - const preferred = - props.prefillTable && tables.includes(props.prefillTable) ? props.prefillTable : sourceTable.value; - sourceTable.value = tables.includes(preferred) ? preferred : ""; + resetSelectedSourceTables(preferredSelection); + sourceTable.value = preferredSelection.length === 1 ? preferredSelection[0] : ""; } else { targetSchema.value = schema; targetTables.value = tables; - const preferred = sourceTable.value && tables.includes(sourceTable.value) ? sourceTable.value : targetTable.value; - targetTable.value = tables.includes(preferred) ? preferred : ""; + const singleSourceTable = selectedSourceTableNames.value.length === 1 ? selectedSourceTableNames.value[0] : ""; + const preferred = + targetTable.value && tables.includes(targetTable.value) + ? targetTable.value + : singleSourceTable && tables.includes(singleSourceTable) + ? singleSourceTable + : ""; + targetTable.value = preferred; } } -function clearResult() { - result.value = null; - syncSql.value = ""; - syncStatements.value = []; - sourceRowCount.value = null; - targetRowCount.value = null; +async function loadColumnsWithCache( + cache: Map, + connectionId: string, + database: string, + schema: string, + table: string, +): Promise { + const key = `${connectionId}:${database}:${schema}:${table}`; + const cached = cache.get(key); + if (cached) return cached; + const columns = (await api.getColumns(connectionId, database, schema, table)) as CompareColumn[]; + cache.set(key, columns); + return columns; +} + +async function inferKeyColumnsForTable( + table: string, + sourceColumnCache?: Map, +): Promise { + if (!sourceConnectionId.value || !sourceDatabase.value || !sourceSchema.value || !table) return []; + const columns = sourceColumnCache + ? await loadColumnsWithCache( + sourceColumnCache, + sourceConnectionId.value, + sourceDatabase.value, + sourceSchema.value, + table, + ) + : (((await api.getColumns( + sourceConnectionId.value, + sourceDatabase.value, + sourceSchema.value, + table, + )) as CompareColumn[]) ?? []); + const primaryKeys = columns.filter((column) => column.is_primary_key).map((column) => column.name); + if (primaryKeys.length > 0) return primaryKeys; + return columns.slice(0, 1).map((column) => column.name); } async function inferKeyColumns() { - if (!sourceConnectionId.value || !sourceDatabase.value || !sourceTable.value) return; - const columns = await api.getColumns( - sourceConnectionId.value, - sourceDatabase.value, - sourceSchema.value, - sourceTable.value, - ); - const primaryKeys = columns.filter((column) => column.is_primary_key).map((column) => column.name); - keyColumnsText.value = (primaryKeys.length ? primaryKeys : columns.slice(0, 1).map((column) => column.name)).join( - ", ", - ); + const table = selectedSourceTableNames.value.length === 1 ? selectedSourceTableNames.value[0] : ""; + if (!table) return; + const inferred = await inferKeyColumnsForTable(table); + keyColumnsText.value = inferred.join(", "); +} + +function resultStatusLabel(status: DataCompareTableStatus): string { + if (status === "different") return t("dataCompare.statusDifferent"); + if (status === "same") return t("dataCompare.statusSame"); + return t("dataCompare.statusError"); +} + +function resultStatusClass(status: DataCompareTableStatus): string { + if (status === "different") return "bg-amber-500/15 text-amber-700"; + if (status === "same") return "bg-emerald-500/15 text-emerald-700"; + return "bg-destructive/15 text-destructive"; } async function startCompare() { if (!canCompare.value || comparing.value) return; + const tasks = buildCompareTasks(); + if (tasks.length === 0) { + toast(t("dataCompare.noComparableTables"), 5000); + return; + } + comparing.value = true; clearResult(); + compareProgressTotal.value = tasks.length; + + const sourceColumnCache = new Map(); + const targetColumnCache = new Map(); + const results: DataCompareTableResult[] = []; + const mergedStatements: string[] = []; + const mergedSqlParts: string[] = []; + try { await Promise.all([ store.ensureConnected(sourceConnectionId.value), store.ensureConnected(targetConnectionId.value), ]); - if (keyColumns.value.length === 0) await inferKeyColumns(); - if (keyColumns.value.length === 0) throw new Error(t("dataCompare.noKeyColumns")); - const sourceColumns = await api.getColumns( - sourceConnectionId.value, - sourceDatabase.value, - sourceSchema.value, - sourceTable.value, - ); - const targetColumns = await api.getColumns( - targetConnectionId.value, - targetDatabase.value, - targetSchema.value, - targetTable.value, - ); - const columns = sourceColumns - .map((column) => column.name) - .filter((column) => targetColumns.some((target) => target.name === column)); - const missingKeys = keyColumns.value.filter((column) => !columns.includes(column)); - if (missingKeys.length > 0) { - throw new Error(t("dataCompare.missingKeyColumns", { columns: missingKeys.join(", ") })); + for (const [index, task] of tasks.entries()) { + compareProgressCurrent.value = index + 1; + compareProgressTable.value = task.sourceTable; + + try { + if (!targetTables.value.includes(task.targetTable)) { + throw new Error(t("dataCompare.targetTableMissing", { table: task.targetTable })); + } + + const resolvedKeys = + keyColumns.value.length > 0 + ? keyColumns.value + : await inferKeyColumnsForTable(task.sourceTable, sourceColumnCache); + if (resolvedKeys.length === 0) { + throw new Error(t("dataCompare.noKeyColumns")); + } + + const sourceColumns = await loadColumnsWithCache( + sourceColumnCache, + sourceConnectionId.value, + sourceDatabase.value, + sourceSchema.value, + task.sourceTable, + ); + const targetColumns = await loadColumnsWithCache( + targetColumnCache, + targetConnectionId.value, + targetDatabase.value, + targetSchema.value, + task.targetTable, + ); + const columns = sourceColumns + .map((column) => column.name) + .filter((column) => targetColumns.some((target) => target.name === column)); + const missingKeys = resolvedKeys.filter((column) => !columns.includes(column)); + if (missingKeys.length > 0) { + throw new Error(t("dataCompare.missingKeyColumns", { columns: missingKeys.join(", ") })); + } + if (columns.length === 0) { + throw new Error(t("dataCompare.noCommonColumns")); + } + + const preparation = await api.prepareDataCompareFromTables({ + sourceConnectionId: sourceConnectionId.value, + sourceDatabase: sourceDatabase.value, + sourceSchema: sourceSchema.value, + sourceTable: task.sourceTable, + targetConnectionId: targetConnectionId.value, + targetDatabase: targetDatabase.value, + targetSchema: targetSchema.value, + targetTable: task.targetTable, + columns, + keyColumns: resolvedKeys, + rowLimit: rowLimitNumber.value, + }); + + const added = preparation.result.added.length; + const removed = preparation.result.removed.length; + const modified = preparation.result.modified.length; + const status: DataCompareTableStatus = added || removed || modified ? "different" : "same"; + + results.push({ + sourceTable: task.sourceTable, + targetTable: task.targetTable, + keyColumns: resolvedKeys, + status, + added, + removed, + modified, + sourceRowCount: preparation.sourceRowCount, + targetRowCount: preparation.targetRowCount, + sourceTruncated: preparation.sourceTruncated, + targetTruncated: preparation.targetTruncated, + syncStatements: preparation.syncStatements, + syncSql: preparation.syncSql, + }); + + if (preparation.syncStatements.length > 0) { + mergedStatements.push(...preparation.syncStatements); + } + if (preparation.syncSql.trim()) { + mergedSqlParts.push(preparation.syncSql.trim()); + } + } catch (e: any) { + results.push({ + sourceTable: task.sourceTable, + targetTable: task.targetTable, + keyColumns: keyColumns.value, + status: "error", + added: 0, + removed: 0, + modified: 0, + sourceRowCount: 0, + targetRowCount: 0, + sourceTruncated: false, + targetTruncated: false, + syncStatements: [], + syncSql: "", + error: e?.message || String(e), + }); + } } - if (columns.length === 0) throw new Error(t("dataCompare.noCommonColumns")); - const preparation = await api.prepareDataCompareFromTables({ - sourceConnectionId: sourceConnectionId.value, - sourceDatabase: sourceDatabase.value, - sourceSchema: sourceSchema.value, - sourceTable: sourceTable.value, - targetConnectionId: targetConnectionId.value, - targetDatabase: targetDatabase.value, - targetSchema: targetSchema.value, - targetTable: targetTable.value, - columns, - keyColumns: keyColumns.value, - rowLimit: rowLimitNumber.value, - }); - sourceRowCount.value = preparation.sourceRowCount; - targetRowCount.value = preparation.targetRowCount; - result.value = preparation.result; - syncStatements.value = preparation.syncStatements; - syncSql.value = preparation.syncSql; + batchResults.value = results; + syncStatements.value = mergedStatements; + syncSql.value = mergedSqlParts.join("\n\n"); } catch (e: any) { toast(e?.message || String(e), 5000); } finally { comparing.value = false; + compareProgressCurrent.value = 0; + compareProgressTotal.value = 0; + compareProgressTable.value = ""; } } @@ -332,6 +599,9 @@ watch(sourceConnectionId, (id) => { sourceSchema.value = ""; sourceSchemas.value = []; sourceTables.value = []; + sourceTable.value = ""; + sourceTableSearch.value = ""; + resetSelectedSourceTables([]); loadDatabases(id, "source").catch((e) => toast(String(e), 5000)); }); watch(targetConnectionId, (id) => { @@ -340,6 +610,7 @@ watch(targetConnectionId, (id) => { targetSchema.value = ""; targetSchemas.value = []; targetTables.value = []; + targetTable.value = ""; loadDatabases(id, "target").catch((e) => toast(String(e), 5000)); }); watch(sourceDatabase, () => { @@ -348,6 +619,8 @@ watch(sourceDatabase, () => { sourceSchemas.value = []; sourceTables.value = []; sourceTable.value = ""; + sourceTableSearch.value = ""; + resetSelectedSourceTables([]); loadSchemas("source", props.prefillSchema).catch((e) => toast(String(e), 5000)); }); watch(targetDatabase, () => { @@ -362,6 +635,8 @@ watch(sourceSchema, () => { clearResult(); sourceTables.value = []; sourceTable.value = ""; + sourceTableSearch.value = ""; + resetSelectedSourceTables([]); if (sourceSchema.value) loadTables("source").catch((e) => toast(String(e), 5000)); }); watch(targetSchema, () => { @@ -370,21 +645,30 @@ watch(targetSchema, () => { targetTable.value = ""; if (targetSchema.value) loadTables("target").catch((e) => toast(String(e), 5000)); }); -watch(sourceTable, (table, previous) => { +watch(selectedSourceTableNames, (tables, previous) => { clearResult(); - if (table !== previous) keyColumnsText.value = ""; - if (table && targetTables.value.includes(table)) targetTable.value = table; - if (sourceTable.value) { - inferKeyColumns().catch(() => {}); + sourceTable.value = tables.length === 1 ? tables[0] : ""; + if (tables.length !== 1) { + keyColumnsText.value = ""; + return; } + const table = tables[0]; + if (targetTables.value.includes(table)) { + targetTable.value = table; + } else if (previous?.length === 1 && targetTable.value === previous[0]) { + targetTable.value = ""; + } + if (table !== previous?.[0]) { + keyColumnsText.value = ""; + } + inferKeyColumns().catch(() => {}); }); watch(targetTable, () => clearResult()); watch( open, async (value) => { if (!value) return; - result.value = null; - syncSql.value = ""; + clearResult(); if (props.prefillConnectionId) { sourceConnectionId.value = props.prefillConnectionId; await loadDatabases(props.prefillConnectionId, "source"); @@ -392,7 +676,10 @@ watch( if (props.prefillDatabase) await loadSchemas("source", props.prefillSchema); if (props.prefillTable) { await loadTables("source"); - if (sourceTables.value.includes(props.prefillTable)) sourceTable.value = props.prefillTable; + if (sourceTables.value.includes(props.prefillTable)) { + resetSelectedSourceTables([props.prefillTable]); + sourceTable.value = props.prefillTable; + } } } }, @@ -402,7 +689,7 @@ watch(