Files
dbx/apps/desktop/src/composables/useDataCompareSession.ts
T

485 lines
20 KiB
TypeScript

import { reactive, shallowReactive } from "vue";
import { uuid } from "@/lib/common/utils";
import * as api from "@/lib/backend/api";
import { openDataCompareSession } from "@/composables/useDialogSources";
import { useExportTracker } from "@/composables/useExportTracker";
import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase, normalizeKeyColumns, type DataCompareCellValue, type DataCompareFromTablesPreparation, type DataCompareResult, type DataCompareSyncPlan, type DataCompareSyncPlanTableOptions } from "@/lib/dataGrid/dataCompare";
import type { ColumnInfo, DatabaseType } from "@/types/database";
export type CompareColumn = ColumnInfo;
/**
* One source-table -> target-table compare task.
*
* Match columns are not part of the task itself: they are stored per source
* table in `DataCompareSessionConfig.keyColumnsByTable` so that every task of a
* batch compare can carry its own selection instead of sharing one global list.
*/
export interface DataCompareTableTask {
sourceTable: string;
targetTable: string;
}
export type DataCompareTableStatus = "different" | "same" | "error";
export type DiffKind = "added" | "removed" | "modified";
export interface SelectableDataCompareRow {
key: string;
keyValues: Record<string, DataCompareCellValue>;
values: Record<string, DataCompareCellValue>;
selected: boolean;
}
export interface SelectableDataCompareModifiedRow {
key: string;
keyValues: Record<string, DataCompareCellValue>;
sourceValues: Record<string, DataCompareCellValue>;
targetValues: Record<string, DataCompareCellValue>;
changes: { column: string; source: DataCompareCellValue; target: DataCompareCellValue }[];
selected: boolean;
}
export interface SelectableDataCompareResult {
added: SelectableDataCompareRow[];
removed: SelectableDataCompareRow[];
modified: SelectableDataCompareModifiedRow[];
}
export interface DataCompareTableResult {
sourceTable: string;
targetTable: string;
keyColumns: string[];
columns: string[];
columnInfo: CompareColumn[];
status: DataCompareTableStatus;
added: number;
removed: number;
modified: number;
sourceRowCount: number;
targetRowCount: number;
sourceTruncated: boolean;
targetTruncated: boolean;
databaseType?: DatabaseType;
preSyncStatements?: string[];
diff: SelectableDataCompareResult;
expanded: boolean;
showAll: Record<DiffKind, boolean>;
error?: string;
}
export interface DataCompareSessionConfig {
sourceConnectionId: string;
sourceDatabase: string;
sourceSchema: string;
sourceDatabases: string[];
sourceSchemas: string[];
sourceTables: string[];
selectedSourceTables: string[];
targetConnectionId: string;
targetDatabase: string;
targetSchema: string;
targetDatabases: string[];
targetSchemas: string[];
targetTables: string[];
targetTable: string;
/**
* Explicit match columns per source table (keyed by the source table name).
*
* An absent entry means "no override": the primary key of that source table is
* inferred automatically. A present entry is an explicit user choice, so an
* empty array means the user cleared the selection and must pick columns
* again — it must not silently fall back to the primary key.
*/
keyColumnsByTable: Record<string, string[]>;
label: string;
}
export interface DataCompareSessionProgress {
current: number;
total: number;
table: string;
}
export type DataCompareSessionStatus = "running" | "completed" | "failed";
export type DataCompareSessionErrorKind = "noKeyColumns" | "missingKeyColumns" | "noCommonColumns";
export interface DataCompareSession {
id: string;
version: number;
status: DataCompareSessionStatus;
config: DataCompareSessionConfig;
progress: DataCompareSessionProgress | null;
batchResults: DataCompareTableResult[];
syncPlan: DataCompareSyncPlan;
error: string | null;
startedAt: number;
finishedAt?: number;
}
export interface DataCompareSessionDependencies {
ensureConnected(connectionId: string): Promise<unknown>;
getConfig(connectionId: string): { db_type?: DatabaseType } | undefined;
formatError?: (kind: DataCompareSessionErrorKind, columns?: string) => string;
}
const sessions = reactive(new Map<string, DataCompareSession>());
export function emptyDataCompareSyncPlan(): DataCompareSyncPlan {
return {
insertCount: 0,
updateCount: 0,
deleteCount: 0,
statementCount: 0,
syncStatements: [],
syncSql: "",
};
}
function cloneConfig(config: DataCompareSessionConfig): DataCompareSessionConfig {
return {
...config,
sourceDatabases: [...config.sourceDatabases],
sourceSchemas: [...config.sourceSchemas],
sourceTables: [...config.sourceTables],
selectedSourceTables: [...config.selectedSourceTables],
targetDatabases: [...config.targetDatabases],
targetSchemas: [...config.targetSchemas],
targetTables: [...config.targetTables],
keyColumnsByTable: normalizeKeyColumnOverrides(config.keyColumnsByTable),
};
}
/**
* Normalizes a per-table match-column map so a cloned session never shares
* arrays with the caller: every entry is trimmed and de-duplicated while an
* explicit empty selection stays an empty array (it must not become "auto").
*/
export function normalizeKeyColumnOverrides(overrides: Record<string, string[] | undefined> | null | undefined): Record<string, string[]> {
const normalized: Record<string, string[]> = {};
for (const [table, columns] of Object.entries(overrides ?? {})) {
if (!table) continue;
normalized[table] = normalizeKeyColumns(columns ?? []);
}
return normalized;
}
function sessionError(error: unknown): string {
return error instanceof Error ? error.message : (error as { message?: string } | null)?.message || String(error);
}
function fallbackCompareErrorMessage(kind: DataCompareSessionErrorKind, columns?: string): string {
if (kind === "missingKeyColumns") return `Missing key columns: ${columns ?? ""}`;
if (kind === "noCommonColumns") return "No common columns";
return "No key columns available";
}
function compareErrorMessage(dependencies: DataCompareSessionDependencies, kind: DataCompareSessionErrorKind, columns?: string): string {
return dependencies.formatError?.(kind, columns) ?? fallbackCompareErrorMessage(kind, columns);
}
function toSelectableDiff(diff: DataCompareResult): SelectableDataCompareResult {
return {
added: diff.added.map((row) => ({ ...row, selected: true })),
removed: diff.removed.map((row) => ({ ...row, selected: true })),
modified: diff.modified.map((row) => ({ ...row, selected: true })),
};
}
export function buildDataCompareSyncPlanTables(results: DataCompareTableResult[], targetSchema: string): DataCompareSyncPlanTableOptions[] {
return results
.filter((table) => table.status === "different")
.map((table) => ({
tableName: table.targetTable,
schema: targetSchema,
columns: table.columns,
keyColumns: table.keyColumns,
columnInfo: table.columnInfo,
diff: {
added: table.diff.added.filter((row) => row.selected).map(({ selected: _selected, ...row }) => row),
removed: table.diff.removed.filter((row) => row.selected).map(({ selected: _selected, ...row }) => row),
modified: table.diff.modified.filter((row) => row.selected).map(({ selected: _selected, ...row }) => row),
},
databaseType: table.databaseType,
preSyncStatements: table.preSyncStatements ?? [],
}))
.filter((table) => table.preSyncStatements.length > 0 || table.diff.added.length > 0 || table.diff.removed.length > 0 || table.diff.modified.length > 0);
}
function publishProgress(session: DataCompareSession, progress: DataCompareSessionProgress): void {
session.version += 1;
session.progress = progress;
const results = session.batchResults;
const failedTableCount = results.filter((item) => item.status === "error").length;
const tracker = useExportTracker();
tracker.updateCompareTask(session.id, {
status: "Running",
compareCurrent: progress.current,
compareTotal: progress.total,
compareCurrentObject: progress.table,
compareResultCount: results.length,
compareSameCount: results.filter((item) => item.status === "same").length,
compareDifferentCount: results.filter((item) => item.status === "different").length,
compareFailedCount: failedTableCount,
compareAddedCount: results.reduce((sum, item) => sum + item.added, 0),
compareRemovedCount: results.reduce((sum, item) => sum + item.removed, 0),
compareModifiedCount: results.reduce((sum, item) => sum + item.modified, 0),
});
}
function updateResults(session: DataCompareSession, results: DataCompareTableResult[]): void {
session.batchResults = [...results];
}
async function loadColumnsWithCache(cache: Map<string, CompareColumn[]>, connectionId: string, database: string, schema: string, table: string): Promise<CompareColumn[]> {
const key = `${connectionId}:${database}:${schema}:${table}`;
const cached = cache.get(key);
if (cached) return cached;
const columns = (await api.getColumns(connectionId, database, schema, table)) as CompareColumn[];
cache.set(key, columns);
return columns;
}
async function inferKeyColumnsForTable(input: DataCompareSessionConfig, table: string, sourceColumnCache: Map<string, CompareColumn[]>): Promise<string[]> {
if (!input.sourceConnectionId || !input.sourceDatabase || !input.sourceSchema || !table) return [];
const columns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, table);
return inferCompareKeyColumns(columns);
}
function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFromTablesPreparation, keyColumns: string[], columns: string[], columnInfo: CompareColumn[], databaseType: DatabaseType | undefined): DataCompareTableResult {
const added = preparation.result.added.length;
const removed = preparation.result.removed.length;
const modified = preparation.result.modified.length;
const status: DataCompareTableStatus = added || removed || modified ? "different" : "same";
return {
sourceTable: task.sourceTable,
targetTable: task.targetTable,
keyColumns,
columns,
columnInfo,
status,
added,
removed,
modified,
sourceRowCount: preparation.sourceRowCount,
targetRowCount: preparation.targetRowCount,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: preparation.targetTruncated,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
expanded: status === "different",
showAll: { added: false, removed: false, modified: false },
};
}
function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataCompareFromTablesPreparation, sourceColumns: CompareColumn[], keyColumns: string[], databaseType: DatabaseType | undefined): DataCompareTableResult {
return {
sourceTable: task.sourceTable,
targetTable: task.targetTable,
keyColumns,
columns: sourceColumns.map((column) => column.name),
columnInfo: sourceColumns,
status: "different",
added: preparation.result.added.length,
removed: 0,
modified: 0,
sourceRowCount: preparation.sourceRowCount,
targetRowCount: 0,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: false,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
expanded: preparation.result.added.length > 0,
showAll: { added: false, removed: false, modified: false },
};
}
function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], databaseType: DatabaseType | undefined, error: unknown): DataCompareTableResult {
return {
sourceTable: task.sourceTable,
targetTable: task.targetTable,
keyColumns,
columns: [],
columnInfo: [],
status: "error",
added: 0,
removed: 0,
modified: 0,
sourceRowCount: 0,
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
databaseType,
preSyncStatements: [],
diff: { added: [], removed: [], modified: [] },
expanded: false,
showAll: { added: false, removed: false, modified: false },
error: sessionError(error),
};
}
/**
* Resolves the configured match columns of one compare task.
*
* Returns the explicit per-source-table override when the user configured one
* (an empty array counts as "explicitly nothing selected") and `undefined`
* otherwise, which means the caller must infer the primary key for that table.
*/
function configuredTaskKeyColumns(input: DataCompareSessionConfig, task: DataCompareTableTask): string[] | undefined {
const override = input.keyColumnsByTable[task.sourceTable];
return override === undefined ? undefined : normalizeKeyColumns(override);
}
async function runDataCompareSession(session: DataCompareSession, tasks: DataCompareTableTask[], dependencies: DataCompareSessionDependencies): Promise<void> {
const input = session.config;
const sourceColumnCache = new Map<string, CompareColumn[]>();
const targetColumnCache = new Map<string, CompareColumn[]>();
const results: DataCompareTableResult[] = [];
const databaseType = dependencies.getConfig(input.targetConnectionId)?.db_type;
try {
publishProgress(session, { current: 0, total: tasks.length, table: "" });
await Promise.all([dependencies.ensureConnected(input.sourceConnectionId), dependencies.ensureConnected(input.targetConnectionId)]);
for (const [index, task] of tasks.entries()) {
publishProgress(session, { current: index + 1, total: tasks.length, table: task.sourceTable });
const configuredKeys = configuredTaskKeyColumns(input, task);
let keyColumns = configuredKeys ?? [];
try {
if (input.targetTables.includes(task.targetTable)) {
const resolvedKeys = configuredKeys ?? (await inferKeyColumnsForTable(input, task.sourceTable, sourceColumnCache));
if (resolvedKeys.length === 0) throw new Error(compareErrorMessage(dependencies, "noKeyColumns"));
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, task.sourceTable);
const targetColumns = await loadColumnsWithCache(targetColumnCache, input.targetConnectionId, input.targetDatabase, input.targetSchema, task.targetTable);
const matched = intersectCompareColumns(sourceColumns, targetColumns);
const columns = matched.columns;
const columnInfo = columns.map((column) => targetColumns.find((target) => target.name === column)).filter((column): column is CompareColumn => !!column);
const canonicalKeyColumns: string[] = [];
const missingKeys: string[] = [];
for (const key of resolvedKeys) {
const canonical = matchColumnNameIgnoreCase(key, columns);
if (!canonical) missingKeys.push(key);
else if (!canonicalKeyColumns.includes(canonical)) canonicalKeyColumns.push(canonical);
}
if (missingKeys.length > 0) throw new Error(compareErrorMessage(dependencies, "missingKeyColumns", missingKeys.join(", ")));
if (columns.length === 0) throw new Error(compareErrorMessage(dependencies, "noCommonColumns"));
const preparation = await api.prepareDataCompareFromTables({
sourceConnectionId: input.sourceConnectionId,
sourceDatabase: input.sourceDatabase,
sourceSchema: input.sourceSchema,
sourceTable: task.sourceTable,
targetConnectionId: input.targetConnectionId,
targetDatabase: input.targetDatabase,
targetSchema: input.targetSchema,
targetTable: task.targetTable,
columns,
keyColumns: canonicalKeyColumns,
sourceColumns: matched.sourceColumns,
});
keyColumns = canonicalKeyColumns;
results.push(buildTableResult(task, preparation, canonicalKeyColumns, columns, columnInfo, databaseType));
} else {
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, task.sourceTable);
const resolvedKeys = configuredTaskKeyColumns(input, task) ?? [];
const sourceColumnNames = sourceColumns.map((column) => column.name);
keyColumns = resolvedKeys.map((key) => matchColumnNameIgnoreCase(key, sourceColumnNames) ?? key);
const preparation = await api.prepareDataCompareMissingTarget({
sourceConnectionId: input.sourceConnectionId,
sourceDatabase: input.sourceDatabase,
sourceSchema: input.sourceSchema,
sourceTable: task.sourceTable,
targetConnectionId: input.targetConnectionId,
targetDatabase: input.targetDatabase,
targetSchema: input.targetSchema,
targetTable: task.targetTable,
keyColumns,
});
results.push(buildMissingTargetResult(task, preparation, sourceColumns, keyColumns, databaseType));
}
} catch (error: unknown) {
results.push(buildErrorResult(task, keyColumns, databaseType, error));
}
updateResults(session, results);
publishProgress(session, { current: index + 1, total: tasks.length, table: task.sourceTable });
}
const plan = await api.buildDataCompareSyncPlan({ tables: buildDataCompareSyncPlanTables(results, input.targetSchema) });
session.syncPlan = plan;
session.progress = null;
session.status = "completed";
session.version += 1;
session.finishedAt = Date.now();
const sameCount = results.filter((item) => item.status === "same").length;
const differentCount = results.filter((item) => item.status === "different").length;
const failedCount = results.filter((item) => item.status === "error").length;
useExportTracker().updateCompareTask(session.id, {
status: failedCount > 0 ? "Error" : "Done",
compareCurrent: tasks.length,
compareTotal: tasks.length,
compareCurrentObject: "",
compareResultCount: results.length,
compareSameCount: sameCount,
compareDifferentCount: differentCount,
compareFailedCount: failedCount,
compareAddedCount: results.reduce((sum, item) => sum + item.added, 0),
compareRemovedCount: results.reduce((sum, item) => sum + item.removed, 0),
compareModifiedCount: results.reduce((sum, item) => sum + item.modified, 0),
errorMessage: failedCount > 0 ? `${failedCount} table(s) failed` : null,
});
} catch (error: unknown) {
const message = sessionError(error);
const lastProgress = session.progress;
session.error = message;
session.progress = null;
session.status = "failed";
session.version += 1;
session.finishedAt = Date.now();
useExportTracker().updateCompareTask(session.id, {
status: "Error",
errorMessage: message,
compareCurrent: lastProgress?.current,
compareTotal: lastProgress?.total,
compareCurrentObject: lastProgress?.table,
});
}
}
export function startDataCompareSession(input: DataCompareSessionConfig, tasks: DataCompareTableTask[], dependencies: DataCompareSessionDependencies): DataCompareSession {
const id = uuid();
const session = shallowReactive<DataCompareSession>({
id,
version: 0,
status: "running",
config: cloneConfig(input),
progress: null,
batchResults: [],
syncPlan: emptyDataCompareSyncPlan(),
error: null,
startedAt: Date.now(),
});
sessions.set(id, session);
const tracker = useExportTracker();
tracker.addDataCompareTask(
id,
input.label,
() => openDataCompareSession(id),
() => removeDataCompareSession(id),
);
void runDataCompareSession(session, tasks, dependencies);
return session;
}
export function getDataCompareSession(id: string | null | undefined): DataCompareSession | undefined {
return id ? sessions.get(id) : undefined;
}
export function removeDataCompareSession(id: string): boolean {
const session = sessions.get(id);
if (session?.status === "running") return false;
return sessions.delete(id);
}