feat(diff): add data compare and schema sync coverage

2026-11-01 00:25:38 +00:00 · 2026-05-09 23:44:49 +08:00
parent a878ec203a
commit 97b77ec46d
16 changed files with 1493 additions and 75 deletions
@@ -0,0 +1,181 @@
+import type { DatabaseType, QueryResult } from "@/types/database";
+import { quoteTableIdentifier } from "./tableSelectSql";
+import { formatGridSqlLiteral } from "./dataGridSql";
+
+export type DataCompareCellValue = QueryResult["rows"][number][number];
+
+export interface DataCompareChangedCell {
+  column: string;
+  source: DataCompareCellValue;
+  target: DataCompareCellValue;
+}
+
+export interface DataCompareRow {
+  key: string;
+  keyValues: Record<string, DataCompareCellValue>;
+  values: Record<string, DataCompareCellValue>;
+}
+
+export interface DataCompareModifiedRow {
+  key: string;
+  keyValues: Record<string, DataCompareCellValue>;
+  sourceValues: Record<string, DataCompareCellValue>;
+  targetValues: Record<string, DataCompareCellValue>;
+  changes: DataCompareChangedCell[];
+}
+
+export interface DataCompareResult {
+  added: DataCompareRow[];
+  removed: DataCompareRow[];
+  modified: DataCompareModifiedRow[];
+}
+
+export interface CompareDataRowsOptions {
+  columns: readonly string[];
+  keyColumns: readonly string[];
+  sourceRows: readonly (readonly DataCompareCellValue[])[];
+  targetRows: readonly (readonly DataCompareCellValue[])[];
+}
+
+export interface GenerateDataSyncSqlOptions {
+  tableName: string;
+  schema?: string;
+  columns: readonly string[];
+  keyColumns: readonly string[];
+  diff: DataCompareResult;
+  databaseType?: DatabaseType;
+}
+
+function rowObject(
+  columns: readonly string[],
+  row: readonly DataCompareCellValue[],
+): Record<string, DataCompareCellValue> {
+  const item: Record<string, DataCompareCellValue> = {};
+  columns.forEach((column, index) => {
+    item[column] = row[index] ?? null;
+  });
+  return item;
+}
+
+function keyFor(row: Record<string, DataCompareCellValue>, keyColumns: readonly string[]): string {
+  return keyColumns.map((column) => JSON.stringify(row[column] ?? null)).join("\u001f");
+}
+
+function keyValues(row: Record<string, DataCompareCellValue>, keyColumns: readonly string[]) {
+  const values: Record<string, DataCompareCellValue> = {};
+  keyColumns.forEach((column) => {
+    values[column] = row[column] ?? null;
+  });
+  return values;
+}
+
+export function compareDataRows(options: CompareDataRowsOptions): DataCompareResult {
+  if (options.keyColumns.length === 0) {
+    throw new Error("At least one key column is required for data comparison");
+  }
+
+  const source = new Map<string, Record<string, DataCompareCellValue>>();
+  const target = new Map<string, Record<string, DataCompareCellValue>>();
+  options.sourceRows.forEach((row) => {
+    const item = rowObject(options.columns, row);
+    const key = keyFor(item, options.keyColumns);
+    if (source.has(key)) throw new Error(`Duplicate source key: ${key}`);
+    source.set(key, item);
+  });
+  options.targetRows.forEach((row) => {
+    const item = rowObject(options.columns, row);
+    const key = keyFor(item, options.keyColumns);
+    if (target.has(key)) throw new Error(`Duplicate target key: ${key}`);
+    target.set(key, item);
+  });
+
+  const added: DataCompareRow[] = [];
+  const removed: DataCompareRow[] = [];
+  const modified: DataCompareModifiedRow[] = [];
+
+  for (const [key, sourceValues] of source) {
+    const targetValues = target.get(key);
+    if (!targetValues) {
+      added.push({ key, keyValues: keyValues(sourceValues, options.keyColumns), values: sourceValues });
+      continue;
+    }
+
+    const changes = options.columns
+      .filter((column) => !options.keyColumns.includes(column))
+      .filter((column) => sourceValues[column] !== targetValues[column])
+      .map((column) => ({ column, source: sourceValues[column] ?? null, target: targetValues[column] ?? null }));
+
+    if (changes.length > 0) {
+      modified.push({
+        key,
+        keyValues: keyValues(sourceValues, options.keyColumns),
+        sourceValues,
+        targetValues,
+        changes,
+      });
+    }
+  }
+
+  for (const [key, targetValues] of target) {
+    if (!source.has(key)) {
+      removed.push({ key, keyValues: keyValues(targetValues, options.keyColumns), values: targetValues });
+    }
+  }
+
+  return { added, removed, modified };
+}
+
+function qualifiedTableName(schema: string | undefined, tableName: string, databaseType?: DatabaseType): string {
+  const table = quoteTableIdentifier(databaseType, tableName);
+  return schema ? `${quoteTableIdentifier(databaseType, schema)}.${table}` : table;
+}
+
+function whereByKey(
+  keyValues: Record<string, DataCompareCellValue>,
+  keyColumns: readonly string[],
+  databaseType?: DatabaseType,
+): string {
+  return keyColumns
+    .map(
+      (column) =>
+        `${quoteTableIdentifier(databaseType, column)} = ${formatGridSqlLiteral(keyValues[column], databaseType)}`,
+    )
+    .join(" AND ");
+}
+
+export function generateDataSyncStatements(options: GenerateDataSyncSqlOptions): string[] {
+  const table = qualifiedTableName(options.schema, options.tableName, options.databaseType);
+  const statements: string[] = [];
+
+  for (const row of options.diff.added) {
+    const columns = options.columns.map((column) => quoteTableIdentifier(options.databaseType, column)).join(", ");
+    const values = options.columns
+      .map((column) => formatGridSqlLiteral(row.values[column], options.databaseType))
+      .join(", ");
+    statements.push(`INSERT INTO ${table} (${columns}) VALUES (${values});`);
+  }
+
+  for (const row of options.diff.modified) {
+    const assignments = row.changes
+      .map(
+        (change) =>
+          `${quoteTableIdentifier(options.databaseType, change.column)} = ${formatGridSqlLiteral(change.source, options.databaseType)}`,
+      )
+      .join(", ");
+    statements.push(
+      `UPDATE ${table} SET ${assignments} WHERE ${whereByKey(row.keyValues, options.keyColumns, options.databaseType)};`,
+    );
+  }
+
+  for (const row of options.diff.removed) {
+    statements.push(
+      `DELETE FROM ${table} WHERE ${whereByKey(row.keyValues, options.keyColumns, options.databaseType)};`,
+    );
+  }
+
+  return statements;
+}
+
+export function generateDataSyncSql(options: GenerateDataSyncSqlOptions): string {
+  return generateDataSyncStatements(options).join("\n");
+}
@@ -1,4 +1,4 @@
-import type { ColumnInfo, IndexInfo, DatabaseType } from "@/types/database";
+import type { ColumnInfo, IndexInfo, ForeignKeyInfo, TriggerInfo, DatabaseType } from "@/types/database";

 export interface ColumnDiff {
  type: "added" | "removed" | "modified";
@@ -9,17 +9,37 @@ export interface ColumnDiff {
 }

 export interface IndexDiff {
-  type: "added" | "removed";
+  type: "added" | "removed" | "modified";
  name: string;
  source?: IndexInfo;
  target?: IndexInfo;
+  changes?: string[];
+}
+
+export interface ForeignKeyDiff {
+  type: "added" | "removed" | "modified";
+  name: string;
+  source?: ForeignKeyInfo;
+  target?: ForeignKeyInfo;
+  changes?: string[];
+}
+
+export interface TriggerDiff {
+  type: "added" | "removed" | "modified";
+  name: string;
+  source?: TriggerInfo;
+  target?: TriggerInfo;
+  changes?: string[];
 }

 export interface TableDiff {
  type: "added" | "removed" | "modified";
+  objectType?: "table" | "view";
  name: string;
  columns?: ColumnDiff[];
  indexes?: IndexDiff[];
+  foreignKeys?: ForeignKeyDiff[];
+  triggers?: TriggerDiff[];
  ddl?: string;
 }

@@ -65,8 +85,33 @@ export function diffIndexes(source: IndexInfo[], target: IndexInfo[]): IndexDiff

  for (const si of source) {
    if (si.is_primary) continue;
-    if (!targetMap.has(si.name)) {
+    const ti = targetMap.get(si.name);
+    if (!ti) {
      diffs.push({ type: "added", name: si.name, source: si });
+      continue;
+    }
+
+    const changes: string[] = [];
+    if (si.is_unique !== ti.is_unique) {
+      changes.push(`unique: ${ti.is_unique ? "YES" : "NO"} → ${si.is_unique ? "YES" : "NO"}`);
+    }
+    if (si.columns.join(",") !== ti.columns.join(",")) {
+      changes.push(`columns: ${ti.columns.join(", ")} → ${si.columns.join(", ")}`);
+    }
+    if ((si.index_type ?? "") !== (ti.index_type ?? "")) {
+      changes.push(`type: ${ti.index_type ?? "default"} → ${si.index_type ?? "default"}`);
+    }
+    if ((si.filter ?? "") !== (ti.filter ?? "")) {
+      changes.push(`filter: ${ti.filter ?? "none"} → ${si.filter ?? "none"}`);
+    }
+    if ((si.included_columns ?? []).join(",") !== (ti.included_columns ?? []).join(",")) {
+      changes.push(
+        `include: ${(ti.included_columns ?? []).join(", ") || "none"} → ${(si.included_columns ?? []).join(", ") || "none"}`,
+      );
+    }
+
+    if (changes.length > 0) {
+      diffs.push({ type: "modified", name: si.name, source: si, target: ti, changes });
    }
  }

@@ -80,6 +125,71 @@ export function diffIndexes(source: IndexInfo[], target: IndexInfo[]): IndexDiff
  return diffs;
 }

+export function diffForeignKeys(source: ForeignKeyInfo[], target: ForeignKeyInfo[]): ForeignKeyDiff[] {
+  const diffs: ForeignKeyDiff[] = [];
+  const targetMap = new Map(target.map((fk) => [fk.name, fk]));
+  const sourceMap = new Map(source.map((fk) => [fk.name, fk]));
+
+  for (const sfk of source) {
+    const tfk = targetMap.get(sfk.name);
+    if (!tfk) {
+      diffs.push({ type: "added", name: sfk.name, source: sfk });
+      continue;
+    }
+
+    const changes: string[] = [];
+    if (sfk.column !== tfk.column) changes.push(`column: ${tfk.column} → ${sfk.column}`);
+    if (sfk.ref_table !== tfk.ref_table) changes.push(`ref table: ${tfk.ref_table} → ${sfk.ref_table}`);
+    if (sfk.ref_column !== tfk.ref_column) changes.push(`ref column: ${tfk.ref_column} → ${sfk.ref_column}`);
+
+    if (changes.length > 0) {
+      diffs.push({ type: "modified", name: sfk.name, source: sfk, target: tfk, changes });
+    }
+  }
+
+  for (const tfk of target) {
+    if (!sourceMap.has(tfk.name)) {
+      diffs.push({ type: "removed", name: tfk.name, target: tfk });
+    }
+  }
+
+  return diffs;
+}
+
+export function diffTriggers(source: TriggerInfo[], target: TriggerInfo[]): TriggerDiff[] {
+  const diffs: TriggerDiff[] = [];
+  const targetMap = new Map(target.map((trigger) => [trigger.name, trigger]));
+  const sourceMap = new Map(source.map((trigger) => [trigger.name, trigger]));
+
+  for (const sourceTrigger of source) {
+    const targetTrigger = targetMap.get(sourceTrigger.name);
+    if (!targetTrigger) {
+      diffs.push({ type: "added", name: sourceTrigger.name, source: sourceTrigger });
+      continue;
+    }
+
+    const changes: string[] = [];
+    if (sourceTrigger.event !== targetTrigger.event) {
+      changes.push(`event: ${targetTrigger.event} → ${sourceTrigger.event}`);
+    }
+    if (sourceTrigger.timing !== targetTrigger.timing) {
+      changes.push(`timing: ${targetTrigger.timing} → ${sourceTrigger.timing}`);
+    }
+
+    if (changes.length > 0) {
+      diffs.push({ type: "modified", name: sourceTrigger.name, source: sourceTrigger, target: targetTrigger, changes });
+    }
+  }
+
+  for (const targetTrigger of target) {
+    if (!sourceMap.has(targetTrigger.name)) {
+      diffs.push({ type: "removed", name: targetTrigger.name, target: targetTrigger });
+    }
+  }
+
+  return diffs;
+}
+
 export function diffTables(
  sourceTables: string[],
  targetTables: string[],
@@ -109,23 +219,79 @@ function columnDef(col: ColumnInfo, dbType: DatabaseType): string {
  return def;
 }

-export function generateSyncSql(diffs: TableDiff[], dbType: DatabaseType): string {
+function qualifiedName(name: string, dbType: DatabaseType, schema?: string): string {
+  return schema ? `${quoteId(schema, dbType)}.${quoteId(name, dbType)}` : quoteId(name, dbType);
+}
+
+function dropIndexSql(tableName: string, indexName: string, dbType: DatabaseType, schema?: string): string {
+  const qt = qualifiedName(tableName, dbType, schema);
+  const qi = qualifiedName(indexName, dbType, schema);
+  if (dbType === "mysql" || dbType === "doris" || dbType === "starrocks") {
+    return `DROP INDEX ${quoteId(indexName, dbType)} ON ${qt};`;
+  }
+  return `DROP INDEX IF EXISTS ${qi};`;
+}
+
+function createIndexSql(tableName: string, idx: IndexInfo, dbType: DatabaseType, schema?: string): string {
+  const qt = qualifiedName(tableName, dbType, schema);
+  const cols = idx.columns.map((c) => quoteId(c, dbType)).join(", ");
+  const unique = idx.is_unique ? "UNIQUE " : "";
+  const idxType = idx.index_type ?? "";
+  const usingClause = idxType && dbType === "postgres" ? ` USING ${idxType}` : "";
+  const typePrefix = idxType && dbType === "sqlserver" ? `${idxType} ` : "";
+  const incCols = idx.included_columns ?? [];
+  const includeClause =
+    incCols.length > 0 && (dbType === "postgres" || dbType === "sqlserver")
+      ? ` INCLUDE (${incCols.map((c) => quoteId(c, dbType)).join(", ")})`
+      : "";
+  const supportsWhere = dbType === "postgres" || dbType === "sqlserver" || dbType === "sqlite";
+  const filter = idx.filter && supportsWhere ? ` WHERE ${idx.filter}` : "";
+  return `CREATE ${unique}${typePrefix}INDEX ${quoteId(idx.name, dbType)} ON ${qt}${usingClause} (${cols})${includeClause}${filter};`;
+}
+
+function dropForeignKeySql(tableName: string, fkName: string, dbType: DatabaseType, schema?: string): string {
+  const qt = qualifiedName(tableName, dbType, schema);
+  const qf = quoteId(fkName, dbType);
+  if (dbType === "mysql" || dbType === "doris" || dbType === "starrocks") {
+    return `ALTER TABLE ${qt} DROP FOREIGN KEY ${qf};`;
+  }
+  return `ALTER TABLE ${qt} DROP CONSTRAINT ${qf};`;
+}
+
+function addForeignKeySql(tableName: string, fk: ForeignKeyInfo, dbType: DatabaseType, schema?: string): string {
+  const qt = qualifiedName(tableName, dbType, schema);
+  return `ALTER TABLE ${qt} ADD CONSTRAINT ${quoteId(fk.name, dbType)} FOREIGN KEY (${quoteId(fk.column, dbType)}) REFERENCES ${quoteId(fk.ref_table, dbType)} (${quoteId(fk.ref_column, dbType)});`;
+}
+
+function dropObjectSql(diff: TableDiff, dbType: DatabaseType, schema?: string): string {
+  const objectType = diff.objectType === "view" ? "VIEW" : "TABLE";
+  return `DROP ${objectType} IF EXISTS ${qualifiedName(diff.name, dbType, schema)};`;
+}
+
+export function generateSyncSql(diffs: TableDiff[], dbType: DatabaseType, schema?: string): string {
  const lines: string[] = [];
  const isMySQL = dbType === "mysql" || dbType === "doris" || dbType === "starrocks";

  for (const diff of diffs) {
-    const qt = quoteId(diff.name, dbType);
+    const qt = qualifiedName(diff.name, dbType, schema);

    if (diff.type === "added" && diff.ddl) {
-      lines.push(`-- Create table: ${diff.name}`);
+      lines.push(`-- Create ${diff.objectType ?? "table"}: ${diff.name}`);
      lines.push(diff.ddl + ";");
      lines.push("");
      continue;
    }

+    if (diff.type === "added" && diff.objectType === "view") {
+      lines.push(`-- View exists only in source: ${diff.name}`);
+      lines.push("-- Source view definition is not available from this driver yet.");
+      lines.push("");
+      continue;
+    }
+
    if (diff.type === "removed") {
-      lines.push(`-- Drop table: ${diff.name}`);
-      lines.push(`DROP TABLE IF EXISTS ${qt};`);
+      lines.push(`-- Drop ${diff.objectType ?? "table"}: ${diff.name}`);
+      lines.push(dropObjectSql(diff, dbType, schema));
      lines.push("");
      continue;
    }
@@ -133,6 +299,14 @@ export function generateSyncSql(diffs: TableDiff[], dbType: DatabaseType): strin
    if (diff.type === "modified") {
      const parts: string[] = [];

+      if (diff.foreignKeys) {
+        for (const fk of diff.foreignKeys) {
+          if (fk.type === "removed" || fk.type === "modified") {
+            lines.push(dropForeignKeySql(diff.name, fk.name, dbType, schema));
+          }
+        }
+      }
+
      if (diff.columns) {
        for (const col of diff.columns) {
          if (col.type === "added" && col.source) {
@@ -164,34 +338,6 @@ export function generateSyncSql(diffs: TableDiff[], dbType: DatabaseType): strin
        }
      }

-      if (diff.indexes) {
-        for (const idx of diff.indexes) {
-          if (idx.type === "added" && idx.source) {
-            const cols = idx.source.columns.map((c) => quoteId(c, dbType)).join(", ");
-            const unique = idx.source.is_unique ? "UNIQUE " : "";
-            const idxType = idx.source.index_type ?? "";
-            const usingClause = idxType && dbType === "postgres" ? ` USING ${idxType}` : "";
-            const typePrefix = idxType && dbType === "sqlserver" ? `${idxType} ` : "";
-            const incCols = idx.source.included_columns ?? [];
-            const includeClause =
-              incCols.length > 0 && (dbType === "postgres" || dbType === "sqlserver")
-                ? ` INCLUDE (${incCols.map((c) => quoteId(c, dbType)).join(", ")})`
-                : "";
-            const supportsWhere = dbType === "postgres" || dbType === "sqlserver" || dbType === "sqlite";
-            const filter = idx.source.filter && supportsWhere ? ` WHERE ${idx.source.filter}` : "";
-            lines.push(
-              `CREATE ${unique}${typePrefix}INDEX ${quoteId(idx.name, dbType)} ON ${qt}${usingClause} (${cols})${includeClause}${filter};`,
-            );
-          } else if (idx.type === "removed") {
-            if (isMySQL) {
-              lines.push(`DROP INDEX ${quoteId(idx.name, dbType)} ON ${qt};`);
-            } else {
-              lines.push(`DROP INDEX IF EXISTS ${quoteId(idx.name, dbType)};`);
-            }
-          }
-        }
-      }
-
      if (parts.length > 0) {
        lines.push(`-- Alter table: ${diff.name}`);
        if (isMySQL) {
@@ -204,6 +350,52 @@ export function generateSyncSql(diffs: TableDiff[], dbType: DatabaseType): strin
        }
        lines.push("");
      }
+
+      if (diff.indexes) {
+        for (const idx of diff.indexes) {
+          if (idx.type === "added" && idx.source) {
+            lines.push(createIndexSql(diff.name, idx.source, dbType, schema));
+          } else if (idx.type === "removed") {
+            lines.push(dropIndexSql(diff.name, idx.name, dbType, schema));
+          } else if (idx.type === "modified" && idx.source) {
+            lines.push(dropIndexSql(diff.name, idx.name, dbType, schema));
+            lines.push(createIndexSql(diff.name, idx.source, dbType, schema));
+          }
+        }
+      }
+
+      if (diff.foreignKeys) {
+        for (const fk of diff.foreignKeys) {
+          if (fk.type === "added" && fk.source) {
+            lines.push(addForeignKeySql(diff.name, fk.source, dbType, schema));
+          } else if (fk.type === "modified" && fk.source) {
+            lines.push(addForeignKeySql(diff.name, fk.source, dbType, schema));
+          }
+        }
+      }
+
+      if (diff.triggers) {
+        for (const trigger of diff.triggers) {
+          lines.push(
+            `-- Trigger ${trigger.type}: ${trigger.name} on ${diff.name}; review trigger definition manually.`,
+          );
+        }
+      }
+
+      if (diff.indexes || diff.foreignKeys || diff.triggers) {
+        if (
+          (diff.indexes?.length ?? 0) > 0 ||
+          (diff.foreignKeys?.length ?? 0) > 0 ||
+          (diff.triggers?.length ?? 0) > 0
+        ) {
+          lines.push("");
+        }
+      }
+
+      if (dbType === "sqlite" && diff.foreignKeys?.length) {
+        lines.push(`-- SQLite foreign key synchronization may require table rebuild for: ${diff.name}`);
+        lines.push("");
+      }
    }
  }

@@ -308,7 +308,7 @@ function isInColumnContext(beforeCursor: string): boolean {
  for (let i = lastWords.length - 1; i >= Math.max(0, lastWords.length - 3); i--) {
    const word = lastWords[i]?.toLowerCase().replace(/[^a-z0-9.]/g, "") ?? "";
    // Operators that indicate column context
-    if (/^[=<>!\+\-\*\/(,]$/.test(word)) return true;
+    if (/^[=<>!+\-*/(,]$/.test(word)) return true;
    // Keywords that directly precede column expressions
    if (["where", "on", "having", "set", "and", "or", "not", "is", "like", "in", "between", "select"].includes(word)) {
      return true;
@@ -37,8 +37,12 @@ function crc32(data: Uint8Array): number {
 }

 function escapeXml(value: string): string {
-  return value
-    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, "")
+  return [...value]
+    .filter((char) => {
+      const code = char.charCodeAt(0);
+      return code === 9 || code === 10 || code === 13 || code >= 32;
+    })
+    .join("")
    .replace(/&/g, "&amp;")
    .replace(/</g, "&lt;")
    .replace(/>/g, "&gt;")
@@ -66,7 +70,12 @@ function sheetRange(columnCount: number, rowCount: number): string {
 }

 function normalizeSheetName(value?: string): string {
-  const name = (value || "Sheet1").replace(/[\[\]:*?/\\]/g, " ").trim() || "Sheet1";
+  const invalidChars = new Set(["[", "]", ":", "*", "?", "/", "\\"]);
+  const name =
+    [...(value || "Sheet1")]
+      .map((char) => (invalidChars.has(char) ? " " : char))
+      .join("")
+      .trim() || "Sheet1";
  return name.slice(0, 31);
 }