From eea3faa2244b301b4ea9cc64efdff2f649be0d83 Mon Sep 17 00:00:00 2001 From: dienaso <89768641@qq.com> Date: Sun, 6 Sep 2026 10:29:48 +0800 Subject: [PATCH] fix(compare): match data compare columns case-insensitively --- .../src/components/diff/DataCompareDialog.vue | 27 ++++++-- .../__tests__/dataCompareKeyInference.spec.ts | 39 ++++++++++- apps/desktop/src/lib/dataGrid/dataCompare.ts | 47 ++++++++++++++ crates/dbx-core/src/data_compare.rs | 64 +++++++++++++++++-- 4 files changed, 165 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/components/diff/DataCompareDialog.vue b/apps/desktop/src/components/diff/DataCompareDialog.vue index d249e934e1..ab89e53046 100644 --- a/apps/desktop/src/components/diff/DataCompareDialog.vue +++ b/apps/desktop/src/components/diff/DataCompareDialog.vue @@ -14,7 +14,7 @@ import { databaseOptionsForConnection, fetchNamespaceOptionsForConnection } from import { isSchemaAware } from "@/lib/database/databaseCapabilities"; import { copyToClipboard } from "@/lib/common/clipboard"; import type { DataCompareCellValue, DataCompareModifiedRow, DataCompareResult, DataCompareRow, DataCompareSyncPlan, DataCompareSyncPlanTableOptions } from "@/lib/dataGrid/dataCompare"; -import { inferCompareKeyColumns } from "@/lib/dataGrid/dataCompare"; +import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase } from "@/lib/dataGrid/dataCompare"; import type { ColumnInfo, DatabaseType } from "@/types/database"; import * as api from "@/lib/backend/api"; import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard"; @@ -546,6 +546,8 @@ async function startCompare() { if (!targetTables.value.includes(task.targetTable)) { const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable); const resolvedKeys = keyColumns.value.length > 0 ? keyColumns.value : []; + const sourceColumnNames = sourceColumns.map((column) => column.name); + const sourceKeyColumns = resolvedKeys.map((key) => matchColumnNameIgnoreCase(key, sourceColumnNames) ?? key); const preparation = await api.prepareDataCompareMissingTarget({ sourceConnectionId: sourceConnectionId.value, sourceDatabase: sourceDatabase.value, @@ -555,12 +557,12 @@ async function startCompare() { targetDatabase: targetDatabase.value, targetSchema: targetSchema.value, targetTable: task.targetTable, - keyColumns: resolvedKeys, + keyColumns: sourceKeyColumns, }); results.push({ sourceTable: task.sourceTable, targetTable: task.targetTable, - keyColumns: resolvedKeys, + keyColumns: sourceKeyColumns, columns: sourceColumns.map((column) => column.name), columnInfo: sourceColumns, status: "different", @@ -591,9 +593,19 @@ async function startCompare() { const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable); const targetColumns = await loadColumnsWithCache(targetColumnCache, targetConnectionId.value, targetDatabase.value, targetSchema.value, task.targetTable); - const columns = sourceColumns.map((column) => column.name).filter((column) => targetColumns.some((target) => target.name === column)); + const matched = intersectCompareColumns(sourceColumns, targetColumns); + const columns = matched.columns; const columnInfo = columns.map((column) => targetColumns.find((target) => target.name === column)).filter((column): column is CompareColumn => !!column); - const missingKeys = resolvedKeys.filter((column) => !columns.includes(column)); + const canonicalKeyColumns: string[] = []; + const missingKeys: string[] = []; + for (const key of resolvedKeys) { + const canonical = matchColumnNameIgnoreCase(key, columns); + if (!canonical) { + missingKeys.push(key); + } else if (!canonicalKeyColumns.includes(canonical)) { + canonicalKeyColumns.push(canonical); + } + } if (missingKeys.length > 0) { throw new Error(t("dataCompare.missingKeyColumns", { columns: missingKeys.join(", ") })); } @@ -611,7 +623,8 @@ async function startCompare() { targetSchema: targetSchema.value, targetTable: task.targetTable, columns, - keyColumns: resolvedKeys, + keyColumns: canonicalKeyColumns, + sourceColumns: matched.sourceColumns, }); const added = preparation.result.added.length; @@ -622,7 +635,7 @@ async function startCompare() { results.push({ sourceTable: task.sourceTable, targetTable: task.targetTable, - keyColumns: resolvedKeys, + keyColumns: canonicalKeyColumns, columns, columnInfo, status, diff --git a/apps/desktop/src/lib/dataGrid/__tests__/dataCompareKeyInference.spec.ts b/apps/desktop/src/lib/dataGrid/__tests__/dataCompareKeyInference.spec.ts index a06c53f443..35d61a9c96 100644 --- a/apps/desktop/src/lib/dataGrid/__tests__/dataCompareKeyInference.spec.ts +++ b/apps/desktop/src/lib/dataGrid/__tests__/dataCompareKeyInference.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { inferCompareKeyColumns } from "../dataCompare"; +import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase } from "../dataCompare"; describe("inferCompareKeyColumns", () => { it("returns all primary-key columns when a primary key exists", () => { @@ -34,3 +34,40 @@ describe("inferCompareKeyColumns", () => { expect(inferCompareKeyColumns([])).toEqual([]); }); }); + +describe("intersectCompareColumns", () => { + it("matches lower-case source columns against upper-case target columns", () => { + const intersection = intersectCompareColumns([{ name: "snid" }, { name: "username" }, { name: "amount" }], [{ name: "SNID" }, { name: "USERNAME" }, { name: "AMOUNT" }]); + expect(intersection.columns).toEqual(["SNID", "USERNAME", "AMOUNT"]); + expect(intersection.sourceColumns).toEqual(["snid", "username", "amount"]); + }); + + it("keeps source order and omits columns missing on either side", () => { + const intersection = intersectCompareColumns([{ name: "Amount" }, { name: "snid" }, { name: "extra" }], [{ name: "SNID" }, { name: "AMOUNT" }]); + expect(intersection.columns).toEqual(["AMOUNT", "SNID"]); + expect(intersection.sourceColumns).toEqual(["Amount", "snid"]); + }); + + it("returns an empty intersection when no column names match ignoring case", () => { + const intersection = intersectCompareColumns([{ name: "snid" }], [{ name: "id" }]); + expect(intersection.columns).toEqual([]); + expect(intersection.sourceColumns).toEqual([]); + }); + + it("deduplicates duplicated source columns", () => { + const intersection = intersectCompareColumns([{ name: "snid" }, { name: "SNID" }], [{ name: "SNID" }]); + expect(intersection.columns).toEqual(["SNID"]); + expect(intersection.sourceColumns).toEqual(["snid"]); + }); +}); + +describe("matchColumnNameIgnoreCase", () => { + it("resolves a name to the canonical entry ignoring case", () => { + expect(matchColumnNameIgnoreCase("SNID", ["snid", "name"])).toBe("snid"); + expect(matchColumnNameIgnoreCase("Name", ["SNID", "NAME"])).toBe("NAME"); + }); + + it("returns undefined when no column matches", () => { + expect(matchColumnNameIgnoreCase("missing", ["snid"])).toBeUndefined(); + }); +}); diff --git a/apps/desktop/src/lib/dataGrid/dataCompare.ts b/apps/desktop/src/lib/dataGrid/dataCompare.ts index e154d2d325..10f4b5a10d 100644 --- a/apps/desktop/src/lib/dataGrid/dataCompare.ts +++ b/apps/desktop/src/lib/dataGrid/dataCompare.ts @@ -65,6 +65,8 @@ export interface DataCompareFromTablesOptions { targetTable: string; columns: string[]; keyColumns: string[]; + /** Source-side column names aligned positionally with `columns`; needed when the two databases store identifier case differently (e.g. SQL Server vs Oracle). */ + sourceColumns?: string[]; fetchBatchSize?: number; degradationThreshold?: DegradationThreshold; samplingStrategy?: SamplingStrategy; @@ -139,3 +141,48 @@ export function inferCompareKeyColumns(columns: Pick[], targetColumns: Pick[]): CompareColumnIntersection { + const targetNamesByLowerName = new Map(); + for (const column of targetColumns) { + const lowerName = column.name.toLowerCase(); + if (!targetNamesByLowerName.has(lowerName)) targetNamesByLowerName.set(lowerName, column.name); + } + + const columns: string[] = []; + const sourceNames: string[] = []; + const matched = new Set(); + for (const column of sourceColumns) { + const lowerName = column.name.toLowerCase(); + const targetName = targetNamesByLowerName.get(lowerName); + if (targetName === undefined || matched.has(lowerName)) continue; + matched.add(lowerName); + columns.push(targetName); + sourceNames.push(column.name); + } + return { columns, sourceColumns: sourceNames }; +} + +/** + * Resolves a column name to the matching entry of `columns`, ignoring case. + * Returns undefined when no column matches. + */ +export function matchColumnNameIgnoreCase(name: string, columns: string[]): string | undefined { + const lowerName = name.toLowerCase(); + return columns.find((column) => column.toLowerCase() === lowerName); +} diff --git a/crates/dbx-core/src/data_compare.rs b/crates/dbx-core/src/data_compare.rs index 12ccb8cdba..72f0431f20 100644 --- a/crates/dbx-core/src/data_compare.rs +++ b/crates/dbx-core/src/data_compare.rs @@ -62,6 +62,8 @@ pub struct DataCompareFromTablesOptions { pub columns: Vec, pub key_columns: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_columns: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub fetch_batch_size: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub degradation_threshold: Option, @@ -356,6 +358,11 @@ pub async fn prepare_data_compare_from_tables( let sampling_strategy = options.sampling_strategy.clone().unwrap_or(SamplingStrategy::Hybrid); let enable_checksum = options.enable_checksum.unwrap_or(true); + // Cross-database comparison matches columns case-insensitively, so `columns` holds + // target-side names while the source side needs its own names in the same order. + let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref()); + let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns); + let (source_rows, target_rows, sampling_rate, verification_method) = match °radation_level { DegradationLevel::Full => { let (src, tgt) = tokio::try_join!( @@ -365,8 +372,8 @@ pub async fn prepare_data_compare_from_tables( &options.source_database, &options.source_schema, &options.source_table, - &options.columns, - &options.key_columns, + &source_column_names, + &source_key_columns, source_database_type, fetch_batch_size, ), @@ -392,8 +399,8 @@ pub async fn prepare_data_compare_from_tables( &options.source_database, &options.source_schema, &options.source_table, - &options.columns, - &options.key_columns, + &source_column_names, + &source_key_columns, source_database_type, &sampling_strategy, threshold.sample_size, @@ -1046,6 +1053,27 @@ fn first_count(rows: &[Vec]) -> Result { .ok_or_else(|| format!("COUNT query returned non-numeric value: {value}")) } +/// Data compare matches columns case-insensitively across databases that store +/// identifiers with different case conventions (e.g. SQL Server keeps the created +/// case while Oracle stores upper case). `columns` carries target-side names, so +/// the source side is queried with its own names in the same positional order. +fn aligned_source_column_names(columns: &[String], source_columns: Option<&Vec>) -> Vec { + match source_columns { + Some(source_columns) if source_columns.len() == columns.len() => source_columns.clone(), + _ => columns.to_vec(), + } +} + +fn aligned_source_key_columns(columns: &[String], source_columns: &[String], key_columns: &[String]) -> Vec { + key_columns + .iter() + .map(|key| match columns.iter().position(|column| column == key) { + Some(index) => source_columns[index].clone(), + None => key.clone(), + }) + .collect() +} + fn build_data_compare_select_sql( database_type: DatabaseType, schema: &str, @@ -1648,6 +1676,7 @@ pub async fn verify_data(state: &AppState, options: VerifyDataOptions) -> Result target_table: options.target_table, columns: options.columns, key_columns: options.key_columns, + source_columns: None, fetch_batch_size: options.fetch_batch_size, degradation_threshold: Some(degradation_threshold), sampling_strategy: Some(sampling_strategy), @@ -2531,4 +2560,31 @@ mod tests { let up = snapshot.iter().find(|e| e.name == "dbx_auto_upgrade_total").unwrap(); assert_eq!(up.value, crate::risk_metrics::MetricValue::Counter(1)); } + + #[test] + fn aligned_source_column_names_keep_source_case_in_positional_order() { + let columns = vec!["SNID".to_string(), "NAME".to_string()]; + let source_columns = vec!["snid".to_string(), "name".to_string()]; + + let aligned = aligned_source_column_names(&columns, Some(&source_columns)); + assert_eq!(aligned, source_columns); + + let fallback = aligned_source_column_names(&columns, None); + assert_eq!(fallback, columns); + + let mismatched = aligned_source_column_names(&columns, Some(&vec!["snid".to_string()])); + assert_eq!(mismatched, columns); + } + + #[test] + fn aligned_source_key_columns_map_keys_by_position() { + let columns = vec!["SNID".to_string(), "NAME".to_string()]; + let source_columns = vec!["snid".to_string(), "name".to_string()]; + + let keys = aligned_source_key_columns(&columns, &source_columns, &["NAME".to_string()]); + assert_eq!(keys, vec!["name".to_string()]); + + let unknown = aligned_source_key_columns(&columns, &source_columns, &["missing".to_string()]); + assert_eq!(unknown, vec!["missing".to_string()]); + } }