fix(compare): match data compare columns case-insensitively
This commit is contained in:
1 parent
ab35a5b1da
commit
eea3faa224
4 files changed
+165
-12
No files matched your search
@@ -14,7 +14,7 @@ import { databaseOptionsForConnection, fetchNamespaceOptionsForConnection } from
|
|||||||
import { isSchemaAware } from "@/lib/database/databaseCapabilities";
|
import { isSchemaAware } from "@/lib/database/databaseCapabilities";
|
||||||
import { copyToClipboard } from "@/lib/common/clipboard";
|
import { copyToClipboard } from "@/lib/common/clipboard";
|
||||||
import type { DataCompareCellValue, DataCompareModifiedRow, DataCompareResult, DataCompareRow, DataCompareSyncPlan, DataCompareSyncPlanTableOptions } from "@/lib/dataGrid/dataCompare";
|
import type { DataCompareCellValue, DataCompareModifiedRow, DataCompareResult, DataCompareRow, DataCompareSyncPlan, DataCompareSyncPlanTableOptions } from "@/lib/dataGrid/dataCompare";
|
||||||
import { inferCompareKeyColumns } from "@/lib/dataGrid/dataCompare";
|
import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase } from "@/lib/dataGrid/dataCompare";
|
||||||
import type { ColumnInfo, DatabaseType } from "@/types/database";
|
import type { ColumnInfo, DatabaseType } from "@/types/database";
|
||||||
import * as api from "@/lib/backend/api";
|
import * as api from "@/lib/backend/api";
|
||||||
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
|
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
|
||||||
@@ -546,6 +546,8 @@ async function startCompare() {
|
|||||||
if (!targetTables.value.includes(task.targetTable)) {
|
if (!targetTables.value.includes(task.targetTable)) {
|
||||||
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable);
|
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable);
|
||||||
const resolvedKeys = keyColumns.value.length > 0 ? keyColumns.value : [];
|
const resolvedKeys = keyColumns.value.length > 0 ? keyColumns.value : [];
|
||||||
|
const sourceColumnNames = sourceColumns.map((column) => column.name);
|
||||||
|
const sourceKeyColumns = resolvedKeys.map((key) => matchColumnNameIgnoreCase(key, sourceColumnNames) ?? key);
|
||||||
const preparation = await api.prepareDataCompareMissingTarget({
|
const preparation = await api.prepareDataCompareMissingTarget({
|
||||||
sourceConnectionId: sourceConnectionId.value,
|
sourceConnectionId: sourceConnectionId.value,
|
||||||
sourceDatabase: sourceDatabase.value,
|
sourceDatabase: sourceDatabase.value,
|
||||||
@@ -555,12 +557,12 @@ async function startCompare() {
|
|||||||
targetDatabase: targetDatabase.value,
|
targetDatabase: targetDatabase.value,
|
||||||
targetSchema: targetSchema.value,
|
targetSchema: targetSchema.value,
|
||||||
targetTable: task.targetTable,
|
targetTable: task.targetTable,
|
||||||
keyColumns: resolvedKeys,
|
keyColumns: sourceKeyColumns,
|
||||||
});
|
});
|
||||||
results.push({
|
results.push({
|
||||||
sourceTable: task.sourceTable,
|
sourceTable: task.sourceTable,
|
||||||
targetTable: task.targetTable,
|
targetTable: task.targetTable,
|
||||||
keyColumns: resolvedKeys,
|
keyColumns: sourceKeyColumns,
|
||||||
columns: sourceColumns.map((column) => column.name),
|
columns: sourceColumns.map((column) => column.name),
|
||||||
columnInfo: sourceColumns,
|
columnInfo: sourceColumns,
|
||||||
status: "different",
|
status: "different",
|
||||||
@@ -591,9 +593,19 @@ async function startCompare() {
|
|||||||
|
|
||||||
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable);
|
const sourceColumns = await loadColumnsWithCache(sourceColumnCache, sourceConnectionId.value, sourceDatabase.value, sourceSchema.value, task.sourceTable);
|
||||||
const targetColumns = await loadColumnsWithCache(targetColumnCache, targetConnectionId.value, targetDatabase.value, targetSchema.value, task.targetTable);
|
const targetColumns = await loadColumnsWithCache(targetColumnCache, targetConnectionId.value, targetDatabase.value, targetSchema.value, task.targetTable);
|
||||||
const columns = sourceColumns.map((column) => column.name).filter((column) => targetColumns.some((target) => target.name === column));
|
const matched = intersectCompareColumns(sourceColumns, targetColumns);
|
||||||
|
const columns = matched.columns;
|
||||||
const columnInfo = columns.map((column) => targetColumns.find((target) => target.name === column)).filter((column): column is CompareColumn => !!column);
|
const columnInfo = columns.map((column) => targetColumns.find((target) => target.name === column)).filter((column): column is CompareColumn => !!column);
|
||||||
const missingKeys = resolvedKeys.filter((column) => !columns.includes(column));
|
const canonicalKeyColumns: string[] = [];
|
||||||
|
const missingKeys: string[] = [];
|
||||||
|
for (const key of resolvedKeys) {
|
||||||
|
const canonical = matchColumnNameIgnoreCase(key, columns);
|
||||||
|
if (!canonical) {
|
||||||
|
missingKeys.push(key);
|
||||||
|
} else if (!canonicalKeyColumns.includes(canonical)) {
|
||||||
|
canonicalKeyColumns.push(canonical);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (missingKeys.length > 0) {
|
if (missingKeys.length > 0) {
|
||||||
throw new Error(t("dataCompare.missingKeyColumns", { columns: missingKeys.join(", ") }));
|
throw new Error(t("dataCompare.missingKeyColumns", { columns: missingKeys.join(", ") }));
|
||||||
}
|
}
|
||||||
@@ -611,7 +623,8 @@ async function startCompare() {
|
|||||||
targetSchema: targetSchema.value,
|
targetSchema: targetSchema.value,
|
||||||
targetTable: task.targetTable,
|
targetTable: task.targetTable,
|
||||||
columns,
|
columns,
|
||||||
keyColumns: resolvedKeys,
|
keyColumns: canonicalKeyColumns,
|
||||||
|
sourceColumns: matched.sourceColumns,
|
||||||
});
|
});
|
||||||
|
|
||||||
const added = preparation.result.added.length;
|
const added = preparation.result.added.length;
|
||||||
@@ -622,7 +635,7 @@ async function startCompare() {
|
|||||||
results.push({
|
results.push({
|
||||||
sourceTable: task.sourceTable,
|
sourceTable: task.sourceTable,
|
||||||
targetTable: task.targetTable,
|
targetTable: task.targetTable,
|
||||||
keyColumns: resolvedKeys,
|
keyColumns: canonicalKeyColumns,
|
||||||
columns,
|
columns,
|
||||||
columnInfo,
|
columnInfo,
|
||||||
status,
|
status,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { inferCompareKeyColumns } from "../dataCompare";
|
import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase } from "../dataCompare";
|
||||||
|
|
||||||
describe("inferCompareKeyColumns", () => {
|
describe("inferCompareKeyColumns", () => {
|
||||||
it("returns all primary-key columns when a primary key exists", () => {
|
it("returns all primary-key columns when a primary key exists", () => {
|
||||||
@@ -34,3 +34,40 @@ describe("inferCompareKeyColumns", () => {
|
|||||||
expect(inferCompareKeyColumns([])).toEqual([]);
|
expect(inferCompareKeyColumns([])).toEqual([]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("intersectCompareColumns", () => {
|
||||||
|
it("matches lower-case source columns against upper-case target columns", () => {
|
||||||
|
const intersection = intersectCompareColumns([{ name: "snid" }, { name: "username" }, { name: "amount" }], [{ name: "SNID" }, { name: "USERNAME" }, { name: "AMOUNT" }]);
|
||||||
|
expect(intersection.columns).toEqual(["SNID", "USERNAME", "AMOUNT"]);
|
||||||
|
expect(intersection.sourceColumns).toEqual(["snid", "username", "amount"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps source order and omits columns missing on either side", () => {
|
||||||
|
const intersection = intersectCompareColumns([{ name: "Amount" }, { name: "snid" }, { name: "extra" }], [{ name: "SNID" }, { name: "AMOUNT" }]);
|
||||||
|
expect(intersection.columns).toEqual(["AMOUNT", "SNID"]);
|
||||||
|
expect(intersection.sourceColumns).toEqual(["Amount", "snid"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns an empty intersection when no column names match ignoring case", () => {
|
||||||
|
const intersection = intersectCompareColumns([{ name: "snid" }], [{ name: "id" }]);
|
||||||
|
expect(intersection.columns).toEqual([]);
|
||||||
|
expect(intersection.sourceColumns).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates duplicated source columns", () => {
|
||||||
|
const intersection = intersectCompareColumns([{ name: "snid" }, { name: "SNID" }], [{ name: "SNID" }]);
|
||||||
|
expect(intersection.columns).toEqual(["SNID"]);
|
||||||
|
expect(intersection.sourceColumns).toEqual(["snid"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("matchColumnNameIgnoreCase", () => {
|
||||||
|
it("resolves a name to the canonical entry ignoring case", () => {
|
||||||
|
expect(matchColumnNameIgnoreCase("SNID", ["snid", "name"])).toBe("snid");
|
||||||
|
expect(matchColumnNameIgnoreCase("Name", ["SNID", "NAME"])).toBe("NAME");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns undefined when no column matches", () => {
|
||||||
|
expect(matchColumnNameIgnoreCase("missing", ["snid"])).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -65,6 +65,8 @@ export interface DataCompareFromTablesOptions {
|
|||||||
targetTable: string;
|
targetTable: string;
|
||||||
columns: string[];
|
columns: string[];
|
||||||
keyColumns: string[];
|
keyColumns: string[];
|
||||||
|
/** Source-side column names aligned positionally with `columns`; needed when the two databases store identifier case differently (e.g. SQL Server vs Oracle). */
|
||||||
|
sourceColumns?: string[];
|
||||||
fetchBatchSize?: number;
|
fetchBatchSize?: number;
|
||||||
degradationThreshold?: DegradationThreshold;
|
degradationThreshold?: DegradationThreshold;
|
||||||
samplingStrategy?: SamplingStrategy;
|
samplingStrategy?: SamplingStrategy;
|
||||||
@@ -139,3 +141,48 @@ export function inferCompareKeyColumns(columns: Pick<ColumnInfo, "name" | "is_pr
|
|||||||
}
|
}
|
||||||
return primaryKeys;
|
return primaryKeys;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CompareColumnIntersection {
|
||||||
|
/** Target-side column names, kept in source order. */
|
||||||
|
columns: string[];
|
||||||
|
/** Source-side names aligned positionally with `columns`. */
|
||||||
|
sourceColumns: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Intersects source and target columns case-insensitively.
|
||||||
|
*
|
||||||
|
* Databases store identifier case differently (SQL Server keeps the created
|
||||||
|
* case, Oracle and Dameng store upper case, PostgreSQL lower case), so a
|
||||||
|
* migrated table's columns only match across engines when compared without
|
||||||
|
* case. Target names are canonical because sync SQL runs against the target.
|
||||||
|
*/
|
||||||
|
export function intersectCompareColumns(sourceColumns: Pick<ColumnInfo, "name">[], targetColumns: Pick<ColumnInfo, "name">[]): CompareColumnIntersection {
|
||||||
|
const targetNamesByLowerName = new Map<string, string>();
|
||||||
|
for (const column of targetColumns) {
|
||||||
|
const lowerName = column.name.toLowerCase();
|
||||||
|
if (!targetNamesByLowerName.has(lowerName)) targetNamesByLowerName.set(lowerName, column.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
const columns: string[] = [];
|
||||||
|
const sourceNames: string[] = [];
|
||||||
|
const matched = new Set<string>();
|
||||||
|
for (const column of sourceColumns) {
|
||||||
|
const lowerName = column.name.toLowerCase();
|
||||||
|
const targetName = targetNamesByLowerName.get(lowerName);
|
||||||
|
if (targetName === undefined || matched.has(lowerName)) continue;
|
||||||
|
matched.add(lowerName);
|
||||||
|
columns.push(targetName);
|
||||||
|
sourceNames.push(column.name);
|
||||||
|
}
|
||||||
|
return { columns, sourceColumns: sourceNames };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves a column name to the matching entry of `columns`, ignoring case.
|
||||||
|
* Returns undefined when no column matches.
|
||||||
|
*/
|
||||||
|
export function matchColumnNameIgnoreCase(name: string, columns: string[]): string | undefined {
|
||||||
|
const lowerName = name.toLowerCase();
|
||||||
|
return columns.find((column) => column.toLowerCase() === lowerName);
|
||||||
|
}
|
||||||
@@ -62,6 +62,8 @@ pub struct DataCompareFromTablesOptions {
|
|||||||
pub columns: Vec<String>,
|
pub columns: Vec<String>,
|
||||||
pub key_columns: Vec<String>,
|
pub key_columns: Vec<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub source_columns: Option<Vec<String>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub fetch_batch_size: Option<usize>,
|
pub fetch_batch_size: Option<usize>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub degradation_threshold: Option<DegradationThreshold>,
|
pub degradation_threshold: Option<DegradationThreshold>,
|
||||||
@@ -356,6 +358,11 @@ pub async fn prepare_data_compare_from_tables(
|
|||||||
let sampling_strategy = options.sampling_strategy.clone().unwrap_or(SamplingStrategy::Hybrid);
|
let sampling_strategy = options.sampling_strategy.clone().unwrap_or(SamplingStrategy::Hybrid);
|
||||||
let enable_checksum = options.enable_checksum.unwrap_or(true);
|
let enable_checksum = options.enable_checksum.unwrap_or(true);
|
||||||
|
|
||||||
|
// Cross-database comparison matches columns case-insensitively, so `columns` holds
|
||||||
|
// target-side names while the source side needs its own names in the same order.
|
||||||
|
let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref());
|
||||||
|
let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns);
|
||||||
|
|
||||||
let (source_rows, target_rows, sampling_rate, verification_method) = match °radation_level {
|
let (source_rows, target_rows, sampling_rate, verification_method) = match °radation_level {
|
||||||
DegradationLevel::Full => {
|
DegradationLevel::Full => {
|
||||||
let (src, tgt) = tokio::try_join!(
|
let (src, tgt) = tokio::try_join!(
|
||||||
@@ -365,8 +372,8 @@ pub async fn prepare_data_compare_from_tables(
|
|||||||
&options.source_database,
|
&options.source_database,
|
||||||
&options.source_schema,
|
&options.source_schema,
|
||||||
&options.source_table,
|
&options.source_table,
|
||||||
&options.columns,
|
&source_column_names,
|
||||||
&options.key_columns,
|
&source_key_columns,
|
||||||
source_database_type,
|
source_database_type,
|
||||||
fetch_batch_size,
|
fetch_batch_size,
|
||||||
),
|
),
|
||||||
@@ -392,8 +399,8 @@ pub async fn prepare_data_compare_from_tables(
|
|||||||
&options.source_database,
|
&options.source_database,
|
||||||
&options.source_schema,
|
&options.source_schema,
|
||||||
&options.source_table,
|
&options.source_table,
|
||||||
&options.columns,
|
&source_column_names,
|
||||||
&options.key_columns,
|
&source_key_columns,
|
||||||
source_database_type,
|
source_database_type,
|
||||||
&sampling_strategy,
|
&sampling_strategy,
|
||||||
threshold.sample_size,
|
threshold.sample_size,
|
||||||
@@ -1046,6 +1053,27 @@ fn first_count(rows: &[Vec<Value>]) -> Result<u64, String> {
|
|||||||
.ok_or_else(|| format!("COUNT query returned non-numeric value: {value}"))
|
.ok_or_else(|| format!("COUNT query returned non-numeric value: {value}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Data compare matches columns case-insensitively across databases that store
|
||||||
|
/// identifiers with different case conventions (e.g. SQL Server keeps the created
|
||||||
|
/// case while Oracle stores upper case). `columns` carries target-side names, so
|
||||||
|
/// the source side is queried with its own names in the same positional order.
|
||||||
|
fn aligned_source_column_names(columns: &[String], source_columns: Option<&Vec<String>>) -> Vec<String> {
|
||||||
|
match source_columns {
|
||||||
|
Some(source_columns) if source_columns.len() == columns.len() => source_columns.clone(),
|
||||||
|
_ => columns.to_vec(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn aligned_source_key_columns(columns: &[String], source_columns: &[String], key_columns: &[String]) -> Vec<String> {
|
||||||
|
key_columns
|
||||||
|
.iter()
|
||||||
|
.map(|key| match columns.iter().position(|column| column == key) {
|
||||||
|
Some(index) => source_columns[index].clone(),
|
||||||
|
None => key.clone(),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn build_data_compare_select_sql(
|
fn build_data_compare_select_sql(
|
||||||
database_type: DatabaseType,
|
database_type: DatabaseType,
|
||||||
schema: &str,
|
schema: &str,
|
||||||
@@ -1648,6 +1676,7 @@ pub async fn verify_data(state: &AppState, options: VerifyDataOptions) -> Result
|
|||||||
target_table: options.target_table,
|
target_table: options.target_table,
|
||||||
columns: options.columns,
|
columns: options.columns,
|
||||||
key_columns: options.key_columns,
|
key_columns: options.key_columns,
|
||||||
|
source_columns: None,
|
||||||
fetch_batch_size: options.fetch_batch_size,
|
fetch_batch_size: options.fetch_batch_size,
|
||||||
degradation_threshold: Some(degradation_threshold),
|
degradation_threshold: Some(degradation_threshold),
|
||||||
sampling_strategy: Some(sampling_strategy),
|
sampling_strategy: Some(sampling_strategy),
|
||||||
@@ -2531,4 +2560,31 @@ mod tests {
|
|||||||
let up = snapshot.iter().find(|e| e.name == "dbx_auto_upgrade_total").unwrap();
|
let up = snapshot.iter().find(|e| e.name == "dbx_auto_upgrade_total").unwrap();
|
||||||
assert_eq!(up.value, crate::risk_metrics::MetricValue::Counter(1));
|
assert_eq!(up.value, crate::risk_metrics::MetricValue::Counter(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aligned_source_column_names_keep_source_case_in_positional_order() {
|
||||||
|
let columns = vec!["SNID".to_string(), "NAME".to_string()];
|
||||||
|
let source_columns = vec!["snid".to_string(), "name".to_string()];
|
||||||
|
|
||||||
|
let aligned = aligned_source_column_names(&columns, Some(&source_columns));
|
||||||
|
assert_eq!(aligned, source_columns);
|
||||||
|
|
||||||
|
let fallback = aligned_source_column_names(&columns, None);
|
||||||
|
assert_eq!(fallback, columns);
|
||||||
|
|
||||||
|
let mismatched = aligned_source_column_names(&columns, Some(&vec!["snid".to_string()]));
|
||||||
|
assert_eq!(mismatched, columns);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aligned_source_key_columns_map_keys_by_position() {
|
||||||
|
let columns = vec!["SNID".to_string(), "NAME".to_string()];
|
||||||
|
let source_columns = vec!["snid".to_string(), "name".to_string()];
|
||||||
|
|
||||||
|
let keys = aligned_source_key_columns(&columns, &source_columns, &["NAME".to_string()]);
|
||||||
|
assert_eq!(keys, vec!["name".to_string()]);
|
||||||
|
|
||||||
|
let unknown = aligned_source_key_columns(&columns, &source_columns, &["missing".to_string()]);
|
||||||
|
assert_eq!(unknown, vec!["missing".to_string()]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user