fix(transfer): validate existing target before overwrite

This commit is contained in:
Melody Ruch authored and GitHub committed 2026-08-15 23:43:10 +08:00
1 parent 7f1a983498
commit e97a5a5b8f
2 files changed
+341 -22

No files matched your search

+190 -22
View File
@@ -1407,6 +1407,71 @@ fn writable_transfer_columns(
.collect()
}
fn transfer_column_names_match(target_db_type: &DatabaseType, left: &str, right: &str) -> bool {
if matches!(
target_db_type,
DatabaseType::Mysql
| DatabaseType::Goldendb
| DatabaseType::Sqlite
| DatabaseType::Rqlite
| DatabaseType::CloudflareD1
| DatabaseType::DuckDb
| DatabaseType::SqlServer
| DatabaseType::Doris
| DatabaseType::StarRocks
| DatabaseType::Hive
| DatabaseType::Kyuubi
| DatabaseType::Impala
| DatabaseType::Spark
| DatabaseType::Access
) {
left.eq_ignore_ascii_case(right)
} else {
left == right
}
}
fn missing_transfer_target_columns(
target_columns: &[db::ColumnInfo],
col_names: &[String],
target_db_type: &DatabaseType,
) -> Vec<String> {
col_names
.iter()
.filter(|name| {
!target_columns.iter().any(|column| transfer_column_names_match(target_db_type, name, &column.name))
})
.cloned()
.collect()
}
fn target_column_can_be_omitted(column: &db::ColumnInfo, target_db_type: &DatabaseType) -> bool {
let extra = column.extra.as_deref().unwrap_or_default().trim().to_ascii_lowercase();
column.is_nullable
|| column.column_default.as_deref().is_some_and(|value| !value.trim().is_empty())
|| extra.contains("generated")
|| extra.contains("identity")
|| extra.contains("auto_increment")
|| extra.contains("autoincrement")
|| extra.contains("computed")
|| (matches!(target_db_type, DatabaseType::SqlServer) && is_sqlserver_rowversion_type(&column.data_type))
}
fn required_unmapped_transfer_target_columns(
target_columns: &[db::ColumnInfo],
col_names: &[String],
target_db_type: &DatabaseType,
) -> Vec<String> {
target_columns
.iter()
.filter(|column| {
!target_column_can_be_omitted(column, target_db_type)
&& !col_names.iter().any(|name| transfer_column_names_match(target_db_type, name, &column.name))
})
.map(|column| column.name.clone())
.collect()
}
fn transfer_key_columns(columns: &[db::ColumnInfo], db_type: &DatabaseType) -> Vec<String> {
let uses_unique_key_model = matches!(db_type, DatabaseType::Doris | DatabaseType::StarRocks);
columns
@@ -6763,6 +6828,57 @@ where
return Ok(0);
}
let needs_target_columns = (request.create_table && target_table_preexisting)
|| (request.mode == TransferMode::Upsert
&& !matches!(
target_db_type,
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
))
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng);
let target_columns = if needs_target_columns {
get_columns_for_transfer(
state,
target_pool_key,
&request.target_connection_id,
&request.target_database,
&request.target_schema,
&target_table,
request.target_catalog.as_deref(),
)
.await
.map_err(|error| format!("Failed to inspect target table '{target_table}' columns before transfer: {error}"))?
} else {
Vec::new()
};
// The user asked DBX to sync structure (create_table), but the target
// table already existed so the create-table DDL above was skipped (see
// "skipping create-table DDL" above). If the untouched target structure
// can't accept the planned insert, fail fast here instead of truncating
// the target's existing data and then hitting an opaque driver error.
if request.create_table && target_table_preexisting {
let missing = missing_transfer_target_columns(&target_columns, &col_names, target_db_type);
if !missing.is_empty() {
return Err(format!(
"Target table '{target_table}' already exists with a different structure and is missing column(s) \
{} present in the source table. DBX does not alter an existing target table's columns during \
transfer — drop the target table or adjust its structure to match the source first.",
missing.join(", ")
));
}
let required = required_unmapped_transfer_target_columns(&target_columns, &col_names, target_db_type);
if !required.is_empty() {
return Err(format!(
"Target table '{target_table}' already exists with a different structure and has required column(s) \
{} that are not present in the source table and have no default or generated value. DBX does not \
alter an existing target table's columns during transfer — drop the target table or adjust its \
structure to match the source first.",
required.join(", ")
));
}
}
// Truncate target if overwrite mode
if request.mode == TransferMode::Overwrite {
let full_table =
@@ -6776,28 +6892,6 @@ where
execute_on_pool(state, target_pool_key, &truncate_sql).await.map_err(|e| format!("Failed to truncate: {e}"))?;
}
let target_columns = if (request.mode == TransferMode::Upsert
&& !matches!(
target_db_type,
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
))
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
{
get_columns_for_transfer(
state,
target_pool_key,
&request.target_connection_id,
&request.target_database,
&request.target_schema,
&target_table,
request.target_catalog.as_deref(),
)
.await
.unwrap_or_default()
} else {
Vec::new()
};
// Determine effective mode and PK columns for upsert
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
if matches!(
@@ -8733,6 +8827,80 @@ mod tests {
assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id", "updated_at"]);
}
#[test]
fn transfer_target_column_validation_reports_columns_absent_from_target() {
let target_columns = vec![test_column("id", "int"), test_column("name", "varchar(32)")];
let col_names = vec!["id".to_string(), "name".to_string(), "extra_col".to_string()];
assert_eq!(
missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql),
vec!["extra_col".to_string()]
);
}
#[test]
fn transfer_target_column_validation_uses_database_case_rules() {
let target_columns = vec![test_column("ID", "int"), test_column("Name", "varchar(32)")];
let col_names = vec!["id".to_string(), "name".to_string()];
assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql).is_empty());
assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Kyuubi).is_empty());
assert_eq!(
missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Postgres),
vec!["id".to_string(), "name".to_string()]
);
}
#[test]
fn transfer_target_column_validation_allows_omittable_target_columns() {
let target_columns = vec![
test_column("id", "int"),
test_column("nullable_note", "varchar(32)"),
db::ColumnInfo {
is_nullable: false,
column_default: Some("CURRENT_TIMESTAMP".to_string()),
..test_column("created_at", "timestamp")
},
db::ColumnInfo {
is_nullable: false,
extra: Some("generated always as (id + 1) stored".to_string()),
..test_column("generated_id", "int")
},
db::ColumnInfo {
is_nullable: false,
extra: Some("identity(1,1)".to_string()),
..test_column("sequence_id", "bigint")
},
db::ColumnInfo {
is_nullable: false,
extra: Some("computed".to_string()),
..test_column("computed_id", "int")
},
db::ColumnInfo { is_nullable: false, ..test_column("row_version", "rowversion") },
];
let col_names = vec!["id".to_string()];
assert!(required_unmapped_transfer_target_columns(&target_columns[..6], &col_names, &DatabaseType::Mysql)
.is_empty());
assert!(
required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::SqlServer).is_empty()
);
}
#[test]
fn transfer_target_column_validation_rejects_required_unmapped_columns() {
let target_columns = vec![
test_column("id", "int"),
db::ColumnInfo { is_nullable: false, ..test_column("required_code", "varchar(32)") },
];
let col_names = vec!["id".to_string()];
assert_eq!(
required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql),
vec!["required_code".to_string()]
);
}
#[test]
fn dameng_identity_insert_wrapper_quotes_schema_and_table() {
let sql = wrap_dameng_identity_insert_sql(