fix(transfer): validate existing target before overwrite
This commit is contained in:
1 parent
7f1a983498
commit
e97a5a5b8f
2 files changed
+341
-22
No files matched your search
+190
-22
@@ -1407,6 +1407,71 @@ fn writable_transfer_columns(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn transfer_column_names_match(target_db_type: &DatabaseType, left: &str, right: &str) -> bool {
|
||||
if matches!(
|
||||
target_db_type,
|
||||
DatabaseType::Mysql
|
||||
| DatabaseType::Goldendb
|
||||
| DatabaseType::Sqlite
|
||||
| DatabaseType::Rqlite
|
||||
| DatabaseType::CloudflareD1
|
||||
| DatabaseType::DuckDb
|
||||
| DatabaseType::SqlServer
|
||||
| DatabaseType::Doris
|
||||
| DatabaseType::StarRocks
|
||||
| DatabaseType::Hive
|
||||
| DatabaseType::Kyuubi
|
||||
| DatabaseType::Impala
|
||||
| DatabaseType::Spark
|
||||
| DatabaseType::Access
|
||||
) {
|
||||
left.eq_ignore_ascii_case(right)
|
||||
} else {
|
||||
left == right
|
||||
}
|
||||
}
|
||||
|
||||
fn missing_transfer_target_columns(
|
||||
target_columns: &[db::ColumnInfo],
|
||||
col_names: &[String],
|
||||
target_db_type: &DatabaseType,
|
||||
) -> Vec<String> {
|
||||
col_names
|
||||
.iter()
|
||||
.filter(|name| {
|
||||
!target_columns.iter().any(|column| transfer_column_names_match(target_db_type, name, &column.name))
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn target_column_can_be_omitted(column: &db::ColumnInfo, target_db_type: &DatabaseType) -> bool {
|
||||
let extra = column.extra.as_deref().unwrap_or_default().trim().to_ascii_lowercase();
|
||||
column.is_nullable
|
||||
|| column.column_default.as_deref().is_some_and(|value| !value.trim().is_empty())
|
||||
|| extra.contains("generated")
|
||||
|| extra.contains("identity")
|
||||
|| extra.contains("auto_increment")
|
||||
|| extra.contains("autoincrement")
|
||||
|| extra.contains("computed")
|
||||
|| (matches!(target_db_type, DatabaseType::SqlServer) && is_sqlserver_rowversion_type(&column.data_type))
|
||||
}
|
||||
|
||||
fn required_unmapped_transfer_target_columns(
|
||||
target_columns: &[db::ColumnInfo],
|
||||
col_names: &[String],
|
||||
target_db_type: &DatabaseType,
|
||||
) -> Vec<String> {
|
||||
target_columns
|
||||
.iter()
|
||||
.filter(|column| {
|
||||
!target_column_can_be_omitted(column, target_db_type)
|
||||
&& !col_names.iter().any(|name| transfer_column_names_match(target_db_type, name, &column.name))
|
||||
})
|
||||
.map(|column| column.name.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn transfer_key_columns(columns: &[db::ColumnInfo], db_type: &DatabaseType) -> Vec<String> {
|
||||
let uses_unique_key_model = matches!(db_type, DatabaseType::Doris | DatabaseType::StarRocks);
|
||||
columns
|
||||
@@ -6763,6 +6828,57 @@ where
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let needs_target_columns = (request.create_table && target_table_preexisting)
|
||||
|| (request.mode == TransferMode::Upsert
|
||||
&& !matches!(
|
||||
target_db_type,
|
||||
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
|
||||
))
|
||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng);
|
||||
let target_columns = if needs_target_columns {
|
||||
get_columns_for_transfer(
|
||||
state,
|
||||
target_pool_key,
|
||||
&request.target_connection_id,
|
||||
&request.target_database,
|
||||
&request.target_schema,
|
||||
&target_table,
|
||||
request.target_catalog.as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| format!("Failed to inspect target table '{target_table}' columns before transfer: {error}"))?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// The user asked DBX to sync structure (create_table), but the target
|
||||
// table already existed so the create-table DDL above was skipped (see
|
||||
// "skipping create-table DDL" above). If the untouched target structure
|
||||
// can't accept the planned insert, fail fast here instead of truncating
|
||||
// the target's existing data and then hitting an opaque driver error.
|
||||
if request.create_table && target_table_preexisting {
|
||||
let missing = missing_transfer_target_columns(&target_columns, &col_names, target_db_type);
|
||||
if !missing.is_empty() {
|
||||
return Err(format!(
|
||||
"Target table '{target_table}' already exists with a different structure and is missing column(s) \
|
||||
{} present in the source table. DBX does not alter an existing target table's columns during \
|
||||
transfer — drop the target table or adjust its structure to match the source first.",
|
||||
missing.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
let required = required_unmapped_transfer_target_columns(&target_columns, &col_names, target_db_type);
|
||||
if !required.is_empty() {
|
||||
return Err(format!(
|
||||
"Target table '{target_table}' already exists with a different structure and has required column(s) \
|
||||
{} that are not present in the source table and have no default or generated value. DBX does not \
|
||||
alter an existing target table's columns during transfer — drop the target table or adjust its \
|
||||
structure to match the source first.",
|
||||
required.join(", ")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate target if overwrite mode
|
||||
if request.mode == TransferMode::Overwrite {
|
||||
let full_table =
|
||||
@@ -6776,28 +6892,6 @@ where
|
||||
execute_on_pool(state, target_pool_key, &truncate_sql).await.map_err(|e| format!("Failed to truncate: {e}"))?;
|
||||
}
|
||||
|
||||
let target_columns = if (request.mode == TransferMode::Upsert
|
||||
&& !matches!(
|
||||
target_db_type,
|
||||
DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala
|
||||
))
|
||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng)
|
||||
{
|
||||
get_columns_for_transfer(
|
||||
state,
|
||||
target_pool_key,
|
||||
&request.target_connection_id,
|
||||
&request.target_database,
|
||||
&request.target_schema,
|
||||
&target_table,
|
||||
request.target_catalog.as_deref(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Determine effective mode and PK columns for upsert
|
||||
let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert {
|
||||
if matches!(
|
||||
@@ -8733,6 +8827,80 @@ mod tests {
|
||||
assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id", "updated_at"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_target_column_validation_reports_columns_absent_from_target() {
|
||||
let target_columns = vec![test_column("id", "int"), test_column("name", "varchar(32)")];
|
||||
let col_names = vec!["id".to_string(), "name".to_string(), "extra_col".to_string()];
|
||||
|
||||
assert_eq!(
|
||||
missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql),
|
||||
vec!["extra_col".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_target_column_validation_uses_database_case_rules() {
|
||||
let target_columns = vec![test_column("ID", "int"), test_column("Name", "varchar(32)")];
|
||||
let col_names = vec!["id".to_string(), "name".to_string()];
|
||||
|
||||
assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql).is_empty());
|
||||
assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Kyuubi).is_empty());
|
||||
assert_eq!(
|
||||
missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Postgres),
|
||||
vec!["id".to_string(), "name".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_target_column_validation_allows_omittable_target_columns() {
|
||||
let target_columns = vec![
|
||||
test_column("id", "int"),
|
||||
test_column("nullable_note", "varchar(32)"),
|
||||
db::ColumnInfo {
|
||||
is_nullable: false,
|
||||
column_default: Some("CURRENT_TIMESTAMP".to_string()),
|
||||
..test_column("created_at", "timestamp")
|
||||
},
|
||||
db::ColumnInfo {
|
||||
is_nullable: false,
|
||||
extra: Some("generated always as (id + 1) stored".to_string()),
|
||||
..test_column("generated_id", "int")
|
||||
},
|
||||
db::ColumnInfo {
|
||||
is_nullable: false,
|
||||
extra: Some("identity(1,1)".to_string()),
|
||||
..test_column("sequence_id", "bigint")
|
||||
},
|
||||
db::ColumnInfo {
|
||||
is_nullable: false,
|
||||
extra: Some("computed".to_string()),
|
||||
..test_column("computed_id", "int")
|
||||
},
|
||||
db::ColumnInfo { is_nullable: false, ..test_column("row_version", "rowversion") },
|
||||
];
|
||||
let col_names = vec!["id".to_string()];
|
||||
|
||||
assert!(required_unmapped_transfer_target_columns(&target_columns[..6], &col_names, &DatabaseType::Mysql)
|
||||
.is_empty());
|
||||
assert!(
|
||||
required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::SqlServer).is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_target_column_validation_rejects_required_unmapped_columns() {
|
||||
let target_columns = vec![
|
||||
test_column("id", "int"),
|
||||
db::ColumnInfo { is_nullable: false, ..test_column("required_code", "varchar(32)") },
|
||||
];
|
||||
let col_names = vec!["id".to_string()];
|
||||
|
||||
assert_eq!(
|
||||
required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql),
|
||||
vec!["required_code".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dameng_identity_insert_wrapper_quotes_schema_and_table() {
|
||||
let sql = wrap_dameng_identity_insert_sql(
|
||||
|
||||
Reference in new issue
Block a user