diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 4b2193553f..92d4343e5c 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -1407,6 +1407,71 @@ fn writable_transfer_columns( .collect() } +fn transfer_column_names_match(target_db_type: &DatabaseType, left: &str, right: &str) -> bool { + if matches!( + target_db_type, + DatabaseType::Mysql + | DatabaseType::Goldendb + | DatabaseType::Sqlite + | DatabaseType::Rqlite + | DatabaseType::CloudflareD1 + | DatabaseType::DuckDb + | DatabaseType::SqlServer + | DatabaseType::Doris + | DatabaseType::StarRocks + | DatabaseType::Hive + | DatabaseType::Kyuubi + | DatabaseType::Impala + | DatabaseType::Spark + | DatabaseType::Access + ) { + left.eq_ignore_ascii_case(right) + } else { + left == right + } +} + +fn missing_transfer_target_columns( + target_columns: &[db::ColumnInfo], + col_names: &[String], + target_db_type: &DatabaseType, +) -> Vec { + col_names + .iter() + .filter(|name| { + !target_columns.iter().any(|column| transfer_column_names_match(target_db_type, name, &column.name)) + }) + .cloned() + .collect() +} + +fn target_column_can_be_omitted(column: &db::ColumnInfo, target_db_type: &DatabaseType) -> bool { + let extra = column.extra.as_deref().unwrap_or_default().trim().to_ascii_lowercase(); + column.is_nullable + || column.column_default.as_deref().is_some_and(|value| !value.trim().is_empty()) + || extra.contains("generated") + || extra.contains("identity") + || extra.contains("auto_increment") + || extra.contains("autoincrement") + || extra.contains("computed") + || (matches!(target_db_type, DatabaseType::SqlServer) && is_sqlserver_rowversion_type(&column.data_type)) +} + +fn required_unmapped_transfer_target_columns( + target_columns: &[db::ColumnInfo], + col_names: &[String], + target_db_type: &DatabaseType, +) -> Vec { + target_columns + .iter() + .filter(|column| { + !target_column_can_be_omitted(column, target_db_type) + && !col_names.iter().any(|name| transfer_column_names_match(target_db_type, name, &column.name)) + }) + .map(|column| column.name.clone()) + .collect() +} + fn transfer_key_columns(columns: &[db::ColumnInfo], db_type: &DatabaseType) -> Vec { let uses_unique_key_model = matches!(db_type, DatabaseType::Doris | DatabaseType::StarRocks); columns @@ -6763,6 +6828,57 @@ where return Ok(0); } + let needs_target_columns = (request.create_table && target_table_preexisting) + || (request.mode == TransferMode::Upsert + && !matches!( + target_db_type, + DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala + )) + || matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng); + let target_columns = if needs_target_columns { + get_columns_for_transfer( + state, + target_pool_key, + &request.target_connection_id, + &request.target_database, + &request.target_schema, + &target_table, + request.target_catalog.as_deref(), + ) + .await + .map_err(|error| format!("Failed to inspect target table '{target_table}' columns before transfer: {error}"))? + } else { + Vec::new() + }; + + // The user asked DBX to sync structure (create_table), but the target + // table already existed so the create-table DDL above was skipped (see + // "skipping create-table DDL" above). If the untouched target structure + // can't accept the planned insert, fail fast here instead of truncating + // the target's existing data and then hitting an opaque driver error. + if request.create_table && target_table_preexisting { + let missing = missing_transfer_target_columns(&target_columns, &col_names, target_db_type); + if !missing.is_empty() { + return Err(format!( + "Target table '{target_table}' already exists with a different structure and is missing column(s) \ + {} present in the source table. DBX does not alter an existing target table's columns during \ + transfer — drop the target table or adjust its structure to match the source first.", + missing.join(", ") + )); + } + + let required = required_unmapped_transfer_target_columns(&target_columns, &col_names, target_db_type); + if !required.is_empty() { + return Err(format!( + "Target table '{target_table}' already exists with a different structure and has required column(s) \ + {} that are not present in the source table and have no default or generated value. DBX does not \ + alter an existing target table's columns during transfer — drop the target table or adjust its \ + structure to match the source first.", + required.join(", ") + )); + } + } + // Truncate target if overwrite mode if request.mode == TransferMode::Overwrite { let full_table = @@ -6776,28 +6892,6 @@ where execute_on_pool(state, target_pool_key, &truncate_sql).await.map_err(|e| format!("Failed to truncate: {e}"))?; } - let target_columns = if (request.mode == TransferMode::Upsert - && !matches!( - target_db_type, - DatabaseType::ClickHouse | DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala - )) - || matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng) - { - get_columns_for_transfer( - state, - target_pool_key, - &request.target_connection_id, - &request.target_database, - &request.target_schema, - &target_table, - request.target_catalog.as_deref(), - ) - .await - .unwrap_or_default() - } else { - Vec::new() - }; - // Determine effective mode and PK columns for upsert let (effective_mode, pk_columns) = if request.mode == TransferMode::Upsert { if matches!( @@ -8733,6 +8827,80 @@ mod tests { assert_eq!(writable.iter().map(|column| column.name.as_str()).collect::>(), vec!["id", "updated_at"]); } + #[test] + fn transfer_target_column_validation_reports_columns_absent_from_target() { + let target_columns = vec![test_column("id", "int"), test_column("name", "varchar(32)")]; + let col_names = vec!["id".to_string(), "name".to_string(), "extra_col".to_string()]; + + assert_eq!( + missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql), + vec!["extra_col".to_string()] + ); + } + + #[test] + fn transfer_target_column_validation_uses_database_case_rules() { + let target_columns = vec![test_column("ID", "int"), test_column("Name", "varchar(32)")]; + let col_names = vec!["id".to_string(), "name".to_string()]; + + assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql).is_empty()); + assert!(missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Kyuubi).is_empty()); + assert_eq!( + missing_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Postgres), + vec!["id".to_string(), "name".to_string()] + ); + } + + #[test] + fn transfer_target_column_validation_allows_omittable_target_columns() { + let target_columns = vec![ + test_column("id", "int"), + test_column("nullable_note", "varchar(32)"), + db::ColumnInfo { + is_nullable: false, + column_default: Some("CURRENT_TIMESTAMP".to_string()), + ..test_column("created_at", "timestamp") + }, + db::ColumnInfo { + is_nullable: false, + extra: Some("generated always as (id + 1) stored".to_string()), + ..test_column("generated_id", "int") + }, + db::ColumnInfo { + is_nullable: false, + extra: Some("identity(1,1)".to_string()), + ..test_column("sequence_id", "bigint") + }, + db::ColumnInfo { + is_nullable: false, + extra: Some("computed".to_string()), + ..test_column("computed_id", "int") + }, + db::ColumnInfo { is_nullable: false, ..test_column("row_version", "rowversion") }, + ]; + let col_names = vec!["id".to_string()]; + + assert!(required_unmapped_transfer_target_columns(&target_columns[..6], &col_names, &DatabaseType::Mysql) + .is_empty()); + assert!( + required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::SqlServer).is_empty() + ); + } + + #[test] + fn transfer_target_column_validation_rejects_required_unmapped_columns() { + let target_columns = vec![ + test_column("id", "int"), + db::ColumnInfo { is_nullable: false, ..test_column("required_code", "varchar(32)") }, + ]; + let col_names = vec!["id".to_string()]; + + assert_eq!( + required_unmapped_transfer_target_columns(&target_columns, &col_names, &DatabaseType::Mysql), + vec!["required_code".to_string()] + ); + } + #[test] fn dameng_identity_insert_wrapper_quotes_schema_and_table() { let sql = wrap_dameng_identity_insert_sql( diff --git a/crates/dbx-core/tests/live_mysql_transfer.rs b/crates/dbx-core/tests/live_mysql_transfer.rs index 9482d224e7..e4a74dbc52 100644 --- a/crates/dbx-core/tests/live_mysql_transfer.rs +++ b/crates/dbx-core/tests/live_mysql_transfer.rs @@ -457,3 +457,154 @@ async fn live_mysql_transfer_preserves_spatial_values_and_modes() { cleanup.unwrap(); test_result.unwrap(); } + +#[tokio::test] +#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"] +async fn live_mysql_transfer_structure_overwrite_rejects_incompatible_target_columns() { + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let connection_id = format!("live-mysql-transfer-struct-{suffix}"); + let source_database = format!("dbx_transfer_struct_src_{}", &suffix[..12]); + let target_database = format!("dbx_transfer_struct_dst_{}", &suffix[..12]); + let config = live_mysql_config(&connection_id); + let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap(); + + let setup = format!( + "CREATE DATABASE `{source_database}`;\ + CREATE DATABASE `{target_database}`;\ + CREATE TABLE `{source_database}`.`orders` (\ + id INT PRIMARY KEY, name VARCHAR(32), extra_col VARCHAR(32)\ + );\ + INSERT INTO `{source_database}`.`orders` VALUES (1, 'alpha', 'x');\ + CREATE TABLE `{target_database}`.`orders` (\ + id INT PRIMARY KEY, name VARCHAR(32)\ + );\ + INSERT INTO `{target_database}`.`orders` (id, name) VALUES (99, 'stale');\ + CREATE TABLE `{source_database}`.`required_orders` (\ + id INT PRIMARY KEY, name VARCHAR(32)\ + );\ + INSERT INTO `{source_database}`.`required_orders` VALUES (1, 'alpha');\ + CREATE TABLE `{target_database}`.`required_orders` (\ + id INT PRIMARY KEY, name VARCHAR(32), required_code VARCHAR(32) NOT NULL\ + );\ + INSERT INTO `{target_database}`.`required_orders` (id, name, required_code) VALUES (99, 'stale', 'keep')" + ); + mysql::execute_query(&setup_pool, &setup, true).await.unwrap(); + + let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-struct-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + state.configs.write().await.insert(connection_id.clone(), config); + let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap(); + let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap(); + + // Mirrors the UI: "structure + data" content always requests create_table, + // and the reporter picked overwrite mode. The target table already exists + // with a column that's missing from the source ("orders" lacks extra_col). + let request = TransferRequest { + transfer_id: format!("live-mysql-transfer-struct-overwrite-{suffix}"), + source_connection_id: connection_id.clone(), + source_database: source_database.clone(), + source_schema: source_database.clone(), + source_catalog: None, + target_connection_id: connection_id.clone(), + target_database: target_database.clone(), + target_schema: target_database.clone(), + target_catalog: None, + tables: vec!["orders".to_string()], + create_table: true, + content: TransferContent::default(), + objects: Vec::new(), + mode: TransferMode::Overwrite, + target_table_name_case: TransferTableNameCase::Preserve, + ownership_policy: TransferOwnershipPolicy::Preserve, + batch_size: 10, + }; + + let test_result = async { + let result = transfer_table( + &state, + &request, + "orders", + 0, + &DatabaseType::Mysql, + &DatabaseType::Mysql, + &source_pool_key, + &target_pool_key, + |_| {}, + ) + .await; + + let error = result.expect_err("expected transfer to reject the incompatible target structure"); + assert!(error.contains("extra_col"), "unexpected missing-source-column error: {error}"); + + // The pre-existing target row must survive: a column-mismatch error must + // be raised BEFORE the destructive TRUNCATE, not surface only after the + // target has already been wiped by an insert that was doomed to fail. + assert_eq!( + query_text( + &setup_pool, + &format!("SELECT CAST(COUNT(*) AS CHAR) FROM `{target_database}`.`orders` WHERE id=99"), + ) + .await, + "1", + "target row was destroyed by TRUNCATE despite the transfer failing" + ); + + let required_request = TransferRequest { + transfer_id: format!("live-mysql-transfer-required-target-overwrite-{suffix}"), + source_connection_id: connection_id.clone(), + source_database: source_database.clone(), + source_schema: source_database.clone(), + source_catalog: None, + target_connection_id: connection_id.clone(), + target_database: target_database.clone(), + target_schema: target_database.clone(), + target_catalog: None, + tables: vec!["required_orders".to_string()], + create_table: true, + content: TransferContent::default(), + objects: Vec::new(), + mode: TransferMode::Overwrite, + target_table_name_case: TransferTableNameCase::Preserve, + ownership_policy: TransferOwnershipPolicy::Preserve, + batch_size: 10, + }; + let required_result = transfer_table( + &state, + &required_request, + "required_orders", + 0, + &DatabaseType::Mysql, + &DatabaseType::Mysql, + &source_pool_key, + &target_pool_key, + |_| {}, + ) + .await; + let required_error = required_result.expect_err("expected transfer to reject a required target-only column"); + assert!(required_error.contains("required_code"), "unexpected required-target-column error: {required_error}"); + assert_eq!( + query_text( + &setup_pool, + &format!("SELECT CAST(COUNT(*) AS CHAR) FROM `{target_database}`.`required_orders` WHERE id=99"), + ) + .await, + "1", + "target row was destroyed by TRUNCATE despite the required target column mismatch" + ); + Ok::<_, String>(()) + } + .await; + + let cleanup = mysql::execute_query( + &setup_pool, + &format!("DROP DATABASE `{source_database}`; DROP DATABASE `{target_database}`"), + true, + ) + .await; + setup_pool.disconnect().await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + cleanup.unwrap(); + test_result.unwrap(); +}