diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 016fe9b8bc..da64c695bf 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -26,6 +26,13 @@ static OCEANBASE_MYSQL_TABLE_OPTION_RE: std::sync::LazyLock = std::sync:: static MYSQL_COLLATE_CLAUSE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { Regex::new(r"(?i)\bCOLLATE\s*=?\s*([A-Za-z0-9_]+)\b").expect("valid MySQL COLLATE clause regex") }); +// An inline FK constraint *definition line*: an optional `CONSTRAINT ` +// prefix followed by `FOREIGN KEY (`. Anchored to the line start so column +// definitions whose COMMENT/DEFAULT strings mention "foreign key" never match. +static INLINE_FOREIGN_KEY_CONSTRAINT_LINE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + Regex::new(r#"(?i)^\s*(?:CONSTRAINT\s+(?:`[^`]*`|"[^"]*"|\S+)\s+)?FOREIGN\s+KEY\s*\("#) + .expect("valid inline foreign key constraint line regex") +}); const MAX_TRANSFER_WRITE_SQL_BYTES: usize = 512 * 1024; const MAX_SQLSERVER_INSERT_ROWS: usize = 1000; @@ -1579,6 +1586,42 @@ fn required_unmapped_transfer_target_columns( .collect() } +/// Fails fast when a preexisting target table's structure can't accept the +/// source columns. DBX never alters an existing target table's columns, so +/// skipping this check (structure-only transfers used to silently skip it, +/// #7660) leaves the transfer reporting success while the target quietly stays +/// out of sync with the source. +fn validate_preexisting_target_columns( + target_columns: &[db::ColumnInfo], + col_names: &[String], + target_db_type: &DatabaseType, + quote_target_column_names: bool, + target_table: &str, +) -> Result<(), String> { + let missing = missing_transfer_target_columns(target_columns, col_names, target_db_type, quote_target_column_names); + if !missing.is_empty() { + return Err(format!( + "Target table '{target_table}' already exists with a different structure and is missing column(s) \ + {} present in the source table. DBX does not alter an existing target table's columns during \ + transfer — drop the target table or adjust its structure to match the source first.", + missing.join(", ") + )); + } + + let required = + required_unmapped_transfer_target_columns(target_columns, col_names, target_db_type, quote_target_column_names); + if !required.is_empty() { + return Err(format!( + "Target table '{target_table}' already exists with a different structure and has required column(s) \ + {} that are not present in the source table and have no default or generated value. DBX does not \ + alter an existing target table's columns during transfer — drop the target table or adjust its \ + structure to match the source first.", + required.join(", ") + )); + } + Ok(()) +} + fn transfer_key_columns(columns: &[db::ColumnInfo], db_type: &DatabaseType) -> Vec { let uses_unique_key_model = matches!(db_type, DatabaseType::Doris | DatabaseType::StarRocks); columns @@ -4866,6 +4909,10 @@ fn transfer_ddl_statements(sql: &str, db_type: &DatabaseType) -> Vec { /// text, which both Postgres and MySQL-family `SHOW CREATE TABLE` output share, /// and on foreign key constraints always being the last items before the closing /// paren (true for both dialects' DDL dumps). +/// +/// Only genuine constraint definition lines match (`CONSTRAINT FOREIGN +/// KEY (` / bare `FOREIGN KEY (`); a column line whose COMMENT or DEFAULT text +/// merely mentions the words must survive (#7660). fn strip_inline_foreign_key_constraint_lines(statement: &str) -> String { if !statement.trim_start().to_ascii_uppercase().starts_with("CREATE TABLE ") { return statement.to_string(); @@ -4873,7 +4920,7 @@ fn strip_inline_foreign_key_constraint_lines(statement: &str) -> String { let mut lines: Vec = Vec::new(); for line in statement.lines() { - if line.to_ascii_uppercase().contains(" FOREIGN KEY ") { + if INLINE_FOREIGN_KEY_CONSTRAINT_LINE_RE.is_match(line) { if let Some(previous) = lines.last_mut() { let trimmed_len = previous.trim_end_matches(char::is_whitespace).len(); if previous[..trimmed_len].ends_with(',') { @@ -7540,6 +7587,28 @@ where // Structure-only transfer: complete the table's post-create schema DDL, // then skip everything data-related. if !should_copy_data(&request.content) { + if request.create_table && target_table_preexisting { + let target_columns = get_columns_for_transfer( + state, + target_pool_key, + &request.target_connection_id, + &request.target_database, + &request.target_schema, + &target_table, + request.target_catalog.as_deref(), + ) + .await + .map_err(|error| { + format!("Failed to inspect target table '{target_table}' columns before transfer: {error}") + })?; + validate_preexisting_target_columns( + &target_columns, + &col_names, + target_db_type, + request.quote_target_column_names, + &target_table, + )?; + } if should_restore_postgres_table_schema { restore_postgres_table_schema_objects( state, @@ -7588,36 +7657,13 @@ where // can't accept the planned insert, fail fast here instead of truncating // the target's existing data and then hitting an opaque driver error. if request.create_table && target_table_preexisting { - let missing = missing_transfer_target_columns( + validate_preexisting_target_columns( &target_columns, &col_names, target_db_type, request.quote_target_column_names, - ); - if !missing.is_empty() { - return Err(format!( - "Target table '{target_table}' already exists with a different structure and is missing column(s) \ - {} present in the source table. DBX does not alter an existing target table's columns during \ - transfer — drop the target table or adjust its structure to match the source first.", - missing.join(", ") - )); - } - - let required = required_unmapped_transfer_target_columns( - &target_columns, - &col_names, - target_db_type, - request.quote_target_column_names, - ); - if !required.is_empty() { - return Err(format!( - "Target table '{target_table}' already exists with a different structure and has required column(s) \ - {} that are not present in the source table and have no default or generated value. DBX does not \ - alter an existing target table's columns during transfer — drop the target table or adjust its \ - structure to match the source first.", - required.join(", ") - )); - } + &target_table, + )?; } // Truncate target if overwrite mode @@ -9098,6 +9144,35 @@ mod tests { assert!(stripped.contains("KEY `fk_child_parent` (`parent_id`)"), "{stripped}"); } + #[test] + fn keeps_columns_whose_comment_mentions_foreign_key() { + // Regression for #7660: the deferred-FK DDL rewrite used to drop any + // line containing " FOREIGN KEY ", so column definitions whose + // COMMENT text merely mentions the words silently vanished from the + // created target table. + let ddl = "CREATE TABLE `title_task` (\n `id` varchar(32) NOT NULL,\n `review_result` varchar(100) DEFAULT NULL,\n `review_mode` tinyint NOT NULL DEFAULT '0' COMMENT 'foreign key of review flow',\n `score_snapshot_json` text COMMENT 'snapshot json, see foreign key docs',\n `task_status` int DEFAULT '0',\n PRIMARY KEY (`id`),\n KEY `idx_task_status` (`task_status`),\n CONSTRAINT `fk_title_task_org` FOREIGN KEY (`org_id`) REFERENCES `org` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"; + + let stripped = strip_inline_foreign_key_constraint_lines(ddl); + + assert!(stripped.contains("`review_mode` tinyint NOT NULL DEFAULT '0'"), "{stripped}"); + assert!(stripped.contains("`score_snapshot_json` text"), "{stripped}"); + assert!(!stripped.contains("FOREIGN KEY"), "{stripped}"); + assert!(stripped.contains("KEY `idx_task_status` (`task_status`)"), "{stripped}"); + } + + #[test] + fn strips_only_foreign_key_constraint_lines_regardless_of_other_mentions() { + // A CHECK constraint whose expression mentions the words stays; a + // nameless inline `FOREIGN KEY (` line (defensive: some MySQL + // dialects emit it) is still stripped. + let ddl = "CREATE TABLE `t` (\n `id` int NOT NULL,\n `kind` varchar(10) NOT NULL,\n CONSTRAINT `chk_kind` CHECK (`kind` <> 'foreign key'),\n FOREIGN KEY (`id`) REFERENCES `parent` (`id`)\n) ENGINE=InnoDB"; + + let stripped = strip_inline_foreign_key_constraint_lines(ddl); + + assert!(stripped.contains("CONSTRAINT `chk_kind` CHECK (`kind` <> 'foreign key')"), "{stripped}"); + assert!(!stripped.contains("REFERENCES"), "{stripped}"); + } + #[test] fn generates_deferred_mysql_foreign_key_alter_statements() { let foreign_keys = vec![db::ForeignKeyInfo { diff --git a/crates/dbx-core/tests/live_mysql_transfer.rs b/crates/dbx-core/tests/live_mysql_transfer.rs index 039f776b4f..0644bf80a1 100644 --- a/crates/dbx-core/tests/live_mysql_transfer.rs +++ b/crates/dbx-core/tests/live_mysql_transfer.rs @@ -98,6 +98,173 @@ async fn query_text(pool: &mysql::MySqlPool, sql: &str) -> String { mysql::execute_query(pool, sql, false).await.unwrap().rows[0][0].as_str().unwrap().to_string() } +#[tokio::test] +#[ignore = "requires disposable MySQL 8 endpoints via DBX_LIVE_MYSQL_TRANSFER_* variables"] +async fn live_mysql_transfer_keeps_columns_whose_comment_mentions_foreign_key() { + // #7660: the deferred-FK DDL rewrite used to delete any `SHOW CREATE + // TABLE` line containing " FOREIGN KEY ", including column definitions + // whose COMMENT text merely mentions the words — silently dropping those + // columns from the created target table. + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let source_connection_id = format!("live-mysql8-transfer-{suffix}"); + let target_connection_id = format!("live-mysql8-transfer-dst-{suffix}"); + let source_database = format!("dbx_7660_src_{}", &suffix[..12]); + let target_database = format!("dbx_7660_dst_{}", &suffix[..12]); + let source_config = live_mysql_config(&source_connection_id); + let target_config = live_mysql_config(&target_connection_id); + let source_setup_pool = mysql::connect(&mysql_url(&source_config), Duration::from_secs(10)).await.unwrap(); + let target_setup_pool = mysql::connect(&mysql_url(&target_config), Duration::from_secs(10)).await.unwrap(); + + mysql::execute_query( + &source_setup_pool, + &format!( + "CREATE DATABASE `{source_database}` CHARACTER SET utf8mb4;\ + CREATE TABLE `{source_database}`.`org` (\ + id BIGINT NOT NULL,\ + name VARCHAR(64) DEFAULT NULL,\ + PRIMARY KEY (id)\ + ) ENGINE=InnoDB;\ + CREATE TABLE `{source_database}`.`title_task` (\ + `id` varchar(32) NOT NULL,\ + `review_result` varchar(100) DEFAULT NULL,\ + `review_mode` tinyint NOT NULL DEFAULT '0' COMMENT 'foreign key of review flow',\ + `score_snapshot_json` text COMMENT 'snapshot json, see foreign key docs',\ + `task_status` int DEFAULT '0',\ + `org_id` bigint DEFAULT NULL,\ + PRIMARY KEY (`id`),\ + KEY `idx_task_status` (`task_status`),\ + KEY `fk_title_task_org` (`org_id`),\ + CONSTRAINT `fk_title_task_org` FOREIGN KEY (`org_id`) REFERENCES `{source_database}`.`org` (`id`)\ + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\ + INSERT INTO `{source_database}`.`org` VALUES (1, 'hq'), (2, 'branch');\ + INSERT INTO `{source_database}`.`title_task` VALUES \ + ('t1', 'pass', 1, 'score=9', 0, 1),\ + ('t2', NULL, 0, NULL, 1, 2)" + ), + true, + ) + .await + .unwrap(); + mysql::execute_query( + &target_setup_pool, + &format!("CREATE DATABASE `{target_database}` CHARACTER SET utf8mb4"), + false, + ) + .await + .unwrap(); + + let task_tmp = std::path::PathBuf::from( + std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"), + ); + let dir = task_tmp.join(format!("live-mysql-fk-comment-transfer-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + state.configs.write().await.insert(source_connection_id.clone(), source_config); + state.configs.write().await.insert(target_connection_id.clone(), target_config); + let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap(); + let target_pool_key = state.get_or_create_pool(&target_connection_id, Some(&target_database)).await.unwrap(); + let request = TransferRequest { + transfer_id: format!("live-mysql-fk-comment-transfer-{suffix}"), + source_connection_id, + source_database: source_database.clone(), + source_schema: source_database.clone(), + source_catalog: None, + target_connection_id, + target_database: target_database.clone(), + target_schema: target_database.clone(), + target_catalog: None, + tables: vec!["org".to_string(), "title_task".to_string()], + create_table: true, + content: TransferContent::default(), + objects: Vec::new(), + mode: TransferMode::Append, + target_table_name_case: TransferTableNameCase::Preserve, + quote_target_column_names: true, + ownership_policy: TransferOwnershipPolicy::Preserve, + batch_size: 10, + }; + + let test_result = async { + let mut pending_fk_alters: Vec<(String, String)> = Vec::new(); + assert_eq!( + transfer_table( + &state, + &request, + "org", + 0, + &DatabaseType::Mysql, + &DatabaseType::Mysql, + &source_pool_key, + &target_pool_key, + &std::collections::HashMap::new(), + &mut pending_fk_alters, + |_| {}, + ) + .await?, + 2 + ); + assert_eq!( + transfer_table( + &state, + &request, + "title_task", + 1, + &DatabaseType::Mysql, + &DatabaseType::Mysql, + &source_pool_key, + &target_pool_key, + &std::collections::HashMap::new(), + &mut pending_fk_alters, + |_| {}, + ) + .await?, + 2 + ); + assert_eq!(pending_fk_alters.len(), 1, "alters: {pending_fk_alters:?}"); + for (_, statement) in &pending_fk_alters { + dbx_core::transfer::execute_on_pool_with_max_rows(&state, &target_pool_key, statement, None).await.unwrap(); + } + + let ddl_result = mysql::execute_query( + &target_setup_pool, + &format!("SHOW CREATE TABLE `{target_database}`.`title_task`"), + false, + ) + .await?; + let target_ddl = ddl_result.rows[0][1].as_str().unwrap(); + assert!(target_ddl.contains("`review_mode` tinyint NOT NULL DEFAULT '0'"), "ddl: {target_ddl}"); + assert!(target_ddl.contains("`score_snapshot_json` text"), "ddl: {target_ddl}"); + assert!(target_ddl.contains("COMMENT 'foreign key of review flow'"), "ddl: {target_ddl}"); + assert!(target_ddl.contains("KEY `idx_task_status` (`task_status`)"), "ddl: {target_ddl}"); + assert!(target_ddl.contains("CONSTRAINT `fk_title_task_org` FOREIGN KEY"), "ddl: {target_ddl}"); + assert_eq!( + query_text( + &target_setup_pool, + &format!( + "SELECT GROUP_CONCAT(CONCAT(id, ':', review_mode, ':', IFNULL(score_snapshot_json, 'N')) \ + ORDER BY id SEPARATOR '|') FROM `{target_database}`.`title_task`" + ), + ) + .await, + "t1:1:score=9|t2:0:N" + ); + Ok::<_, String>(()) + } + .await; + + let source_cleanup = + mysql::execute_query(&source_setup_pool, &format!("DROP DATABASE `{source_database}`"), false).await; + let target_cleanup = + mysql::execute_query(&target_setup_pool, &format!("DROP DATABASE `{target_database}`"), false).await; + source_setup_pool.disconnect().await.unwrap(); + target_setup_pool.disconnect().await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + source_cleanup.unwrap(); + target_cleanup.unwrap(); + test_result.unwrap(); +} + #[tokio::test] #[ignore = "requires disposable MySQL 8 source and 5.7 target endpoints via DBX_LIVE_MYSQL_TRANSFER_SOURCE_*/TARGET_* variables"] async fn live_mysql_transfer_downgrades_unsupported_source_collations() { @@ -627,3 +794,90 @@ async fn live_mysql_transfer_structure_overwrite_rejects_incompatible_target_col cleanup.unwrap(); test_result.unwrap(); } + +#[tokio::test] +#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"] +async fn live_mysql_transfer_structure_only_rejects_incompatible_target_columns() { + // #7660: structure-only transfers onto a preexisting target used to skip + // the incompatible-structure validation entirely and report success while + // the target quietly kept missing source columns. + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let connection_id = format!("live-mysql-transfer-structonly-{suffix}"); + let source_database = format!("dbx_transfer_structonly_src_{}", &suffix[..12]); + let target_database = format!("dbx_transfer_structonly_dst_{}", &suffix[..12]); + let config = live_mysql_config(&connection_id); + let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap(); + + let setup = format!( + "CREATE DATABASE `{source_database}`;\ + CREATE DATABASE `{target_database}`;\ + CREATE TABLE `{source_database}`.`orders` (\ + id INT PRIMARY KEY, name VARCHAR(32), extra_col VARCHAR(32)\ + );\ + CREATE TABLE `{target_database}`.`orders` (\ + id INT PRIMARY KEY, name VARCHAR(32)\ + )" + ); + mysql::execute_query(&setup_pool, &setup, true).await.unwrap(); + + let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-structonly-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + state.configs.write().await.insert(connection_id.clone(), config); + let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap(); + let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap(); + + let request = TransferRequest { + transfer_id: format!("live-mysql-transfer-structonly-{suffix}"), + source_connection_id: connection_id.clone(), + source_database: source_database.clone(), + source_schema: source_database.clone(), + source_catalog: None, + target_connection_id: connection_id.clone(), + target_database: target_database.clone(), + target_schema: target_database.clone(), + target_catalog: None, + tables: vec!["orders".to_string()], + create_table: true, + content: TransferContent::StructureOnly, + objects: Vec::new(), + mode: TransferMode::Append, + target_table_name_case: TransferTableNameCase::Preserve, + quote_target_column_names: true, + ownership_policy: TransferOwnershipPolicy::Preserve, + batch_size: 10, + }; + + let test_result = async { + let result = transfer_table( + &state, + &request, + "orders", + 0, + &DatabaseType::Mysql, + &DatabaseType::Mysql, + &source_pool_key, + &target_pool_key, + &std::collections::HashMap::new(), + &mut Vec::new(), + |_| {}, + ) + .await; + let error = result.expect_err("structure-only transfer onto an incompatible preexisting target should fail"); + assert!(error.contains("missing column(s) extra_col"), "error should name the missing column: {error}"); + Ok::<_, String>(()) + } + .await; + + let cleanup = mysql::execute_query( + &setup_pool, + &format!("DROP DATABASE `{source_database}`; DROP DATABASE `{target_database}`"), + true, + ) + .await; + setup_pool.disconnect().await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + cleanup.unwrap(); + test_result.unwrap(); +}